MUSE Pipeline Reference Manual  1.0.2
muse_cube_combine.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2014 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #include <muse.h>
23 #include <string.h>
24 
25 /*----------------------------------------------------------------------------*/
56 /*----------------------------------------------------------------------------*/
57 
60 #define PRINT_USAGE(rc) \
61  fprintf(stderr, "Usage: %s CUBE_OUT CUBE_IN_1 CUBE_IN_2 [ CUBE_IN_3 ... ]\n",\
62  argv[0]); \
63  cpl_end(); return (rc);
64 
65 int main(int argc, char **argv)
66 {
67  const char *idstring = "muse_cube_combine";
68  cpl_init(CPL_INIT_DEFAULT);
69  cpl_msg_set_level(CPL_MSG_DEBUG);
70  cpl_msg_set_time_on();
71  cpl_msg_set_component_on();
72  cpl_errorstate state = cpl_errorstate_get();
73 
74  if (argc < 4) {
75  /* three filenames are needed at least */
76  PRINT_USAGE(1);
77  }
78 
79  char *oname = NULL; /* output cube */
80  int i, noverlap = 1;
81 
82  /* argument processing */
83  for (i = 1; i < argc; i++) {
84  if (strncmp(argv[i], "-xxx", 5) == 0) { // XXX
85  /* skip to next arg to get start value */
86  i++;
87 // if (i < argc) {
88 // x1 = atof(argv[i]);
89 // } else {
90 // PRINT_USAGE(2);
91 // }
92  } else if (strncmp(argv[i], "-", 1) == 0) { /* unallowed options */
93  PRINT_USAGE(9);
94  } else {
95  if (oname) {
96  break; /* we have the required name, skip the rest */
97  }
98  oname = argv[i];
99  }
100  } /* for i (all arguments) */
101  if (!oname) {
102  PRINT_USAGE(10);
103  }
104  FILE *fp = fopen(oname, "r");
105  if (fp) {
106  cpl_msg_error(idstring, "Output cube \"%s\" is already present!", oname);
107  cpl_msg_error(idstring, "Please specify another output name or rename the "
108  "existing file with this name.");
109  fclose(fp);
110  PRINT_USAGE(11);
111  }
112 
113  int ncubes = argc - i;
114  cpl_msg_info(idstring, "Will write combination of %d cubes to \"%s\".",
115  ncubes, oname);
116 
117  /* create table of the cubes involved here */
118  cpl_table *table = cpl_table_new(ncubes);
119  cpl_table_new_column(table, "FILENAME", CPL_TYPE_STRING);
120  cpl_table_new_column(table, "LMIN", CPL_TYPE_DOUBLE);
121  cpl_table_new_column(table, "LMAX", CPL_TYPE_DOUBLE);
122  cpl_table_new_column(table, "P1", CPL_TYPE_INT);
123  cpl_table_new_column(table, "P2", CPL_TYPE_INT);
124 
125  /* check ranges and grids of the cubes involved, save in the table */
126  char *ctype1ref = NULL, *ctype2ref = NULL,
127  *cunit1ref = NULL, *cunit2ref = NULL;
128  double crpix1ref = NAN, crpix2ref = NAN, crval1ref = NAN, crval2ref = NAN,
129  cd11ref = NAN, cd12ref = NAN, cd21ref = NAN, cd22ref = NAN;
130  char *ctype3ref = NULL;
131  double crpix3ref = NAN,
132  crval3ref = NAN,
133  cd33ref = NAN;
134  int ic;
135  for (ic = 0; ic < ncubes; ic++) {
136  int argidx = ic + 2;
137  char *iname = argv[argidx];
138  /* search for the data extension */
139  int iext = cpl_fits_find_extension(iname, "DATA");
140  if (iext < 0) {
141  cpl_msg_warning(idstring, "Could not open cube \"%s\"", iname);
142  continue;
143  }
144 
145  /* load primary header and get lambda ranges */
146  cpl_propertylist *header = cpl_propertylist_load(iname, 0);
147  double lmin = -FLT_MAX,
148  lmax = FLT_MAX;
149  /* loop through the recipe parameters stored in the cube */
150  cpl_errorstate es = cpl_errorstate_get();
151  int ipar = 0;
152  do {
153  ipar++;
154  char *hname = cpl_sprintf("ESO PRO REC1 PARAM%d NAME", ipar);
155  const char *name = cpl_propertylist_get_string(header, hname);
156  cpl_free(hname);
157  cpl_boolean islmin = name && !strncmp(name, "lambdamin", 10),
158  islmax = name && !strncmp(name, "lambdamax", 10);
159  if (islmin || islmax) {
160  char *hval = cpl_sprintf("ESO PRO REC1 PARAM%d VALUE", ipar);
161  if (islmin) {
162  lmin = atof(cpl_propertylist_get_string(header, hval));
163  cpl_msg_debug(idstring, "lmin = %f", lmin);
164  } else {
165  lmax = atof(cpl_propertylist_get_string(header, hval));
166  cpl_msg_debug(idstring, "lmax = %f", lmax);
167  }
168  cpl_free(hval);
169  } /* if */
170  } while (cpl_errorstate_is_equal(es));
171  cpl_errorstate_set(es);
172  cpl_propertylist_delete(header);
173 
174  /* load DATA header and get lambda solution */
175  header = cpl_propertylist_load(iname, iext);
176  double crpix3 = cpl_propertylist_get_double(header, "CRPIX3"),
177  crval3 = cpl_propertylist_get_double(header, "CRVAL3"),
178  cd33 = cpl_propertylist_get_double(header, "CD3_3");
179  int naxis3 = cpl_propertylist_get_int(header, "NAXIS3");
180 
181  const char *cunit3 = cpl_propertylist_get_string(header, "CUNIT3"),
182  *lunit = cpl_table_get_column_unit(table, "LMIN");
183  if (!lunit) {
184  cpl_table_set_column_unit(table, "LMIN", cunit3);
185  cpl_table_set_column_unit(table, "LMAX", cunit3);
186  } else if (strncmp(lunit, cunit3, strlen(lunit) + 1)) {
187  cpl_msg_warning(idstring, "Cube %d does not contain wavelengths in "
188  "%s (%s), skipping it", ic + 1, lunit, cunit3);
189  cpl_propertylist_delete(header);
190  continue;
191  }
192  const char *ctype3 = cpl_propertylist_get_string(header, "CTYPE3");
193  if (!ctype3ref) {
194  ctype3ref = cpl_strdup(ctype3);
195  crpix3ref = crpix3;
196  crval3ref = crval3;
197  cd33ref = cd33;
198  } else if (strncmp(ctype3ref, ctype3, strlen(ctype3ref)) ||
199  fabs(crpix3 - crpix3ref) > DBL_EPSILON ||
200  fabs(crval3 - crval3ref) > DBL_EPSILON ||
201  fabs(cd33 - cd33ref) > DBL_EPSILON) {
202  cpl_msg_warning(idstring, "Cube %d does not match in spectral WCS, "
203  "skipping it", ic + 1);
204  cpl_propertylist_delete(header);
205  continue;
206  }
207  double lpx1 = (1. - crpix3) * cd33 + crval3,
208  lpx2 = (naxis3 - crpix3) * cd33 + crval3;
209  cpl_msg_debug(idstring, "Cube %d, axis 3 WCS: %f %f %f (%s, %d pixels, "
210  "%f..%f %s)", ic + 1, crpix3, crval3, cd33, ctype3, naxis3,
211  lpx1, lpx2, cunit3);
212  /* check min/max wavelengths against boundary conditions */
213  if (lmin < lpx1) {
214  lmin = lpx1;
215  }
216  if (lmax > lpx2) {
217  lmax = lpx2;
218  }
219 
220  /* check spatial WCS against first cube */
221  const char *ctype1 = cpl_propertylist_get_string(header, "CTYPE1"),
222  *ctype2 = cpl_propertylist_get_string(header, "CTYPE2"),
223  *cunit1 = cpl_propertylist_get_string(header, "CUNIT1"),
224  *cunit2 = cpl_propertylist_get_string(header, "CUNIT2");
225  double crpix1 = cpl_propertylist_get_double(header, "CRPIX1"),
226  crpix2 = cpl_propertylist_get_double(header, "CRVAL2"),
227  crval1 = cpl_propertylist_get_double(header, "CRVAL1"),
228  crval2 = cpl_propertylist_get_double(header, "CRVAL2"),
229  cd11 = cpl_propertylist_get_double(header, "CD1_1"),
230  cd12 = cpl_propertylist_get_double(header, "CD1_2"),
231  cd21 = cpl_propertylist_get_double(header, "CD2_1"),
232  cd22 = cpl_propertylist_get_double(header, "CD2_2");
233  if (!ctype1ref) {
234  ctype1ref = cpl_strdup(ctype1);
235  ctype2ref = cpl_strdup(ctype2);
236  cunit1ref = cpl_strdup(cunit1);
237  cunit2ref = cpl_strdup(cunit2);
238  crpix1ref = crpix1;
239  crval1ref = crval1;
240  crpix2ref = crpix2;
241  crval2ref = crval2;
242  cd11ref = cd11;
243  cd12ref = cd12;
244  cd21ref = cd21;
245  cd22ref = cd22;
246  } else if (strncmp(ctype1ref, ctype1, strlen(ctype1ref)) ||
247  strncmp(ctype2ref, ctype2, strlen(ctype2ref)) ||
248  strncmp(cunit1ref, cunit1, strlen(cunit1ref)) ||
249  strncmp(cunit2ref, cunit2, strlen(cunit2ref)) ||
250  fabs(crpix1 - crpix1ref) > DBL_EPSILON ||
251  fabs(crval1 - crval1ref) > DBL_EPSILON ||
252  fabs(crpix2 - crpix2ref) > DBL_EPSILON ||
253  fabs(crval2 - crval2ref) > DBL_EPSILON ||
254  fabs(cd11 - cd11ref) > DBL_EPSILON ||
255  fabs(cd12 - cd12ref) > DBL_EPSILON ||
256  fabs(cd21 - cd21ref) > DBL_EPSILON ||
257  fabs(cd22 - cd22ref) > DBL_EPSILON) {
258  cpl_msg_warning(idstring, "Cube %d does not match in spatial WCS, "
259  "skipping it", ic + 1);
260  cpl_propertylist_delete(header);
261  continue;
262  }
263  cpl_propertylist_delete(header);
264 
265  /* compute good valid range of pixels in dispersion direction */
266  int l1 = lround((lmin - crval3) / cd33 + crpix3),
267  l2 = lround((lmax - crval3) / cd33 + crpix3);
268  cpl_msg_debug(idstring, "Cube %d: %d..%d (%f..%f Angstrom)", ic + 1,
269  l1, l2, lmin, lmax);
270  if (l1 <= 1) {
271  l1 = 1;
272  } else if (l1 >= naxis3) {
273  cpl_msg_warning(idstring, "Something is wrong with cube %d: l1 = %d!",
274  ic + 1, l1);
275  continue;
276  } else {
277  /* throw away plane(s) from the starting edge to mitigate resampling edge effects */
278  l1 += noverlap;
279  }
280  if (l2 >= naxis3) {
281  l2 = naxis3;
282  } else if (l2 <= 1) {
283  cpl_msg_warning(idstring, "Something is wrong with cube %d: l2 = %d!",
284  ic + 1, l2);
285  continue;
286  } else {
287  l2 -= noverlap; /* throw away plane(s) from the ending edge */
288  }
289  cpl_msg_debug(idstring, "Cube %d: ---> %d..%d", ic + 1, l1, l2);
290 
291  /* all was OK, so we can save info of this cube to the table */
292  cpl_table_set_string(table, "FILENAME", ic, iname);
293  cpl_table_set(table, "LMIN", ic, lmin);
294  cpl_table_set(table, "LMAX", ic, lmax);
295  cpl_table_set(table, "P1", ic, l1);
296  cpl_table_set(table, "P2", ic, l2);
297  } /* for ic (all input cubes) */
298  cpl_free(ctype1ref);
299  cpl_free(ctype2ref);
300  cpl_free(cunit1ref);
301  cpl_free(cunit2ref);
302  cpl_free(ctype3ref);
303 
304  /* erase bad entries (cubes with missing properties) */
305  cpl_table_select_all(table);
306  cpl_table_and_selected_invalid(table, "FILENAME");
307  cpl_table_erase_selected(table);
308  ncubes = cpl_table_get_nrow(table);
309  if (ncubes < 2) {
310  cpl_msg_error(idstring, "Only %d cube%s with valid info %s found!", ncubes,
311  ncubes ? "" : "s", ncubes ? "was" : "were");
312  cpl_end();
313  return 12;
314  }
315 
316  /* now sort the table by increasing first plane */
317  cpl_propertylist *order = cpl_propertylist_new();
318  cpl_propertylist_append_bool(order, "P1", CPL_FALSE);
319  cpl_table_sort(table, order);
320  cpl_propertylist_delete(order);
321 #if 1
322  printf("Stored and sorted %d cubes:\n", ncubes);
323  cpl_table_dump(table, 0, ncubes, stdout);
324  fflush(stdout);
325 #endif
326 
327  /* clean up overlaps and gaps: *
328  * - if there are overlaps just move the 2nd cube to start at a *
329  * higher plane *
330  * - if there is a gap just move the 1st cube to be more extended, *
331  * even though this will incur resampling edge effects (warn!) */
332  int nover = 0, ngaps = 0;
333  for (ic = 1; ic < cpl_table_get_nrow(table); ic++) { /* start at 2nd row */
334  int p2a = cpl_table_get_int(table, "P2", ic - 1, NULL),
335  p1b = cpl_table_get_int(table, "P1", ic, NULL);
336  double lmaxa = cpl_table_get_double(table, "LMAX", ic - 1, NULL);
337  int pdiff = p1b - p2a;
338  if (pdiff == 1) { /* no gap, no overlap */
339  continue;
340  }
341  if (pdiff < 1) { /* overlap */
342  p2a += pdiff - 1;
343  lmaxa += cd33ref * (pdiff - 1);
344  nover++;
345  } /* if overlap */
346  if (pdiff > 1) { /* gap */
347  p2a += pdiff - 1;
348  lmaxa += cd33ref * (pdiff - 1);
349  ngaps++;
350  } /* if gap */
351  cpl_table_set_int(table, "P2", ic - 1, p2a);
352  cpl_table_set_double(table, "LMAX", ic - 1, lmaxa);
353  } /* for ic (all other valid cubes) */
354 #if 1
355  printf("Cleaned up %d overlaps and %d gaps.\n", nover, ngaps);
356  cpl_table_dump(table, 0, ncubes, stdout);
357  fflush(stdout);
358 #endif
359 
360  /* now create the initial cube from the first exposure in the table */
361  muse_datacube *cube = cpl_calloc(1, sizeof(muse_datacube));
362  const char *fn = cpl_table_get_string(table, "FILENAME", 0);
363  cpl_msg_info(idstring, "Loading cube from \"%s\"", fn);
364  cube->header = cpl_propertylist_load(fn, 0);
365  /* find DATA extension */
366  int iext = cpl_fits_find_extension(fn, "DATA");
367  /* load the WCS from the first cube extension *
368  * and merge it into the cube header */
369  cpl_propertylist *wcs = cpl_propertylist_load(fn, iext);
370  cpl_propertylist_erase_regexp(wcs, MUSE_WCS_KEYS, 1); /* remove others */
371  cpl_propertylist_append(cube->header, wcs);
372  cpl_propertylist_delete(wcs);
373  /* load the cubes from the DATA and STAT extensions */
374  cube->data = cpl_imagelist_load(fn, CPL_TYPE_FLOAT, iext);
375  iext = cpl_fits_find_extension(fn, "STAT");
376  cube->stat = cpl_imagelist_load(fn, CPL_TYPE_FLOAT, iext);
377 
378  /* now create flags vector (starting from array) to erase unused planes */
379  int nplanes = cpl_imagelist_get_size(cube->data),
380  l1 = cpl_table_get_int(table, "P1", 0, NULL),
381  l2 = cpl_table_get_int(table, "P2", 0, NULL);
382  cpl_array *aflags = cpl_array_new(nplanes, CPL_TYPE_DOUBLE);
383  cpl_array_fill_window_double(aflags, 0, nplanes, -1.); /* all bad */
384  cpl_array_fill_window_double(aflags, l1 - 1, l2 - l1 + 1, 1.); /* the good ones */
385 #if 0
386  cpl_array_dump(aflags, 0, 5, stdout);
387  cpl_array_dump(aflags, l1 - 4, 10, stdout);
388  cpl_array_dump(aflags, l2 - 4, 10, stdout);
389  cpl_array_dump(aflags, nplanes - 5, 6, stdout);
390  fflush(stdout);
391 #endif
392  cpl_vector *vflags = cpl_vector_wrap(nplanes, cpl_array_unwrap(aflags));
393  cpl_imagelist_erase(cube->data, vflags);
394  cpl_imagelist_erase(cube->stat, vflags);
395  cpl_vector_delete(vflags);
396  cpl_msg_debug(idstring, "%d of %d planes left in first cube",
397  (int)cpl_imagelist_get_size(cube->data), nplanes);
398  nplanes = cpl_imagelist_get_size(cube->data);
399  /* update the spectral part of the WCS with the new setup */
400  cpl_propertylist_update_double(cube->header, "CRPIX3", 1.);
401  cpl_propertylist_update_double(cube->header, "CRVAL3",
402  cpl_table_get(table, "LMIN", 0, NULL));
403 
404  /* one-by-one fully load the other cubes, and copy *
405  * the valid data from them into the output cube */
406  for (ic = 1; ic < cpl_table_get_nrow(table); ic++) {
407  fn = cpl_table_get_string(table, "FILENAME", ic);
408  cpl_msg_info(idstring, "Loading cube from \"%s\"", fn);
409  iext = cpl_fits_find_extension(fn, "DATA");
410  cpl_imagelist *cubedata = cpl_imagelist_load(fn, CPL_TYPE_FLOAT, iext);
411  iext = cpl_fits_find_extension(fn, "STAT");
412  cpl_imagelist *cubestat = cpl_imagelist_load(fn, CPL_TYPE_FLOAT, iext);
413 
414  /* simply append (copy) the valid image planes to the output cube imagelists */
415  l1 = cpl_table_get_int(table, "P1", ic, NULL);
416  l2 = cpl_table_get_int(table, "P2", ic, NULL);
417  int l;
418  for (l = l1 - 1; l < l2; l++, nplanes = cpl_imagelist_get_size(cube->data)) {
419  cpl_error_code rc1, rc2;
420  rc1 = cpl_imagelist_set(cube->data,
421  cpl_image_duplicate(cpl_imagelist_get(cubedata, l)),
422  nplanes);
423  rc2 = cpl_imagelist_set(cube->stat,
424  cpl_image_duplicate(cpl_imagelist_get(cubestat, l)),
425  nplanes);
426  if (rc1 != rc2 || rc1 != CPL_ERROR_NONE || rc2 != CPL_ERROR_NONE) {
427  cpl_msg_warning(idstring, "Could not append plane %d of cube %d to output cube",
428  l, ic + 1);
429  } /* if bad result */
430  } /* for l (wavelength planes to copy) */
431  cpl_imagelist_delete(cubedata);
432  cpl_imagelist_delete(cubestat);
433  cpl_msg_debug(idstring, "output cube now has %d/%d planes after copying %d "
434  "planes from cube %d", nplanes,
435  (int)cpl_imagelist_get_size(cube->data), l2 - l1 + 1, ic + 1);
436  } /* for ic (all other valid cubes) */
437  cpl_table_delete(table);
438 
439  /* clean up the output header of the recipe options that are now obsolete */
440  cpl_propertylist_erase_regexp(cube->header, "ESO PRO REC1 ", 0);
441 
442  int rc = muse_datacube_save(cube, oname);
443  muse_datacube_delete(cube);
444  if (rc == CPL_ERROR_NONE) {
445  cpl_msg_info(idstring, "Saved cube to \"%s\".", oname);
446  } else {
447  cpl_msg_error(idstring, "Error while saving cube to \"%s\": %s (%d)", oname,
448  cpl_error_get_message(), rc);
449  rc = 20;
450  }
451 
452  if (!cpl_errorstate_is_equal(state)) {
453  cpl_errorstate_dump(state, CPL_FALSE, muse_cplerrorstate_dump_some);
454  rc = 50;
455  }
456  cpl_memory_dump();
457  cpl_end();
458  return rc;
459 }
460 
Structure definition of a MUSE datacube.
Definition: muse_datacube.h:47
void muse_datacube_delete(muse_datacube *aCube)
Deallocate memory associated to a muse_datacube object.
cpl_error_code muse_datacube_save(muse_datacube *aCube, const char *aFilename)
Save the three cube extensions and the FITS headers of a MUSE datacube to a file. ...
void muse_cplerrorstate_dump_some(unsigned aCurrent, unsigned aFirst, unsigned aLast)
Dump some CPL errors.
cpl_imagelist * data
the cube containing the actual data values
Definition: muse_datacube.h:75
cpl_propertylist * header
the FITS header
Definition: muse_datacube.h:56
cpl_imagelist * stat
the cube containing the data variance
Definition: muse_datacube.h:85