MUSE Pipeline Reference Manual  1.0.2
muse_twilight.c
1 /* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim:set sw=2 sts=2 et cin: */
3 /*
4  * This file is part of the MUSE Instrument Pipeline
5  * Copyright (C) 2014 European Southern Observatory
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  */
21 
22 #ifdef HAVE_CONFIG_H
23 #include <config.h>
24 #endif
25 
26 /*----------------------------------------------------------------------------*
27  * Includes *
28  *----------------------------------------------------------------------------*/
29 #include <string.h>
30 
31 #include <muse.h>
32 #include "muse_twilight_z.h"
33 
34 /*---------------------------------------------------------------------------*
35  * Functions code *
36  *---------------------------------------------------------------------------*/
37 
38 /*---------------------------------------------------------------------------*/
45 /*---------------------------------------------------------------------------*/
46 static cpl_error_code
47 muse_twilight_qc_header(muse_image *aImage, muse_imagelist *aList)
48 {
49  cpl_ensure_code(aImage && aList, CPL_ERROR_NULL_INPUT);
50  unsigned char ifu = muse_utils_get_ifu(aImage->header);
51  cpl_msg_debug(__func__, "Adding QC keywords for IFU %hhu", ifu);
52 
53  unsigned stats = CPL_STATS_MEDIAN | CPL_STATS_MEAN | CPL_STATS_STDEV
54  | CPL_STATS_MIN | CPL_STATS_MAX;
55 
56  /* write the image statistics for input skyflat field exposures */
57  unsigned int k;
58  for (k = 0; k < muse_imagelist_get_size(aList); k++) {
59  char *keyword = cpl_sprintf(QC_TWILIGHTm_PREFIXi, ifu, k+1);
61  aImage->header, keyword, stats);
62  cpl_free(keyword);
63  keyword = cpl_sprintf(QC_TWILIGHTm_PREFIXi" "QC_BASIC_NSATURATED, ifu, k+1);
64  int nsaturated = cpl_propertylist_get_int(muse_imagelist_get(aList, k)->header,
65  MUSE_HDR_TMP_NSAT);
66  cpl_propertylist_update_int(aImage->header, keyword, nsaturated);
67  cpl_free(keyword);
68  } /* for k (all images in list) */
69 
70  /* properties of the resulting master frame */
71  char *kw = cpl_sprintf(QC_TWILIGHTm_MASTER_PREFIX, ifu);
72  stats |= CPL_STATS_FLUX;
73  muse_basicproc_stats_append_header(aImage->data, aImage->header, kw, stats);
74  cpl_free(kw);
75  return CPL_ERROR_NONE;
76 } /* muse_twilight_qc_header() */
77 
78 /*----------------------------------------------------------------------------*/
120 /*----------------------------------------------------------------------------*/
121 static muse_datacube *
122 muse_twilight_reconstruct(muse_processing *aProcessing,
123  muse_twilight_params_t *aParams,
124  muse_basicproc_params *aBPars,
125  muse_combinepar *aCPars,
126  muse_resampling_params *aRPars)
127 {
128  cpl_ensure(aProcessing && aParams, CPL_ERROR_NULL_INPUT, NULL);
129  double lmin = aParams->lambdamin,
130  lmax = aParams->lambdamax;
131  cpl_ensure(lmax > lmin, CPL_ERROR_ILLEGAL_INPUT, NULL);
132  /* the geometry table is common for all IFUs */
133  cpl_table *geo = muse_table_load(aProcessing, MUSE_TAG_GEOMETRY_TABLE, 0);
134  cpl_ensure(geo, CPL_ERROR_FILE_NOT_FOUND, NULL);
135 
136  muse_pixtable **pts = cpl_calloc(kMuseNumIFUs, sizeof(muse_pixtable));
137  unsigned char nifu;
138  #pragma omp parallel for default(none) /* as req. by Ralf */ \
139  shared(aBPars, aCPars, aProcessing, geo, pts)
140  for (nifu = 1; nifu <= kMuseNumIFUs; nifu++) {
141  cpl_table *trace = muse_table_load(aProcessing, MUSE_TAG_TRACE_TABLE, nifu),
142  *wavecal = muse_table_load(aProcessing, MUSE_TAG_WAVECAL_TABLE, nifu);
143  if (!trace || !wavecal) {
144  cpl_msg_warning(__func__, "Calibrations could not be loaded for IFU "
145  "%2hhu:%s%s", nifu, !trace ? " "MUSE_TAG_TRACE_TABLE : "",
146  !wavecal ? " "MUSE_TAG_WAVECAL_TABLE : "");
147  cpl_table_delete(trace);
148  cpl_table_delete(wavecal);
149  continue; /* skip this IFU */
150  }
151  /* load and pre-process all raw files */
152  cpl_msg_debug(__func__, "load raw files of IFU %2hhu", nifu);
153  muse_imagelist *images = muse_basicproc_load(aProcessing, nifu, aBPars);
154  if (!images) {
155  cpl_msg_warning(__func__, "Basic processing failed for IFU %2hhu: %s",
156  nifu, cpl_error_get_message());
157  cpl_table_delete(trace);
158  cpl_table_delete(wavecal);
159  continue;
160  }
161  /* combine the raw exposures into a master image */
162  cpl_msg_debug(__func__, "combine raw files of IFU %2hhu", nifu);
163  muse_image *masterimage = muse_combine_images(aCPars, images);
164  muse_twilight_qc_header(masterimage, images);
165  muse_imagelist_delete(images);
166 
167  /* apply bad-pixel interpolation on this image, so that the *
168  * per-IFU integrated fluxes are independent of bad pixels */
169  int nbad = muse_quality_image_reject_using_dq(masterimage->data,
170  masterimage->dq,
171  masterimage->stat);
172  cpl_detector_interpolate_rejected(masterimage->data);
173  /* doing this for the variance is ugly but we cannot do much else */
174  cpl_detector_interpolate_rejected(masterimage->stat);
175  /* still leave them marked in the DQ extension */
176  cpl_msg_debug(__func__, "interpolated over %d bad pixels in IFU %2hhu",
177  nbad, nifu);
178 
179  /* create pixel table for this master image */
180  cpl_msg_debug(__func__, "create pixel table for IFU %2hhu", nifu);
181  pts[nifu - 1] = muse_pixtable_create(masterimage, trace, wavecal, geo);
182  cpl_table_delete(trace);
183  cpl_table_delete(wavecal);
184  /* propagate the QC parameters from the image header */
185  if (pts[nifu - 1]) {
186  cpl_propertylist_copy_property_regexp(pts[nifu - 1]->header,
187  masterimage->header,
188  QC_TWILIGHT_REGEXP, 0);
189  } /* if pixel table created */
190  muse_image_delete(masterimage);
191  } /* for nifu (all IFUs) */
192  cpl_table_delete(geo);
193 
194  muse_pixtable *pt = pts[0];
195  muse_pixtable_restrict_wavelength(pt, lmin, lmax);
196  if (muse_pixtable_get_nrow(pt) < 1) {
197  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_OUTPUT,
198  "After cutting to %.2f..%.2f Angstrom in wavelength "
199  "no pixels for cube reconstruction are left!", lmin,
200  lmax);
201  for (nifu = 1; nifu <= kMuseNumIFUs; nifu++) {
202  muse_pixtable_delete(pts[nifu - 1]);
203  } /* for nifu (all IFUs) */
204  cpl_free(pts);
205  return NULL;
206  } /* if */
207  double fluxref = cpl_table_get_column_mean(pt->table, MUSE_PIXTABLE_DATA)
209  cpl_msg_debug(__func__, "pixel table of IFU 1, fluxref = %e, %"CPL_SIZE_FORMAT
210  " rows", fluxref, muse_pixtable_get_nrow(pt));
211  /* merge all pixel tables a la muse_pixtable_load_merge_channels() *
212  * but first restricting the wavelength range and with flux *
213  * correction using the data we have in the table */
214  cpl_propertylist_append_double(pt->header, MUSE_HDR_FLAT_FLUX_SKY"1", fluxref);
215  char *kw = cpl_sprintf(QC_TWILIGHTm_INTFLUX, 1);
216  cpl_propertylist_append_float(pt->header, kw, fluxref);
217  cpl_free(kw);
218  for (nifu = 2; nifu <= kMuseNumIFUs; nifu++) {
219  if (!pts[nifu - 1]) {
220  continue;
221  } /* if */
222  muse_pixtable_restrict_wavelength(pts[nifu - 1], lmin, lmax);
223  if (muse_pixtable_get_nrow(pts[nifu - 1]) < 1) {
224  continue;
225  } /* if */
226  double flux = cpl_table_get_column_mean(pts[nifu - 1]->table, MUSE_PIXTABLE_DATA)
227  * muse_pixtable_get_nrow(pts[nifu - 1]),
228  scale = flux / fluxref;
229  kw = cpl_sprintf(MUSE_HDR_FLAT_FLUX_SKY"%hhu", nifu);
230  cpl_propertylist_append_double(pt->header, kw, flux);
231  cpl_free(kw);
232  cpl_msg_debug(__func__, "merging pixel table of IFU %2hhu, flux = %e, scale = %f",
233  nifu, flux, scale);
234  cpl_table_divide_scalar(pts[nifu - 1]->table, MUSE_PIXTABLE_DATA, scale);
235  cpl_table_divide_scalar(pts[nifu - 1]->table, MUSE_PIXTABLE_STAT, scale*scale);
236  cpl_table_insert(pt->table, pts[nifu - 1]->table, muse_pixtable_get_nrow(pt));
237  /* again, propagate the QC parameters to the merged pixel table */
238  cpl_propertylist_copy_property_regexp(pt->header, pts[nifu - 1]->header,
239  QC_TWILIGHT_REGEXP, 0);
240  /* add the pixel-table based flux as QC as well */
241  kw = cpl_sprintf(QC_TWILIGHTm_INTFLUX, nifu);
242  cpl_propertylist_append_float(pt->header, kw, flux);
243  cpl_free(kw);
244  muse_pixtable_delete(pts[nifu - 1]);
245  } /* for nifu (all IFUs) */
246  cpl_free(pts);
247  /* reset limits in the header, necessary due to manual operation on table data */
249 
250  /* now just reconstruct the cube and collapse over the full range */
251  muse_datacube *cube = muse_resampling_cube(pt, aRPars, NULL);
253  if (!cube) {
254  return NULL;
255  }
256  /* also create a corresponding white-light image */
257  muse_image *white = muse_datacube_collapse(cube, NULL);
258  cube->recimages = muse_imagelist_new();
259  cube->recnames = cpl_array_new(1, CPL_TYPE_STRING);
260  muse_imagelist_set(cube->recimages, white, 0);
261  cpl_array_set_string(cube->recnames, 0, "white");
262 
263  /* want NANs instead of zeros, and save space */
265 
266  return cube;
267 } /* muse_twilight_reconstruct() */
268 
269 /*----------------------------------------------------------------------------*/
293 /*----------------------------------------------------------------------------*/
294 static cpl_image *
295 muse_twilight_image_normalize_and_fit(muse_image *aImage,
296  unsigned int aXOrder,
297  unsigned int aYOrder,
298  const cpl_mask *aBPM,
299  const cpl_mask *aVign)
300 {
301  cpl_ensure(aImage && aImage->data, CPL_ERROR_NULL_INPUT, NULL);
302 
303  /* reject values from the white-light mask, and extra infinite pixels */
304  if (aBPM) {
305  cpl_image_reject_from_mask(aImage->data, aBPM);
306  }
307  cpl_image_reject_value(aImage->data, CPL_VALUE_NOTFINITE | CPL_VALUE_ZERO);
308 
309  /* filter the image to remove outliers (like strongly deviant slices) */
310  /* use a 5x7 median filter */
311  cpl_image *filtered = cpl_image_duplicate(aImage->data);
312  cpl_mask *mask = cpl_mask_new(5, 7);
313  cpl_mask_not(mask);
314  cpl_errorstate prestate = cpl_errorstate_get();
315  cpl_image_filter_mask(filtered, aImage->data, mask, CPL_FILTER_MEDIAN,
316  CPL_BORDER_FILTER);
317  cpl_mask_delete(mask);
318  if (!cpl_errorstate_is_equal(prestate)) {
319  cpl_msg_warning(__func__, "filtering cube plane failed: %s",
320  cpl_error_get_message());
321  }
322 #if 0
323  /* now also use a wide (~10 pixel FWHM) 2D Gaussian to smooth the output even more */
324  cpl_image *filtered2 = cpl_image_duplicate(filtered);
325  cpl_matrix *gauss = muse_matrix_new_gaussian_2d(7, 7, 10. * CPL_MATH_SIG_FWHM);
326  cpl_image_filter(filtered2, filtered, gauss, CPL_FILTER_LINEAR,
327  CPL_BORDER_FILTER);
328  cpl_matrix_delete(gauss);
329  if (!cpl_errorstate_is_equal(prestate)) {
330  cpl_msg_warning(__func__, "filtering 2 failed: %s", cpl_error_get_message());
331  } else {
332  cpl_image_reject_from_mask(filtered2, bpm2);
333  cpl_image_fill_rejected(filtered2, 1.);
334  muse_processing_save_cimage(aProcessing, -1, filtered2, wmuse->header,
335  "TWILIGHT_IMAGE2_SMOOTHED" /* XXX */);
336  cpl_detector_interpolate_rejected(filtered2);
337  muse_processing_save_cimage(aProcessing, -1, filtered2, wmuse->header,
338  "TWILIGHT_IMAGE2_INTERPOLATED" /* XXX */);
339  }
340  cpl_image_delete(filtered2);
341 #endif
342 
343  /* compute the mean of the good pixels and use that to normalize the image */
344  double mean = cpl_image_get_mean(filtered);
345  cpl_msg_debug(__func__, "mean = %f", mean);
346  cpl_image_multiply_scalar(filtered, 1. / mean);
347 
348  /* mask out the vignetted area (if given), before applying the fit */
349  cpl_mask *bpm = cpl_image_get_bpm(filtered);
350  if (aVign) {
351  cpl_mask_or(bpm, aVign);
352  }
353  cpl_image *fit = muse_utils_image_fit_polynomial(filtered,
354  aXOrder, aYOrder);
355  /* normalize again */
356  mean = cpl_image_get_mean(fit);
357  cpl_msg_debug(__func__, "mean(fit) = %f", mean);
358  cpl_image_multiply_scalar(fit, 1. / mean);
359  cpl_image_delete(filtered);
360  return fit;
361 } /* muse_twilight_image_normalize_and_fit() */
362 
363 /*---------------------------------------------------------------------------*/
370 /*---------------------------------------------------------------------------*/
371 int
372 muse_twilight_compute(muse_processing *aProcessing,
373  muse_twilight_params_t *aParams)
374 {
375  muse_datacube *cube = NULL,
376  *cubefit = cpl_calloc(1, sizeof(muse_datacube));
377  cubefit->data = cpl_imagelist_new();
378  if (getenv("MUSE_TWILIGHT_SKIP")) {
379  char *fn = getenv("MUSE_TWILIGHT_SKIP");
380  cube = muse_datacube_load(fn);
381  cpl_msg_warning(__func__, "Skipping active: loaded previously generated "
382  "cube from \"%s\"", fn);
383  /* ensure that the first collapse image is "white" */
384  if (!cube->recimages) { /* no reconstructed images at all */
385  /* create the white-light image */
386  cube->recimages = muse_imagelist_new();
387  cube->recnames = cpl_array_new(1, CPL_TYPE_STRING);
388  muse_image *white = muse_datacube_collapse(cube, NULL);
389  muse_imagelist_set(cube->recimages, white, 0);
390  cpl_array_set_string(cube->recnames, 0, "white");
391  } else if (!cube->recnames ||
392  (cube->recnames &&
393  strncmp("white", cpl_array_get_string(cube->recnames, 0), 6))) {
394  /* replace first reconstructed image with white-light */
395  muse_image *white = muse_datacube_collapse(cube, NULL);
396  muse_imagelist_set(cube->recimages, white, 0);
397  cpl_array_set_string(cube->recnames, 0, "white");
398  } /* else */
399 
400  /* add something to the usedframes in aProcessing for saving below to work */
401  cpl_frameset *frames = muse_frameset_find_tags(aProcessing->inframes,
402  aProcessing->intags, -1, 0);
403  cpl_size iframe, nframes = cpl_frameset_get_size(frames);
404  for (iframe = 0; iframe < nframes; iframe++) {
405  cpl_frame *frame = cpl_frameset_get_position(frames, iframe);
406  muse_processing_append_used(aProcessing, frame, CPL_FRAME_GROUP_RAW, 1);
407  } /* for iframe (all raw inputs) */
408  cpl_frameset_delete(frames);
409  cpl_msg_warning(__func__, "Skipping active: faked used frames using only "
410  "raw inputs:");
411  cpl_frameset_dump(aProcessing->usedframes, stdout);
412  fflush(stdout);
413  cubefit->header = cpl_propertylist_duplicate(cube->header);
414  } /* if skip */
415 
416  cpl_error_code rc = CPL_ERROR_NONE;
417  if (!cube) { /* skipping did not happen */
419  "muse.muse_twilight");
420  muse_combinepar *cpars = muse_combinepar_new(aProcessing->parameters,
421  "muse.muse_twilight");
422  muse_resampling_type resample
425  rp->crtype = muse_postproc_get_cr_type(aParams->crtype_s);
426  rp->crsigma = aParams->crsigma;
427  rp->dlambda = aParams->dlambda;
428  rp->pfx = 1.;
429  rp->pfy = 1.;
430  rp->pfl = 1.;
431  cube = muse_twilight_reconstruct(aProcessing, aParams, bpars, cpars, rp);
433  muse_combinepar_delete(cpars);
435  if (!cube) {
436  muse_datacube_delete(cubefit);
437  return -1;
438  }
439  /* duplicate the cube header and its WCS before saving destroys the WCS */
440  cubefit->header = cpl_propertylist_duplicate(cube->header);
441  rc = muse_processing_save_cube(aProcessing, -1, cube,
442  MUSE_TAG_CUBE_SKYFLAT,
444  /* the QC parameters are saved with the unprocessed cube, now delete them */
445  cpl_propertylist_erase_regexp(cubefit->header, QC_TWILIGHT_REGEXP, 0);
446  } /* if no skipping */
447 
448  /* now mark empty areas in the white-light image as bad; just *
449  * access the first reconstructed image, since the white-light *
450  * image is the only one created by muse_twilight_reconstruct() */
451  muse_image *wmuse = muse_imagelist_get(cube->recimages, 0);
452  if (!wmuse) {
453  muse_datacube_delete(cube);
454  cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_OUTPUT,
455  "White-light image is missing!");
456  muse_datacube_delete(cubefit);
457  return -1;
458  }
459  cpl_image *white = cpl_image_duplicate(wmuse->data);
460  cpl_image_reject_value(white, CPL_VALUE_NOTFINITE | CPL_VALUE_ZERO);
461  cpl_mask *bpm = cpl_image_get_bpm(white),
462  *bpm2 = cpl_mask_duplicate(bpm);
463  /* force a rejection of the vertical border pixels */
464  cpl_mask *border = cpl_mask_new(1, cpl_mask_get_size_y(bpm2));
465  cpl_mask_not(border);
466  cpl_mask_copy(bpm2, border, 1, 1);
467  cpl_mask_copy(bpm2, border, cpl_mask_get_size_x(bpm2), 1);
468  cpl_mask_delete(border);
469  /* use dilation to make the mask wider, i.e. to *
470  * exclude more of the (usually bad) border */
471  cpl_mask *kernel = cpl_mask_new(5, 1);
472  cpl_mask_not(kernel);
473  cpl_mask_filter(bpm2, bpm, kernel, CPL_FILTER_DILATION, CPL_BORDER_NOP);
474  cpl_mask_delete(kernel);
475  bpm = cpl_image_set_bpm(white, bpm2);
476  cpl_mask_delete(bpm);
477  bpm = bpm2; /* easier handle to remember */
478 
479  /* apply a mask, if one was given, on top of the one that the image already has */
480  muse_mask *maskimage = muse_processing_mask_load(aProcessing,
481  MUSE_TAG_VIGN_MASK);
482  /* adapt the mask to the size of the actual cube */
483  if (maskimage) {
484  cpl_mask *mask = muse_cplmask_adapt_to_image(maskimage->mask,
485  cpl_imagelist_get(cube->data, 0));
486  cpl_mask_delete(maskimage->mask);
487  maskimage->mask = mask;
488  }
489 #if 0 // only for smoothing in wavelength direction (see below)
490  /* also compute the wavelength of each plane */
491  double crpix3 = cpl_propertylist_get_double(cube->header, "CRPIX3"),
492  crval3 = cpl_propertylist_get_double(cube->header, "CRVAL3"),
493  cd33 = cpl_propertylist_get_double(cube->header, "CD3_3");
494  cpl_propertylist *pbla = cpl_propertylist_new();
495  cpl_propertylist_copy_property_regexp(pbla, cube->header, "^C", 0);
496  cpl_propertylist_dump(pbla, stdout);
497  fflush(stdout);
498  cpl_msg_debug(__func__, "%f %f %f", crpix3, crval3, cd33);
499  cpl_vector *pos = cpl_vector_new(npl);
500 #endif
501  /* and we can normalize, fit, and then normalize again, every plane in the cube */
502  muse_image *image = muse_image_new();
503  int ipl, npl = cpl_imagelist_get_size(cube->data);
504  for (ipl = 0; ipl < npl; ipl++) {
505  image->data = cpl_imagelist_get(cube->data, ipl);
506  image->stat = cpl_imagelist_get(cube->stat, ipl);
507  cpl_mask *vignmask = maskimage ? maskimage->mask : NULL;
508  cpl_image *fit = muse_twilight_image_normalize_and_fit(image,
509  aParams->xorder,
510  aParams->yorder,
511  bpm, vignmask);
512  cpl_imagelist_set(cubefit->data, fit, ipl);
513 #if 0 // only for smoothing in wavelength direction (see below)
514  /* wavelength of this plane */
515  cpl_vector_set(pos, ipl, ipl+1);//(ipl + 1. - crpix3) * cd33 + crval3);
516 #endif
517  } /* for ipl (all cube image planes) */
518  image->data = NULL;
519  image->stat = NULL;
520  muse_image_delete(image);
521  muse_datacube_delete(cube);
522 
523 #if 0 // XXX try to smooth with a fit in wavelenth direction?
524  cpl_image *error = cpl_image_new(cpl_image_get_size_x(white),
525  cpl_image_get_size_y(white),
526  CPL_TYPE_DOUBLE);
527  cpl_imagelist *cubefit2 = cpl_fit_imagelist_polynomial(pos, cubefit->data,
528  0, 2, CPL_FALSE,
529  CPL_TYPE_DOUBLE, error);
530  cpl_vector_dump(pos, stdout);
531  fflush(stdout);
532  cpl_vector_delete(pos);
533  muse_processing_save_cimage(aProcessing, -1, error, wmuse->header, "ERROR_IMAGE");
534  cpl_image_delete(error);
535  /* evaluate the fits and construct a new cube out of them */
536  cpl_imagelist_save(cubefit2, "cubefit2.fits", CPL_TYPE_UNSPECIFIED, NULL, CPL_IO_CREATE);
537  cpl_imagelist_delete(cubefit2);
538 #endif
539 
540 #if 0 /* extra debug output for the globally fitted cube, without vignetted area */
541  rc = muse_processing_save_cube(aProcessing, -1, cubefit,
542  "TWILIGHT_CUBE_FIT", MUSE_CUBE_TYPE_FITS);
543 #endif
544 
545  muse_image *whitefit = muse_datacube_collapse(cubefit, NULL);
546  cpl_image_delete(whitefit->dq);
547  whitefit->dq = NULL;
548  cpl_propertylist_update_string(whitefit->header, "OBJECT",
549  "white from global fit");
550  cubefit->recimages = muse_imagelist_new();
551  muse_imagelist_set(cubefit->recimages, whitefit, 0);
552  cubefit->recnames = cpl_array_new(1, CPL_TYPE_STRING);
553  cpl_array_set_string(cubefit->recnames, 0, "white_global_fit");
554 
555  /* normalize the non-smoothed white-light image and divide it by *
556  * the smoothed version; then fit the masked (vignetted) area */
557  double mean = cpl_image_get_mean(white);
558  cpl_image_divide_scalar(white, mean);
559  cpl_image *wdiv = cpl_image_divide_create(white, whitefit->data);
560  /* append to reconstructed images */
561  image = muse_image_new();
562  image->header = cpl_propertylist_new();
563  cpl_propertylist_append_string(image->header, "OBJECT", "normalized "
564  "white-light divided by white global fit");
565  image->data = wdiv;
566  image->dq = cpl_image_new_from_mask(cpl_image_get_bpm(wdiv));
567  muse_imagelist_set(cubefit->recimages, image,
568  muse_imagelist_get_size(cubefit->recimages));
569  cpl_array_set_size(cubefit->recnames,
570  cpl_array_get_size(cubefit->recnames) + 1);
571  cpl_array_set_string(cubefit->recnames, cpl_array_get_size(cubefit->recnames) - 1,
572  "white_div_global");
573  /* keep mask due to non-rectangular FOV around */
574  cpl_mask *fovmask = cpl_mask_duplicate(cpl_image_get_bpm(white));
575  if (maskimage) {
576  bpm = cpl_image_get_bpm(white);
577  cpl_mask_or(bpm, maskimage->mask);
578  } /* if maskimage */
579  cpl_image_delete(white);
580 
581  /* now invert the mask image (we need to fit the region that it contains!) */
582  if (maskimage) {
583  /* create mask of only the vignetted area */
584  cpl_mask *vignmask = cpl_mask_duplicate(fovmask);
585  cpl_mask_not(maskimage->mask);
586  cpl_mask_or(vignmask, maskimage->mask);
587  cpl_image *wdiv2 = cpl_image_duplicate(wdiv);
588  cpl_image_reject_from_mask(wdiv2, vignmask);
589 
590  /* search for pixels strongly deviating from unity and reject them */
591  if (aParams->vignmaskedges > 0.) {
592  cpl_msg_info(__func__, "Excluding strong edges (stronger than %.1f%%) "
593  "from the vignetted area", aParams->vignmaskedges * 100.);
594  cpl_image *wdivX = cpl_image_duplicate(wdiv2);
595  cpl_mask *wdivXmask = cpl_mask_duplicate(cpl_image_get_bpm(wdivX));
596  cpl_image_fill_rejected(wdivX, 1.);
597  cpl_image_accept_all(wdivX);
598  int i, nx = cpl_image_get_size_x(wdivX),
599  j, ny = cpl_image_get_size_y(wdivX);
600  /* first column-wise */
601  for (i = 1; i <= nx; i++) {
602  for (j = 2; j <= ny; j++) {
603  int err;
604  double v1 = cpl_image_get(wdivX, i, j - 1, &err),
605  v2 = cpl_image_get(wdivX, i, j, &err);
606  if (fabs(v2 - v1) < aParams->vignmaskedges) {
607  continue;
608  }
609  /* large difference found */
610  cpl_msg_debug(__func__, "%03d %03d v1 %f V2 %f --> delta %f",
611  i, j, v1, v2, fabs(v2 - v1));
612  double dv1 = fabs(v1 - 1.),
613  dv2 = fabs(v2 - 1.);
614  if (dv1 > dv2) {
615  cpl_mask_set(wdivXmask, i, j - 1, CPL_BINARY_1);
616  } else {
617  cpl_mask_set(wdivXmask, i, j, CPL_BINARY_1);
618  }
619  }
620  }
621  /* now row-wise */
622  for (j = 1; j <= ny; j++) {
623  for (i = 2; i <= nx; i++) {
624  int err;
625  double v1 = cpl_image_get(wdivX, i - 1, j, &err),
626  v2 = cpl_image_get(wdivX, i, j, &err);
627  if (fabs(v2 - v1) < aParams->vignmaskedges) {
628  continue;
629  }
630  /* large difference found */
631  cpl_msg_debug(__func__, "%03d %03d v1 %f V2 %f --> delta %f",
632  i, j, v1, v2, fabs(v2 - v1));
633  double dv1 = fabs(v1 - 1.),
634  dv2 = fabs(v2 - 1.);
635  if (dv1 > dv2) {
636  cpl_mask_set(wdivXmask, i - 1, j, CPL_BINARY_1);
637  } else {
638  cpl_mask_set(wdivXmask, i, j, CPL_BINARY_1);
639  }
640  }
641  }
642  /* transfer the edited mask to the divided image for the smoothing */
643  cpl_image_reject_from_mask(wdiv2, wdivXmask);
644 
645  /* append to reconstructed images, before throwing away */
646  image = muse_image_new();
647  image->header = cpl_propertylist_new();
648  cpl_propertylist_append_string(image->header, "OBJECT",
649  "white-light divided by white global, mask excl edges");
650  image->data = wdivX;
651  image->dq = cpl_image_new_from_mask(wdivXmask);
652  cpl_mask_delete(wdivXmask);
653  muse_imagelist_set(cubefit->recimages, image,
654  muse_imagelist_get_size(cubefit->recimages));
655  cpl_array_set_size(cubefit->recnames,
656  cpl_array_get_size(cubefit->recnames) + 1);
657  cpl_array_set_string(cubefit->recnames, cpl_array_get_size(cubefit->recnames) - 1,
658  "white_div_global_excl_edges");
659  } /* if vignmaskedges positive */
660 
661  /* do the smoothing in one of the several possible ways */
662  cpl_image *fit = NULL;
663  if (aParams->vignsmooth == MUSE_TWILIGHT_PARAM_VIGNSMOOTH_GAUSSIAN) {
664  /* use Gaussian filter to smooth the vignetted area */
665  double fwhm = (aParams->vignxpar + aParams->vignypar) / 2.;
666  cpl_msg_info(__func__, "Running %.1f pix FWHM Gaussian filter...", fwhm);
667  fit = cpl_image_duplicate(wdiv2);
668  /* create odd halfwidth of about fwhm */
669  int hw = lround(fwhm);
670  hw = hw % 2 ? hw : hw + 1;
671  cpl_msg_debug(__func__, "using width 2 x %d + 1 for Gaussian matrix", hw);
672  cpl_matrix *gauss = muse_matrix_new_gaussian_2d(hw, hw,
673  fwhm * CPL_MATH_SIG_FWHM);
674  cpl_errorstate prestate = cpl_errorstate_get();
675  cpl_image_filter(fit, wdiv2, gauss, CPL_FILTER_LINEAR, CPL_BORDER_FILTER);
676  cpl_matrix_delete(gauss);
677  cpl_image_reject_from_mask(fit, cpl_image_get_bpm(wdiv2)); /* re-reject! */
678  /* again, replace values less than 1 */
679  cpl_image_threshold(fit, 1., FLT_MAX, 1., FLT_MAX);
680  if (!cpl_errorstate_is_equal(prestate)) {
681  cpl_msg_warning(__func__, "filtering vignetted area failed: %s",
682  cpl_error_get_message());
683  }
684 #if 0 /* test of trying to run the filtering on a larger area which is *
685  * set to one towards the center of the field; implementation not *
686  * finished but only tested with a hand-edited wider mask */
687  cpl_msg_info(__func__, "Running Gaussian filter on extended region...");
688  /* use Gaussian filter to smooth the vignetted area */
689  fit = cpl_image_duplicate(wdiv2);
690  //cpl_mask_save(cpl_image_get_bpm(wdiv2), "masktest.fits", NULL, CPL_IO_CREATE);
691  /* threshold the rest of the image */
692  cpl_image *wdiv3 = cpl_image_duplicate(wdiv2);
693  cpl_image_fill_rejected(wdiv3, 1.);
694  cpl_mask *masktest = cpl_mask_load("masktest3.fits", 0, 0);
695  cpl_image_reject_from_mask(wdiv3, masktest);
696  cpl_mask_delete(masktest);
697  /* extend the vignetted area to the top and left */
698  /* find top pixel in each column */
699  /* extend it from there 5 pix to the left and 5 pix to the top */
700  cpl_matrix *gauss = muse_matrix_new_gaussian_2d(7, 7,
701  10. * CPL_MATH_SIG_FWHM);
702  cpl_errorstate prestate = cpl_errorstate_get();
703  cpl_image_filter(fit, wdiv3, gauss, CPL_FILTER_LINEAR, CPL_BORDER_FILTER);
704  cpl_image_delete(wdiv3);
705  cpl_matrix_delete(gauss);
706  cpl_image_reject_from_mask(fit, cpl_image_get_bpm(wdiv2)); /* re-reject! */
707  /* again, replace values less than 1 */
708  cpl_image_threshold(fit, 1., FLT_MAX, 1., FLT_MAX);
709  if (!cpl_errorstate_is_equal(prestate)) {
710  cpl_msg_warning(__func__, "filtering vignetted area failed: %s",
711  cpl_error_get_message());
712  }
713 #endif
714  } else if (aParams->vignsmooth == MUSE_TWILIGHT_PARAM_VIGNSMOOTH_MEDIAN) {
715  cpl_msg_info(__func__, "Running %dx%d median filter...",
716  aParams->vignxpar, aParams->vignypar);
717  fit = cpl_image_duplicate(wdiv2);
718  cpl_mask *mask = cpl_mask_new(aParams->vignxpar, aParams->vignypar);
719  cpl_mask_not(mask);
720  cpl_errorstate prestate = cpl_errorstate_get();
721  cpl_image_filter_mask(fit, wdiv2, mask, CPL_FILTER_MEDIAN,
722  CPL_BORDER_FILTER);
723  cpl_mask_delete(mask);
724  /* again, replace values less than 1 */
725  cpl_image_threshold(fit, 1., FLT_MAX, 1., FLT_MAX);
726  if (!cpl_errorstate_is_equal(prestate)) {
727  cpl_msg_warning(__func__, "filtering vignetted ared failed: %s",
728  cpl_error_get_message());
729  }
730  } else { /* MUSE_TWILIGHT_PARAM_VIGNSMOOTH_POLYFIT */
731  cpl_msg_info(__func__, "Running polyfit...");
732  /* since we want to correct only vignetting, everything in this image *
733  * should be one or above, so that we should replace lower values */
734  cpl_image *wdiv3 = cpl_image_duplicate(wdiv2);
735  cpl_image_threshold(wdiv3, 1., FLT_MAX, 1., FLT_MAX);
736 
737  /* do the fit */
738  fit = muse_utils_image_fit_polynomial(wdiv3, aParams->vignxpar,
739  aParams->vignypar);
740  cpl_image_delete(wdiv3);
741  /* again, replace values less than 1 */
742  cpl_image_threshold(fit, 1., FLT_MAX, 1., FLT_MAX);
743  } /* else */
744 
745  /* set the original rejection mask again */
746  cpl_image_reject_from_mask(fit, vignmask);
747  /* fill everything but the fitted area with ones, so that we *
748  * don't add rubbish when now dividing by this vignetting fix */
749  cpl_image_fill_rejected(fit, 1.);
750 
751  /* append to reconstructed images */
752  image = muse_image_new();
753  image->header = cpl_propertylist_new();
754  cpl_propertylist_append_string(image->header, "OBJECT", "vignetting fit");
755  image->data = fit;
756  image->dq = cpl_image_new_from_mask(cpl_image_get_bpm(fit));
757  muse_imagelist_set(cubefit->recimages, image,
758  muse_imagelist_get_size(cubefit->recimages));
759  cpl_array_set_size(cubefit->recnames,
760  cpl_array_get_size(cubefit->recnames) + 1);
761  cpl_array_set_string(cubefit->recnames, cpl_array_get_size(cubefit->recnames) - 1,
762  "vign_fit");
763 
764  /* divide by the result to assess how well it worked, and append to recimages */
765  image = muse_image_new();
766  image->header = cpl_propertylist_new();
767  cpl_propertylist_append_string(image->header, "OBJECT",
768  "white image corrected by vignetting fit");
769  image->data = cpl_image_divide_create(wdiv2, fit);
770  image->dq = cpl_image_new_from_mask(cpl_image_get_bpm(image->data));
771  muse_imagelist_set(cubefit->recimages, image,
772  muse_imagelist_get_size(cubefit->recimages));
773  cpl_array_set_size(cubefit->recnames,
774  cpl_array_get_size(cubefit->recnames) + 1);
775  cpl_array_set_string(cubefit->recnames, cpl_array_get_size(cubefit->recnames) - 1,
776  "white_cor");
777  cpl_image_delete(wdiv2);
778 
779  /* apply the vignetting fit on top of the twilight cube */
780  cpl_msg_info(__func__, "Combining smooth field of view and vignetted corner"
781  " in %d planes", npl);
782  for (ipl = 0; ipl < npl; ipl++) {
783  cpl_image *cimage = cpl_imagelist_get(cubefit->data, ipl);
784  cpl_image_accept_all(cimage);
785  double mean1 = cpl_image_get_mean(cimage);
786  cpl_image_multiply(cimage, fit);
787  /* since the image multiplication merges the bad pixel masks, we need *
788  * to reset them again before computing the normalization factor */
789  cpl_image_accept_all(cimage);
790  double mean2 = cpl_image_get_mean(cimage);
791  cpl_image_multiply_scalar(cimage, 1. / mean2);
792  double mean3 = cpl_image_get_mean(cimage);
793  if (fabs(mean3 - 1.) > FLT_EPSILON) {
794  cpl_msg_warning(__func__, "normalization failed in plane %d: mean("
795  "plane) = %f -> %f -> %f", ipl + 1, mean1, mean2, mean3);
796  } /* if */
797  } /* for ipl (all cube image planes) */
798  cpl_mask_delete(vignmask);
799  } /* if maskimage */
800  cpl_mask_delete(fovmask);
801  muse_mask_delete(maskimage);
802 
803  rc = muse_processing_save_cube(aProcessing, -1, cubefit,
804  MUSE_TAG_TWILIGHT_CUBE,
806  muse_datacube_delete(cubefit);
807 
808  return rc == CPL_ERROR_NONE ? 0 : -1;
809 } /* muse_twilight_compute() */
double crsigma
Sigma rejection factor to use for cosmic ray rejection during final resampling. A zero or negative va...
int muse_processing_save_cimage(muse_processing *aProcessing, int aIFU, cpl_image *aImage, cpl_propertylist *aHeader, const char *aTag)
Save a computed FITS image to disk.
muse_imagelist * muse_basicproc_load(muse_processing *aProcessing, unsigned char aIFU, muse_basicproc_params *aBPars)
Load the raw input files from disk and do basic processing.
Structure definition of a MUSE datacube.
Definition: muse_datacube.h:47
Structure definition for a collection of muse_images.
void muse_image_delete(muse_image *aImage)
Deallocate memory associated to a muse_image object.
Definition: muse_image.c:85
unsigned char muse_utils_get_ifu(const cpl_propertylist *aHeaders)
Find out the IFU/channel from which this header originated.
Definition: muse_utils.c:99
cpl_image * data
the data extension
Definition: muse_image.h:46
cpl_size muse_pixtable_get_nrow(const muse_pixtable *aPixtable)
get the number of rows within the pixel table
muse_datacube * muse_datacube_load(const char *aFilename)
Load header, DATA and optionally STAT and DQ extensions as well as the reconstructed images of a MUSE...
muse_resampling_crstats_type muse_postproc_get_cr_type(const char *aCRTypeString)
Select correct cosmic ray rejection type for crtype string.
cpl_image * stat
the statistics extension
Definition: muse_image.h:64
void muse_datacube_delete(muse_datacube *aCube)
Deallocate memory associated to a muse_datacube object.
void muse_imagelist_delete(muse_imagelist *aList)
Free the memory of the MUSE image list.
muse_image * muse_datacube_collapse(muse_datacube *aCube, cpl_table *aFilter)
Integrate a FITS NAXIS=3 datacube along the wavelength direction.
const char * crtype_s
Type of statistics used for detection of cosmic rays during final resampling. "iraf" uses the varianc...
muse_basicproc_params * muse_basicproc_params_new(cpl_parameterlist *aParameters, const char *aPrefix)
Create a new structure of basic processing parameters.
muse_image * muse_combine_images(muse_combinepar *aCPars, muse_imagelist *aImages)
Combine several images into one.
Definition: muse_combine.c:741
Structure definition of MUSE three extension FITS file.
Definition: muse_image.h:40
cpl_array * recnames
the reconstructed image filter names
Definition: muse_datacube.h:70
muse_mask * muse_processing_mask_load(muse_processing *aProcessing, const char *aTag)
Load a mask file and its FITS header.
cpl_table * table
The pixel table.
void muse_basicproc_params_delete(muse_basicproc_params *aBPars)
Free a structure of basic processing parameters.
cpl_propertylist * header
the FITS header
Definition: muse_image.h:72
unsigned int muse_imagelist_get_size(muse_imagelist *aList)
Return the number of stored images.
cpl_frameset * usedframes
void muse_combinepar_delete(muse_combinepar *aCPars)
Clear the combination parameters.
Definition: muse_combine.c:715
muse_resampling_crstats_type crtype
cpl_image * dq
the data quality extension
Definition: muse_image.h:56
cpl_error_code muse_pixtable_restrict_wavelength(muse_pixtable *aPixtable, double aLow, double aHigh)
Restrict a pixel table to a certain wavelength range.
Structure definition of MUSE pixel table.
cpl_error_code muse_datacube_convert_dq(muse_datacube *aCube)
Convert the DQ extension of a datacube to NANs in DATA and STAT.
muse_image * muse_imagelist_get(muse_imagelist *aList, unsigned int aIdx)
Get the muse_image of given list index.
int vignxpar
Parameter used by the vignetting smoothing: x order for polyfit (default, recommended 4)...
double vignmaskedges
Pixels on edges stronger than this fraction in the normalized image are excluded from the fit to the ...
int vignypar
Parameter used by the vignetting smoothing: y order for polyfit (default, recommended 4)...
muse_resampling_params * muse_resampling_params_new(muse_resampling_type aMethod)
Create the resampling parameters structure.
Structure to hold the parameters of the muse_twilight recipe.
cpl_error_code muse_processing_save_cube(muse_processing *aProcessing, int aIFU, void *aCube, const char *aTag, muse_cube_type aType)
Save a MUSE datacube to disk.
double dlambda
Sampling for twilight reconstruction, this should result in planes of equal wavelength coverage...
int vignsmooth
Type of smoothing to use for the vignetted region given by the VIGNETTING_MASK; gaussian uses (vignxp...
muse_combinepar * muse_combinepar_new(cpl_parameterlist *aParameters, const char *aPrefix)
Create a new set of combination parameters.
Definition: muse_combine.c:672
double lambdamin
Minimum wavelength for twilight reconstruction.
cpl_imagelist * data
the cube containing the actual data values
Definition: muse_datacube.h:75
void muse_processing_append_used(muse_processing *aProcessing, cpl_frame *aFrame, cpl_frame_group aGroup, int aDuplicate)
Add a frame to the set of used frames.
int yorder
Polynomial order to use in y direction to fit the full field of view.
int xorder
Polynomial order to use in x direction to fit the full field of view.
muse_pixtable * muse_pixtable_create(muse_image *aImage, cpl_table *aTrace, cpl_table *aWave, cpl_table *aGeoTable)
Create the pixel table for one CCD.
muse_datacube * muse_resampling_cube(muse_pixtable *aPixtable, muse_resampling_params *aParams, muse_pixgrid **aPixgrid)
Resample a pixel table onto a regular grid structure representing a FITS NAXIS=3 datacube.
cpl_error_code muse_basicproc_stats_append_header(cpl_image *aImage, cpl_propertylist *aHeader, const char *aPrefix, unsigned aStats)
Compute image statistics of an image and add them to a header.
cpl_propertylist * header
the FITS header
Definition: muse_datacube.h:56
muse_resampling_type muse_postproc_get_resampling_type(const char *aResampleString)
Select correct resampling type for resample string.
cpl_image * muse_utils_image_fit_polynomial(const cpl_image *aImage, unsigned short aXOrder, unsigned short aYOrder)
Create a smooth version of a 2D image by fitting it with a 2D polynomial.
Definition: muse_utils.c:1091
Handling of "mask" files.
Definition: muse_mask.h:42
cpl_array * intags
muse_imagelist * muse_imagelist_new(void)
Create a new (empty) MUSE image list.
cpl_frameset * inframes
cpl_table * muse_table_load(muse_processing *aProcessing, const char *aTag, unsigned char aIFU)
load a table according to its tag and IFU/channel number
Definition: muse_utils.c:721
Structure of basic processing parameters.
muse_resampling_type
Resampling types.
Resampling parameters.
void muse_resampling_params_delete(muse_resampling_params *aParams)
Delete a resampling parameters structure.
void muse_mask_delete(muse_mask *aMask)
Deallocate memory associated to a muse_mask object.
Definition: muse_mask.c:68
muse_image * muse_image_new(void)
Allocate memory for a new muse_image object.
Definition: muse_image.c:66
cpl_matrix * muse_matrix_new_gaussian_2d(int aXHalfwidth, int aYHalfwidth, double aSigma)
Create a matrix that contains a normalized 2D Gaussian.
Definition: muse_utils.c:1043
double lambdamax
Maximum wavelength for twilight reconstruction.
cpl_mask * mask
The mask data.
Definition: muse_mask.h:48
void muse_pixtable_delete(muse_pixtable *aPixtable)
Deallocate memory associated to a pixel table object.
cpl_error_code muse_imagelist_set(muse_imagelist *aList, muse_image *aImage, unsigned int aIdx)
Set the muse_image of given list index.
cpl_error_code muse_pixtable_compute_limits(muse_pixtable *aPixtable)
(Re-)Compute the limits of the coordinate columns of a pixel table.
int muse_quality_image_reject_using_dq(cpl_image *aData, cpl_image *aDQ, cpl_image *aStat)
Reject pixels of one or two images on a DQ image.
Definition: muse_quality.c:628
muse_imagelist * recimages
the reconstructed image data
Definition: muse_datacube.h:63
cpl_mask * muse_cplmask_adapt_to_image(const cpl_mask *aMask, const cpl_image *aImage)
Adapt mask with masked region in one quadrant to size of an image.
cpl_parameterlist * parameters
cpl_imagelist * stat
the cube containing the data variance
Definition: muse_datacube.h:85
cpl_propertylist * header
The FITS header.
cpl_frameset * muse_frameset_find_tags(const cpl_frameset *aFrames, const cpl_array *aTags, unsigned char aIFU, cpl_boolean aInvert)
return frameset containing data from an IFU/channel with the given tag(s)
Definition: muse_utils.c:243
const char * resample_s
The resampling technique to use for the final output cube. (as string)