KMOS Pipeline Reference Manual  1.3.11
kmo_illumination_flat.c
00001 /*
00002  * This file is part of the KMOS Pipeline
00003  * Copyright (C) 2002,2003 European Southern Observatory
00004  *
00005  * This program is free software; you can redistribute it and/or modify
00006  * it under the terms of the GNU General Public License as published by
00007  * the Free Software Foundation; either version 2 of the License, or
00008  * (at your option) any later version.
00009  *
00010  * This program is distributed in the hope that it will be useful,
00011  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00012  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00013  * GNU General Public License for more details.
00014  *
00015  * You should have received a copy of the GNU General Public License
00016  * along with this program; if not, write to the Free Software
00017  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00018  */
00019 
00020 #ifdef HAVE_CONFIG_H
00021 #include <config.h>
00022 #endif
00023 
00024 /*-----------------------------------------------------------------------------
00025  *                              Includes
00026  *----------------------------------------------------------------------------*/
00027 
00028 #include <math.h>
00029 #include <string.h>
00030 
00031 #include <cpl.h>
00032 
00033 #include "kmclipm_priv_splines.h"
00034 
00035 #include "kmo_priv_reconstruct.h"
00036 #include "kmo_priv_functions.h"
00037 #include "kmo_priv_flat.h"
00038 #include "kmo_priv_wave_cal.h"
00039 #include "kmo_functions.h"
00040 #include "kmo_cpl_extensions.h"
00041 #include "kmo_dfs.h"
00042 #include "kmo_error.h"
00043 #include "kmo_constants.h"
00044 #include "kmo_debug.h"
00045 
00046 /*-----------------------------------------------------------------------------
00047  *                          Functions prototypes
00048  *----------------------------------------------------------------------------*/
00049 
00050 static int kmo_illumination_flat_create(cpl_plugin *);
00051 static int kmo_illumination_flat_exec(cpl_plugin *);
00052 static int kmo_illumination_flat_destroy(cpl_plugin *);
00053 static int kmo_illumination_flat(cpl_parameterlist *, cpl_frameset *);
00054 
00055 /*-----------------------------------------------------------------------------
00056  *                          Static variables
00057  *----------------------------------------------------------------------------*/
00058 
00059 static char kmo_illumination_flat_description[] =
00060 "This recipe creates the spatial non-uniformity calibration frame needed for\n"
00061 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n"
00062 "generates the spectral calibration frame needed in this recipe. As input at\n"
00063 "least a flatfield frame is required.\n"
00064 "Contrary to kmo_illumination it doesn't use flat sky frames but rather the\n"
00065 "flatfield frames from the internal flat lamp. This recipe can be used if no\n"
00066 "acceptable flat sky frames are available.\n"
00067 "The created product, the illumination correction, can be used as input for\n"
00068 "kmo_std_star and kmo_sci_red.\n"
00069 "\n"
00070 "BASIC PARAMETERS:\n"
00071 "-----------------\n"
00072 "--imethod\n"
00073 "The interpolation method used for reconstruction.\n"
00074 "\n"
00075 "ADVANCED PARAMETERS\n"
00076 "-------------------\n"
00077 "--flux\n"
00078 "Specify if flux conservation should be applied.\n"
00079 "\n"
00080 "--neighborhoodRange\n"
00081 "Defines the range to search for neighbors during reconstruction\n"
00082 "\n"
00083 "--b_samples\n"
00084 "The number of samples in spectral direction for the reconstructed cube.\n"
00085 "Ideally this number should be greater than 2048, the detector size.\n"
00086 "\n"
00087 "--b_start\n"
00088 "--b_end\n"
00089 "Used to define manually the start and end wavelength for the reconstructed\n"
00090 "cube. By default the internally defined values are used.\n"
00091 "\n"
00092 "--cmethod\n"
00093 "Following methods of frame combination are available:\n"
00094 "   * 'ksigma' (Default)\n"
00095 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00096 "   are examined. If they deviate significantly, they will be rejected according\n"
00097 "   to the conditions:\n"
00098 "       val > mean + stdev * cpos_rej\n"
00099 "   and\n"
00100 "       val < mean - stdev * cneg_rej\n"
00101 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00102 "   parameters. In the first iteration median and percentile level are used.\n"
00103 "\n"
00104 "   * 'median'\n"
00105 "   At each pixel position the median is calculated.\n"
00106 "\n"
00107 "   * 'average'\n"
00108 "   At each pixel position the average is calculated.\n"
00109 "\n"
00110 "   * 'sum'\n"
00111 "   At each pixel position the sum is calculated.\n"
00112 "\n"
00113 "   * 'min_max'\n"
00114 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00115 "   --cmax and --cmin apply to this method.\n"
00116 "\n"
00117 "--cpos_rej\n"
00118 "--cneg_rej\n"
00119 "--citer\n"
00120 "see --cmethod='ksigma'\n"
00121 "\n"
00122 "--cmax\n"
00123 "--cmin\n"
00124 "see --cmethod='min_max'\n"
00125 "\n"
00126 "--pix_scale\n"
00127 "Change the pixel scale [arcsec]. Default of 0.2\" results into cubes of\n"
00128 "14x14pix, a scale of 0.1\" results into cubes of 28x28pix, etc.\n"
00129 "\n"
00130 "--suppress_extension\n"
00131 "If set to TRUE, the arbitrary filename extensions are supressed. If multiple\n"
00132 "products with the same category are produced, they will be numered consecutively\n"
00133 "starting from 0.\n"
00134 "\n"
00135 "-------------------------------------------------------------------------------\n"
00136 "  Input files:\n"
00137 "\n"
00138 "   DO                    KMOS                                                  \n"
00139 "   category              Type   Explanation                    Required #Frames\n"
00140 "   --------              -----  -----------                    -------- -------\n"
00141 "   FLAT_SKY_FLAT          F2D   Flatlamp-on exposures             Y      1-n   \n"
00142 "                                (at least 3 frames recommended)                \n"
00143 "   XCAL                   F2D   x calibration frame               Y       1    \n"
00144 "   YCAL                   F2D   y calibration frame               Y       1    \n"
00145 "   LCAL                   F2D   Wavelength calib. frame           Y       1    \n"
00146 "   WAVE_BAND              F2L   Table with start-/end-wavelengths Y       1    \n"
00147 "\n"
00148 "  Output files:\n"
00149 "\n"
00150 "   DO                    KMOS\n"
00151 "   category              Type   Explanation\n"
00152 "   --------              -----  -----------\n"
00153 "   ILLUM_CORR             F2I    Illumination calibration frame   \n"
00154 "-------------------------------------------------------------------------------\n"
00155 "\n";
00156 
00157 /*-----------------------------------------------------------------------------
00158  *                              Functions code
00159  *----------------------------------------------------------------------------*/
00160 
00177 int cpl_plugin_get_info(cpl_pluginlist *list)
00178 {
00179     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00180     cpl_plugin *plugin = &recipe->interface;
00181 
00182     cpl_plugin_init(plugin,
00183                         CPL_PLUGIN_API,
00184                         KMOS_BINARY_VERSION,
00185                         CPL_PLUGIN_TYPE_RECIPE,
00186                         "kmo_illumination_flat",
00187                         "Alternative to kmo_illumination based on flatfield frames.",
00188                         kmo_illumination_flat_description,
00189                         "Alex Agudo Berbel",
00190                         "usd-help@eso.org",
00191                         kmos_get_license(),
00192                         kmo_illumination_flat_create,
00193                         kmo_illumination_flat_exec,
00194                         kmo_illumination_flat_destroy);
00195 
00196     cpl_pluginlist_append(list, plugin);
00197 
00198     return 0;
00199 }
00200 
00208 static int kmo_illumination_flat_create(cpl_plugin *plugin)
00209 {
00210     cpl_recipe *recipe;
00211     cpl_parameter *p;
00212 
00213     /* Check that the plugin is part of a valid recipe */
00214     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00215         recipe = (cpl_recipe *)plugin;
00216     else
00217         return -1;
00218 
00219     /* Create the parameters list in the cpl_recipe object */
00220     recipe->parameters = cpl_parameterlist_new();
00221 
00222     /* Fill the parameters list */
00223     /* --imethod */
00224     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.imethod",
00225                                 CPL_TYPE_STRING,
00226                                 "Method to use for interpolation: "
00227                                 "[\"NN\" (nearest neighbour), "
00228                                 "\"lwNN\" (linear weighted nearest neighbor), "
00229                                 "\"swNN\" (square weighted nearest neighbor), "
00230                                 "\"MS\" (Modified Shepard's method), "
00231                                 "\"CS\" (Cubic spline)]",
00232                                 "kmos.kmo_illumination_flat",
00233                                 "CS");
00234     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod");
00235     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00236     cpl_parameterlist_append(recipe->parameters, p);
00237 
00238     /* --neighborhoodRange */
00239     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.neighborhoodRange",
00240                                 CPL_TYPE_DOUBLE,
00241                                 "Defines the range to search for neighbors. "
00242                                 "in pixels",
00243                                 "kmos.kmo_illumination_flat",
00244                                 1.001);
00245     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange");
00246     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00247     cpl_parameterlist_append(recipe->parameters, p);
00248 
00249     /* --flux */
00250     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.flux",
00251                                 CPL_TYPE_BOOL,
00252                                 "TRUE: Apply flux conservation. FALSE: otherwise",
00253                                 "kmos.kmo_illumination_flat",
00254                                 FALSE);
00255     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00256     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00257     cpl_parameterlist_append(recipe->parameters, p);
00258 
00259     /* --pix_scale */
00260     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.pix_scale",
00261                                 CPL_TYPE_DOUBLE,
00262                                 "Change the pixel scale [arcsec]. "
00263                                 "Default of 0.2\" results into cubes of 14x14pix, "
00264                                 "a scale of 0.1\" results into cubes of 28x28pix, "
00265                                 "etc.",
00266                                 "kmos.kmo_illumination_flat",
00267                                 KMOS_PIX_RESOLUTION);
00268     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pix_scale");
00269     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00270     cpl_parameterlist_append(recipe->parameters, p);
00271 
00272     /* --suppress_extension */
00273     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.suppress_extension",
00274                                 CPL_TYPE_BOOL,
00275                                 "Suppress arbitrary filename extension. "
00276                                 "(TRUE (apply) or FALSE (don't apply)",
00277                                 "kmos.kmo_illumination_flat",
00278                                 FALSE);
00279     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "suppress_extension");
00280     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00281     cpl_parameterlist_append(recipe->parameters, p);
00282 
00283     // add parameters for band-definition
00284     kmos_band_pars_create(recipe->parameters,
00285                          "kmos.kmo_illumination_flat");
00286 
00287     // add parameters for combining
00288     return kmos_combine_pars_create(recipe->parameters,
00289                                    "kmos.kmo_illumination_flat",
00290                                    DEF_REJ_METHOD,
00291                                    FALSE);
00292 }
00293 
00299 static int kmo_illumination_flat_exec(cpl_plugin *plugin)
00300 {
00301     cpl_recipe  *recipe;
00302 
00303     /* Get the recipe out of the plugin */
00304     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00305         recipe = (cpl_recipe *)plugin;
00306     else return -1;
00307 
00308     return kmo_illumination_flat(recipe->parameters, recipe->frames);
00309 }
00310 
00316 static int kmo_illumination_flat_destroy(cpl_plugin *plugin)
00317 {
00318     cpl_recipe *recipe;
00319 
00320     /* Get the recipe out of the plugin */
00321     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00322         recipe = (cpl_recipe *)plugin;
00323     else return -1 ;
00324 
00325     cpl_parameterlist_delete(recipe->parameters);
00326     return 0 ;
00327 }
00328 
00343 static int kmo_illumination_flat(cpl_parameterlist *parlist, cpl_frameset *frameset)
00344 {
00345     int              ret_val                    = 0,
00346                      nr_devices                 = 0,
00347                      ifu_nr                     = 0,
00348                      nx                         = 0,
00349                      ny                         = 0,
00350                      cmax                       = 0,
00351                      cmin                       = 0,
00352                      citer                      = 0,
00353                      *bounds                    = NULL,
00354                      cnt                        = 0,
00355                      qc_max_dev_id              = 0,
00356                      qc_max_nonunif_id          = 0,
00357                      flux                       = FALSE,
00358                      background                 = FALSE,
00359                      suppress_extension         = FALSE,
00360                      mhalf                      = 3,    //width of median filter is mhalf*2 + 1
00361                      boxsize                    = 0,
00362                      i = 0, j = 0, ix = 0, iy = 0, det_nr = 0,
00363                      firstx = 0, lastx = 0, firsty = 0, lasty = 0,
00364                      xmin = 0, xmax = 0, ymin = 0, ymax = 0;
00365     const int        *punused_ifus              = NULL;
00366     float            *pbad_pix_mask             = NULL,
00367                      *pdata                     = NULL,
00368                      *pnoise                    = NULL;
00369     double           exptime                    = 0.,
00370                      cpos_rej                   = 0.0,
00371                      cneg_rej                   = 0.0,
00372                      neighborhoodRange          = 1.001,
00373                      mean_data                  = 0.0,
00374                      qc_spat_unif               = 0.0,
00375                      qc_max_dev                 = 0.0,
00376                      qc_max_nonunif             = 0.0,
00377                      tmp_stdev                  = 0.0,
00378                      tmp_mean                   = 0.0,
00379                      rotangle                   = 0.0,
00380                      tmp_rotangle               = 0.0,
00381                      rotangle_found             = 0.0,
00382                      pix_scale                  = 0.0;
00383     char             *keyword                   = NULL,
00384                      *fn_lut                    = NULL,
00385                      *suffix                    = NULL,
00386                      *fn_suffix                 = NULL,
00387                      *extname                   = NULL,
00388                      *filter                    = NULL,
00389                      content[256];
00390     const char       *method                    = NULL,
00391                      *cmethod                   = NULL,
00392                      *filter_id_l               = NULL,
00393                      *filter_id                 = NULL,
00394                      *tmp_str                   = NULL;
00395     cpl_array        *calTimestamp              = NULL,
00396                      **unused_ifus_before       = NULL,
00397                      **unused_ifus_after        = NULL;
00398     cpl_frame        *frame                     = NULL,
00399                      *xcalFrame                 = NULL,
00400                      *ycalFrame                 = NULL,
00401                      *lcalFrame                 = NULL;
00402     cpl_image        *img_in                    = NULL,
00403                      *img_dark                  = NULL,
00404                      *img_flat                  = NULL,
00405                      *combined_data             = NULL,
00406                      *xcal                      = NULL,
00407                      *ycal                      = NULL,
00408                      *lcal                      = NULL,
00409                      *bad_pix_mask              = NULL,
00410                      *data_ifu                  = NULL,
00411                      *noise_ifu                 = NULL,
00412                      **stored_data_images       = NULL,
00413                      **stored_noise_images      = NULL,
00414                      *tmp_img                   = NULL;
00415     cpl_imagelist    *cube_data                 = NULL,
00416                      *detector_in               = NULL,
00417                      **stored_data_cubes        = NULL;
00418     cpl_propertylist *main_header               = NULL,
00419                      *tmp_header                = NULL,
00420                      *sub_header                = NULL,
00421                      **stored_sub_headers       = NULL;
00422     cpl_table        *band_table                = NULL;
00423     cpl_vector       *identified_slices         = NULL,
00424                      *calAngles                 = NULL;
00425     main_fits_desc   desc_sky,
00426                      desc_xcal,
00427                      desc_ycal,
00428                      desc_lcal;
00429     gridDefinition   gd;
00430     enum kmo_frame_type fr_type;
00431 
00432     KMO_TRY
00433     {
00434         kmo_init_fits_desc(&desc_sky);
00435         kmo_init_fits_desc(&desc_xcal);
00436         kmo_init_fits_desc(&desc_ycal);
00437         kmo_init_fits_desc(&desc_lcal);
00438 
00439         /* --- check input --- */
00440         KMO_TRY_ASSURE((parlist != NULL) &&
00441                        (frameset != NULL),
00442                        CPL_ERROR_NULL_INPUT,
00443                        "Not all input data is provided!");
00444 
00445         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, FLAT_SKY_FLAT) >= 1,
00446                        CPL_ERROR_ILLEGAL_INPUT,
00447                        "One or more FLAT_SKY_FLAT frames are required!");
00448 
00449         if (cpl_frameset_count_tags(frameset, FLAT_SKY_FLAT) < 3) {
00450             cpl_msg_warning(cpl_func, "It is recommended to provide at least "
00451                                       "3 FLAT_SKY_FLAT frames!");
00452         }
00453 
00454         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, XCAL) == 1,
00455                        CPL_ERROR_ILLEGAL_INPUT,
00456                        "Exactly one XCAL frame is required!");
00457 
00458         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, YCAL) == 1,
00459                        CPL_ERROR_ILLEGAL_INPUT,
00460                        "Exactly one YCAL frame is required!");
00461 
00462         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, LCAL) == 1,
00463                        CPL_ERROR_ILLEGAL_INPUT,
00464                        "Exactly one LCAL frame is required!");
00465 
00466         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, WAVE_BAND) == 1,
00467                        CPL_ERROR_ILLEGAL_INPUT,
00468                        "Exactly one WAVE_BAND frame is required!");
00469 
00470         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_illumination_flat") == 1,
00471                        CPL_ERROR_ILLEGAL_INPUT,
00472                        "Cannot identify RAW and CALIB frames!");
00473 
00474         /* --- get parameters --- */
00475         cpl_msg_info("", "--- Parameter setup for kmo_illumination_flat ---");
00476 
00477         KMO_TRY_EXIT_IF_NULL(
00478             method = kmo_dfs_get_parameter_string(parlist, "kmos.kmo_illumination_flat.imethod"));
00479 
00480         KMO_TRY_ASSURE((strcmp(method, "NN") == 0) ||
00481                        (strcmp(method, "lwNN") == 0) ||
00482                        (strcmp(method, "swNN") == 0) ||
00483                        (strcmp(method, "MS") == 0) ||
00484                        (strcmp(method, "CS") == 0),
00485                        CPL_ERROR_ILLEGAL_INPUT,
00486                        "method must be either \"NN\", \"lwNN\", "
00487                        "\"swNN\", \"MS\" or \"CS\"!");
00488 
00489         KMO_TRY_EXIT_IF_ERROR(
00490             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.imethod"));
00491 
00492         neighborhoodRange = kmo_dfs_get_parameter_double(parlist, "kmos.kmo_illumination_flat.neighborhoodRange");
00493         KMO_TRY_CHECK_ERROR_STATE();
00494 
00495         KMO_TRY_ASSURE(neighborhoodRange > 0.0,
00496                        CPL_ERROR_ILLEGAL_INPUT,
00497                        "neighborhoodRange must be greater than 0.0");
00498 
00499         KMO_TRY_EXIT_IF_ERROR(
00500             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.neighborhoodRange"));
00501 
00502         flux = kmo_dfs_get_parameter_bool(parlist, "kmos.kmo_illumination_flat.flux");
00503         KMO_TRY_ASSURE((flux == 0) || (flux == 1),
00504                        CPL_ERROR_ILLEGAL_INPUT,
00505                        "flux must be either FALSE or TRUE!");
00506         KMO_TRY_EXIT_IF_ERROR(
00507             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.flux"));
00508 
00509         pix_scale = kmo_dfs_get_parameter_double(parlist, "kmos.kmo_illumination_flat.pix_scale");
00510         KMO_TRY_CHECK_ERROR_STATE();
00511         KMO_TRY_EXIT_IF_ERROR(
00512            kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.pix_scale"));
00513         KMO_TRY_ASSURE((pix_scale >= 0.01) && (pix_scale <= 0.4),
00514                        CPL_ERROR_ILLEGAL_INPUT,
00515                        "pix_scale must be between 0.01 and 0.4 (results in cubes "
00516                        "with 7x7 to 280x280 pixels)!");
00517 
00518         suppress_extension = kmo_dfs_get_parameter_bool(parlist, "kmos.kmo_illumination_flat.suppress_extension");
00519         KMO_TRY_CHECK_ERROR_STATE();
00520         KMO_TRY_EXIT_IF_ERROR(
00521             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.suppress_extension"));
00522 
00523         KMO_TRY_ASSURE((suppress_extension == TRUE) || (suppress_extension == FALSE),
00524                        CPL_ERROR_ILLEGAL_INPUT,
00525                        "suppress_extension must be TRUE or FALSE!");
00526 
00527         kmos_band_pars_load(parlist, "kmos.kmo_illumination_flat");
00528 
00529         KMO_TRY_EXIT_IF_ERROR(
00530             kmos_combine_pars_load(parlist, "kmos.kmo_illumination_flat",
00531                                   &cmethod, &cpos_rej, &cneg_rej,
00532                                   &citer, &cmin, &cmax, FALSE));
00533         cpl_msg_info("", "-------------------------------------------");
00534 
00535         // check if filter_id, grating_id and rotator offset match for all
00536         // detectors
00537         KMO_TRY_EXIT_IF_ERROR(
00538             kmo_check_frameset_setup(frameset, FLAT_SKY_FLAT, TRUE, FALSE, TRUE));
00539         KMO_TRY_EXIT_IF_ERROR(
00540             kmo_check_frame_setup(frameset, FLAT_SKY_FLAT, XCAL, TRUE, FALSE, TRUE));
00541         KMO_TRY_EXIT_IF_ERROR(
00542             kmo_check_frame_setup(frameset, XCAL, YCAL, TRUE, FALSE, TRUE));
00543         KMO_TRY_EXIT_IF_ERROR(
00544             kmo_check_frame_setup(frameset, XCAL, LCAL, TRUE, FALSE, TRUE));
00545 
00546         KMO_TRY_EXIT_IF_NULL(
00547             frame = kmo_dfs_get_frame(frameset, XCAL));
00548         KMO_TRY_EXIT_IF_NULL(
00549             suffix = kmo_dfs_get_suffix(frame, TRUE, FALSE));
00550 
00551         KMO_TRY_EXIT_IF_ERROR(
00552             kmo_check_frame_setup_md5_xycal(frameset));
00553         KMO_TRY_EXIT_IF_ERROR(
00554             kmo_check_frame_setup_md5(frameset));
00555 
00556         cpl_msg_info("", "Detected instrument setup:   %s", suffix+1);
00557         cpl_msg_info("", "(grating 1, 2 & 3)");
00558 
00559         // check which IFUs are active for all frames
00560         KMO_TRY_EXIT_IF_NULL(
00561             unused_ifus_before = kmo_get_unused_ifus(frameset, 0, 0));
00562 
00563         KMO_TRY_EXIT_IF_NULL(
00564             unused_ifus_after = kmo_duplicate_unused_ifus(unused_ifus_before));
00565 
00566         kmo_print_unused_ifus(unused_ifus_before, FALSE);
00567 
00568         // load desc for XCAL and check
00569         KMO_TRY_EXIT_IF_NULL(
00570             xcalFrame = kmo_dfs_get_frame(frameset, XCAL));
00571         desc_xcal = kmo_identify_fits_header(cpl_frame_get_filename(xcalFrame));
00572         KMO_TRY_CHECK_ERROR_STATE_MSG("XCAL frame doesn't seem to "
00573                                       "be in KMOS-format!");
00574         KMO_TRY_ASSURE((desc_xcal.nr_ext % KMOS_NR_DETECTORS == 0) &&
00575                        (desc_xcal.ex_badpix == FALSE) &&
00576                        (desc_xcal.fits_type == f2d_fits) &&
00577                        (desc_xcal.frame_type == detector_frame),
00578                        CPL_ERROR_ILLEGAL_INPUT,
00579                        "XCAL isn't in the correct format!!!");
00580         nx = desc_xcal.naxis1;
00581         ny = desc_xcal.naxis2;
00582         nr_devices = desc_xcal.nr_ext;
00583 
00584         // load desc for YCAL and check
00585         KMO_TRY_EXIT_IF_NULL(
00586             ycalFrame = kmo_dfs_get_frame(frameset, YCAL));
00587         desc_ycal = kmo_identify_fits_header(cpl_frame_get_filename(ycalFrame));
00588         KMO_TRY_CHECK_ERROR_STATE_MSG("YCAL frame doesn't seem to "
00589                                       "be in KMOS-format!");
00590         KMO_TRY_ASSURE((desc_ycal.nr_ext == desc_xcal.nr_ext) &&
00591                        (desc_ycal.ex_badpix == desc_xcal.ex_badpix) &&
00592                        (desc_ycal.fits_type == desc_xcal.fits_type) &&
00593                        (desc_ycal.frame_type == desc_xcal.frame_type),
00594                        CPL_ERROR_ILLEGAL_INPUT,
00595                        "YCAL isn't in the correct format!!!");
00596         KMO_TRY_ASSURE((desc_ycal.naxis1 == desc_xcal.naxis1) &&
00597                        (desc_ycal.naxis2 == desc_xcal.naxis2),
00598                        CPL_ERROR_ILLEGAL_INPUT,
00599                        "XCAL and YCAL frame haven't same dimensions! "
00600                        "(x,y): (%d,%d) vs (%d,%d)",
00601                        nx, ny, desc_ycal.naxis1, desc_ycal.naxis2);
00602 
00603         // load desc for LCAL and check
00604         KMO_TRY_EXIT_IF_NULL(
00605             lcalFrame = kmo_dfs_get_frame(frameset, LCAL));
00606         desc_lcal = kmo_identify_fits_header(cpl_frame_get_filename(lcalFrame));
00607         KMO_TRY_CHECK_ERROR_STATE_MSG("LCAL frame doesn't seem to "
00608                                       "be in KMOS-format!");
00609         KMO_TRY_ASSURE((desc_lcal.ex_badpix == desc_xcal.ex_badpix) &&
00610                        (desc_lcal.fits_type == desc_xcal.fits_type) &&
00611                        (desc_lcal.frame_type == desc_xcal.frame_type),
00612                        CPL_ERROR_ILLEGAL_INPUT,
00613                        "LCAL isn't in the correct format!!!");
00614         KMO_TRY_ASSURE((desc_lcal.naxis1 == desc_xcal.naxis1) &&
00615                        (desc_lcal.naxis2 == desc_xcal.naxis2),
00616                        CPL_ERROR_ILLEGAL_INPUT,
00617                        "XCAL and LCAL frame haven't same dimensions! "
00618                        "(x,y): (%d,%d) vs (%d,%d)",
00619                        nx, ny, desc_lcal.naxis1, desc_lcal.naxis2);
00620         KMO_TRY_EXIT_IF_NULL(
00621             tmp_header = kmo_dfs_load_primary_header(frameset, LCAL));
00622 
00623         // load desc for FLAT_SKY_FLAT and check
00624         nr_devices = KMOS_NR_DETECTORS;
00625         KMO_TRY_EXIT_IF_NULL(
00626             frame = kmo_dfs_get_frame(frameset, FLAT_SKY_FLAT));
00627 
00628         KMO_TRY_EXIT_IF_NULL(
00629             main_header = kmclipm_propertylist_load(
00630                                          cpl_frame_get_filename(frame), 0));
00631         rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00632         KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00633         kmclipm_strip_angle(&rotangle);
00634         exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00635         KMO_TRY_CHECK_ERROR_STATE("EXPTIME keyword in main header "
00636                                   "missing!");
00637         cpl_propertylist_delete(main_header); main_header = NULL;
00638 
00639         cnt = 1;
00640         while (frame != NULL) {
00641             KMO_TRY_EXIT_IF_NULL(
00642                 main_header = kmclipm_propertylist_load(
00643                                              cpl_frame_get_filename(frame), 0));
00644 
00645             desc_sky = kmo_identify_fits_header(
00646                         cpl_frame_get_filename(frame));
00647             KMO_TRY_CHECK_ERROR_STATE_MSG("FLAT_SKY_FLAT frame doesn't seem to "
00648                                           "be in KMOS-format!");
00649             KMO_TRY_ASSURE((desc_sky.nr_ext == 3) &&
00650                            (desc_sky.ex_badpix == FALSE) &&
00651                            (desc_sky.fits_type == raw_fits) &&
00652                            (desc_sky.frame_type == detector_frame),
00653                            CPL_ERROR_ILLEGAL_INPUT,
00654                            "FLAT_SKY_FLAT isn't in the correct format!!!");
00655             kmo_free_fits_desc(&desc_sky);
00656             kmo_init_fits_desc(&desc_sky);
00657 
00658             KMO_TRY_ASSURE(
00659                 (kmo_check_lamp(main_header, INS_LAMP1_ST) == FALSE) &&
00660                 (kmo_check_lamp(main_header, INS_LAMP2_ST) == FALSE),
00661                 CPL_ERROR_ILLEGAL_INPUT,
00662                 "Arc lamps must be switched off!");
00663 
00664             KMO_TRY_ASSURE(cpl_propertylist_get_double(main_header, EXPTIME) == exptime,
00665                            CPL_ERROR_ILLEGAL_INPUT,
00666                            "EXPTIME isn't the same for all frames: (is %g and %g).",
00667                            cpl_propertylist_get_double(main_header, EXPTIME), exptime);
00668 
00669             // assert that filters have correct IDs and that all detectors of
00670             // all input frames have the same filter set
00671             for (i = 1; i <= KMOS_NR_DETECTORS; i++) {
00672                 // ESO INS FILTi ID
00673                 KMO_TRY_EXIT_IF_NULL(
00674                     keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, IFU_FILTID_POSTFIX));
00675                 KMO_TRY_EXIT_IF_NULL(
00676                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00677 
00678                 KMO_TRY_EXIT_IF_NULL(
00679                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00680                 cpl_free(keyword); keyword = NULL;
00681 
00682                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00683                                (strcmp(filter_id, "YJ") == 0) ||
00684                                (strcmp(filter_id, "H") == 0) ||
00685                                (strcmp(filter_id, "K") == 0) ||
00686                                (strcmp(filter_id, "HK") == 0),
00687                                CPL_ERROR_ILLEGAL_INPUT,
00688                                "Filter ID in primary header must be either 'IZ', "
00689                                "'YJ', 'H', 'K' or " "'HK' !");
00690 
00691                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00692                                CPL_ERROR_ILLEGAL_INPUT,
00693                                "Filter IDs must be the same for FLAT_SKY_FLAT frame"
00694                                " and lcal frame!"
00695                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00696                                i, cpl_frame_get_filename(frame),
00697                                filter_id, filter_id_l);
00698 
00699                 // ESO INS GRATi ID
00700                 KMO_TRY_EXIT_IF_NULL(
00701                     keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, IFU_GRATID_POSTFIX));
00702                 KMO_TRY_EXIT_IF_NULL(
00703                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00704 
00705                 KMO_TRY_EXIT_IF_NULL(
00706                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00707                 cpl_free(keyword); keyword = NULL;
00708 
00709                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00710                                (strcmp(filter_id, "YJ") == 0) ||
00711                                (strcmp(filter_id, "H") == 0) ||
00712                                (strcmp(filter_id, "K") == 0) ||
00713                                (strcmp(filter_id, "HK") == 0),
00714                                CPL_ERROR_ILLEGAL_INPUT,
00715                                "Grating ID in primary header must be either "
00716                                "'IZ', 'YJ', 'H', 'K' or " "'HK' !");
00717 
00718                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00719                                CPL_ERROR_ILLEGAL_INPUT,
00720                                "Grating IDs must be the same for FLAT_SKY_FLAT frame"
00721                                " and lcal frame!"
00722                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00723                                i, cpl_frame_get_filename(frame),
00724                                filter_id, filter_id_l);
00725 
00726                 tmp_rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00727                 KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00728                 kmclipm_strip_angle(&tmp_rotangle);
00729                 KMO_TRY_ASSURE((abs(rotangle - tmp_rotangle) < 10.0) ||
00730                                (abs(rotangle - tmp_rotangle) > 360.-10.) ,
00731                         CPL_ERROR_ILLEGAL_INPUT,
00732                         "OCS ROT NAANGLE of sky flat frames differ too much: %f %f",
00733                         rotangle, tmp_rotangle);
00734             }
00735             cpl_propertylist_delete(main_header); main_header = NULL;
00736 
00737             // get next FLAT_SKY_FLAT frame
00738             frame = kmo_dfs_get_frame(frameset, NULL);
00739             KMO_TRY_CHECK_ERROR_STATE();
00740             cnt++;
00741         }
00742 
00743         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00744 
00745         if (cpl_frameset_count_tags(frameset, FLAT_SKY_FLAT) == 1) {
00746             cpl_msg_warning(cpl_func, "cmethod is changed to 'average' "
00747                             "since there is only one input frame! (The output "
00748                             "file won't have any noise extensions)");
00749 
00750             cmethod = "average";
00751         }
00752 
00753         KMO_TRY_EXIT_IF_NULL(
00754             frame = kmo_dfs_get_frame(frameset, FLAT_SKY_FLAT));
00755         KMO_TRY_EXIT_IF_NULL(
00756             main_header = kmo_dfs_load_primary_header(frameset, FLAT_SKY_FLAT));
00757         KMO_TRY_EXIT_IF_NULL(
00758             keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, 1, IFU_GRATID_POSTFIX));
00759         KMO_TRY_EXIT_IF_NULL(
00760             filter = cpl_sprintf("%s", cpl_propertylist_get_string(main_header, keyword)));
00761         cpl_free(keyword); keyword = NULL;
00762 
00763         // setup grid definition, wavelength start and end points will be set
00764         // in the detector loop
00765         KMO_TRY_EXIT_IF_ERROR(
00766             kmclipm_setup_grid(&gd, method, neighborhoodRange, pix_scale, 0.));
00767 
00768         // create filename for LUT
00769         KMO_TRY_EXIT_IF_NULL(
00770             fn_lut = cpl_sprintf("%s%s", "lut", suffix));
00771 
00772         // extract bounds
00773         KMO_TRY_EXIT_IF_NULL(
00774             tmp_header = kmo_dfs_load_primary_header(frameset, XCAL));
00775         KMO_TRY_EXIT_IF_NULL(
00776             bounds = kmclipm_extract_bounds(tmp_header));
00777         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00778 
00779         // get timestamps of xcal, ycal & lcal
00780         KMO_TRY_EXIT_IF_NULL(
00781             calTimestamp = kmo_get_timestamps(xcalFrame, ycalFrame, lcalFrame));
00782 
00783         // create arrays to hold reconstructed data and noise cubes and
00784         // their headers
00785         KMO_TRY_EXIT_IF_NULL(
00786             stored_data_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
00787                                                             sizeof(cpl_imagelist*)));
00788         KMO_TRY_EXIT_IF_NULL(
00789             stored_data_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
00790                                                          sizeof(cpl_image*)));
00791         KMO_TRY_EXIT_IF_NULL(
00792             stored_noise_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
00793                                                           sizeof(cpl_image*)));
00794         KMO_TRY_EXIT_IF_NULL(
00795             stored_sub_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
00796                                                                 sizeof(cpl_propertylist*)));
00797         KMO_TRY_EXIT_IF_NULL(
00798             calAngles = cpl_vector_new(3));
00799 
00800         //
00801         // loop through all detectors
00802         //
00803         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
00804             cpl_msg_info("","Processing detector No. %d", det_nr);
00805 
00806             KMO_TRY_EXIT_IF_NULL(
00807                 detector_in = cpl_imagelist_new());
00808 
00809             // load data of det_nr of all FLAT_SKY_FLAT frames into an imagelist
00810             KMO_TRY_EXIT_IF_NULL(
00811                 img_in = kmo_dfs_load_image(frameset, FLAT_SKY_FLAT, det_nr, FALSE, TRUE, NULL));
00812 
00813             cnt = 0;
00814             while (img_in != NULL) {
00815                 cpl_imagelist_set(detector_in, img_in, cnt);
00816                 KMO_TRY_CHECK_ERROR_STATE();
00817 
00818                 /* load same extension of next FLAT_SKY_FLAT frame*/
00819                 img_in = kmo_dfs_load_image(frameset, NULL, det_nr, FALSE, TRUE, NULL);
00820                 KMO_TRY_CHECK_ERROR_STATE();
00821 
00822                 cnt++;
00823             }
00824 
00825             //
00826             // process imagelist
00827             //
00828 
00829             // combine imagelist (data only) and create noise (stdev of data)
00830             cpl_msg_info("","Combining frames...");
00831             KMO_TRY_EXIT_IF_ERROR(
00832                 kmclipm_combine_frames(detector_in,
00833                                        NULL,
00834                                        NULL,
00835                                        cmethod,
00836                                        cpos_rej,
00837                                        cneg_rej,
00838                                        citer,
00839                                        cmax,
00840                                        cmin,
00841                                        &combined_data,
00842                                        NULL,
00843                                        -1.0));
00844 
00845             if (img_dark == NULL) {
00846                 KMO_TRY_EXIT_IF_NULL(
00847                     img_dark = cpl_image_duplicate(combined_data));
00848                 KMO_TRY_EXIT_IF_ERROR(
00849                     cpl_image_multiply_scalar(img_dark, 0));
00850             }
00851 
00852             if (img_flat == NULL) {
00853                 KMO_TRY_EXIT_IF_NULL(
00854                     img_flat = cpl_image_duplicate(combined_data));
00855                 KMO_TRY_EXIT_IF_ERROR(
00856                     cpl_image_multiply_scalar(img_flat, 0));
00857                 cpl_image_add_scalar(img_flat, 1);
00858             }
00859 
00860             if (kmclipm_omit_warning_one_slice > 10) {
00861 // AA: commmented this out: Too unclear for the user, no benefit to know about this number
00862 //                cpl_msg_warning(cpl_func, "Previous warning (number of "
00863 //                                          "identified slices) occured %d times.",
00864 //                                kmclipm_omit_warning_one_slice);
00865                 kmclipm_omit_warning_one_slice = FALSE;
00866             }
00867 
00868             cpl_imagelist_delete(detector_in); detector_in = NULL;
00869 
00870             // load calibration files
00871             KMO_TRY_EXIT_IF_NULL(
00872                 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle,
00873                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
00874 
00875             KMO_TRY_EXIT_IF_ERROR(
00876                 cpl_vector_set(calAngles, 0, rotangle_found));
00877             KMO_TRY_EXIT_IF_NULL(
00878                 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle,
00879                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
00880             KMO_TRY_EXIT_IF_ERROR(
00881                 cpl_vector_set(calAngles, 1, rotangle_found));
00882             KMO_TRY_EXIT_IF_NULL(
00883                 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle,
00884                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
00885             KMO_TRY_EXIT_IF_ERROR(
00886                 cpl_vector_set(calAngles, 2, rotangle_found));
00887 
00888             // load bad pixel mask from XCAL and set NaNs to 0 and all other values to 1
00889             KMO_TRY_EXIT_IF_NULL(
00890                 bad_pix_mask = cpl_image_duplicate(xcal));
00891 
00892             KMO_TRY_EXIT_IF_NULL(
00893                 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask));
00894             for (ix = 0; ix < nx; ix++) {
00895                 for (iy = 0; iy < ny; iy++) {
00896                     if (isnan(pbad_pix_mask[ix+nx*iy])) {
00897                         pbad_pix_mask[ix+nx*iy] = 0.;
00898                     } else {
00899                         pbad_pix_mask[ix+nx*iy] = 1.;
00900                     }
00901                 }
00902             }
00903             KMO_TRY_CHECK_ERROR_STATE();
00904 
00905             //
00906             // reconstruct
00907             //
00908             print_warning_once_reconstruct = FALSE;
00909 
00910             // ESO INS FILTi ID
00911             KMO_TRY_EXIT_IF_NULL(
00912                 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr,
00913                                       IFU_FILTID_POSTFIX));
00914             KMO_TRY_EXIT_IF_NULL(
00915                 filter_id = cpl_propertylist_get_string(main_header, keyword));
00916             cpl_free(keyword); keyword = NULL;
00917 
00918             KMO_TRY_EXIT_IF_NULL(
00919                 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0));
00920             KMO_TRY_EXIT_IF_ERROR(
00921                 kmclipm_setup_grid_band_lcal(&gd, filter_id, band_table));
00922             cpl_table_delete(band_table); band_table = NULL;
00923 
00924             cpl_msg_info("","Reconstructing cubes...");
00925             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
00926                 // update sub-header
00927                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + i + 1;
00928 
00929                 // load raw image and sub-header
00930                 KMO_TRY_EXIT_IF_NULL(
00931                     sub_header = kmo_dfs_load_sub_header(frameset, FLAT_SKY_FLAT,
00932                                                          det_nr, FALSE));
00933 
00934                 KMO_TRY_EXIT_IF_NULL(
00935                     punused_ifus = cpl_array_get_data_int_const(
00936                                                   unused_ifus_after[det_nr-1]));
00937 
00938                 // check if IFU is valid according to main header keywords &
00939                 // calibration files
00940                 KMO_TRY_EXIT_IF_NULL(
00941                     keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr,
00942                                           IFU_VALID_POSTFIX));
00943                 KMO_TRY_CHECK_ERROR_STATE();
00944 
00945                 // just to see if keyword exists
00946                 cpl_propertylist_get_string(main_header, keyword);
00947                 cpl_free(keyword); keyword = NULL;
00948 
00949                 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) &&
00950                     (bounds[2*(ifu_nr-1)] != -1) &&
00951                     (bounds[2*(ifu_nr-1)+1] != -1) &&
00952                     (punused_ifus[i] == 0))
00953                 {
00954                     // IFU is valid
00955                     cpl_error_reset();
00956 
00957                     // calculate WCS
00958                     KMO_TRY_EXIT_IF_ERROR(
00959                         kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd));
00960 
00961                     // reconstruct data
00962                     KMO_TRY_EXIT_IF_ERROR(
00963                         kmo_reconstruct_sci_image(ifu_nr,
00964                                                 bounds[2*(ifu_nr-1)],
00965                                                 bounds[2*(ifu_nr-1)+1],
00966                                                 combined_data,
00967                                                 NULL,
00968                                                 img_dark,
00969                                                 NULL,
00970                                                 img_flat,
00971                                                 NULL,
00972                                                 xcal,
00973                                                 ycal,
00974                                                 lcal,
00975                                                 &gd,
00976                                                 calTimestamp,
00977                                                 calAngles,
00978                                                 fn_lut,
00979                                                 &cube_data,
00980                                                 NULL,
00981                                                 flux,
00982                                                 background,
00983                                                 NULL,
00984                                                 NULL,
00985                                                 NULL));
00986                     KMO_TRY_CHECK_ERROR_STATE();
00987                 } else {
00988                     // IFU is invalid
00989                     cpl_error_reset();
00990                 } // if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) ...
00991 
00992                 // save output
00993                 KMO_TRY_EXIT_IF_NULL(
00994                     extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA));
00995 
00996                 KMO_TRY_EXIT_IF_ERROR(
00997                     kmclipm_update_property_string(sub_header, EXTNAME,
00998                                                    extname,
00999                                                    "FITS extension name"));
01000 
01001                 cpl_free(extname); extname = NULL;
01002 
01003                 // store cube and sub header into array for later
01004                 stored_data_cubes[ifu_nr - 1] = cube_data;
01005                 stored_sub_headers[ifu_nr - 1] = sub_header;
01006 
01007                 cpl_image_delete(data_ifu); data_ifu = NULL;
01008                 cpl_image_delete(noise_ifu); noise_ifu = NULL;
01009                 cube_data = NULL;
01010             } // for i IFUs
01011 
01012             // free memory
01013             cpl_image_delete(combined_data); combined_data = NULL;
01014             cpl_image_delete(xcal); xcal = NULL;
01015             cpl_image_delete(ycal); ycal = NULL;
01016             cpl_image_delete(lcal); lcal = NULL;
01017             cpl_image_delete(bad_pix_mask); bad_pix_mask = NULL;
01018         } // for nr_devices
01019 
01020         cpl_image_delete(img_dark); img_dark = NULL;
01021         cpl_image_delete(img_flat); img_flat = NULL;
01022 
01023         //
01024         // collapse cubes using rejection and apply median filtering
01025         //
01026         KMO_TRY_EXIT_IF_NULL(
01027             identified_slices = cpl_vector_new(gd.l.dim));
01028         KMO_TRY_EXIT_IF_ERROR(
01029             cpl_vector_fill(identified_slices, 1));
01030 
01031         cpl_msg_info("","Collapsing cubes...");
01032         for (j = 0; j < nr_devices; j++) {
01033             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01034                 ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i;
01035                 KMO_TRY_EXIT_IF_NULL(
01036                     punused_ifus = cpl_array_get_data_int_const(
01037                                                   unused_ifus_after[j]));
01038                 if (punused_ifus[i] == 0) {
01039                     if (stored_data_cubes[ifu_nr] != NULL) {
01040                         KMO_TRY_EXIT_IF_ERROR(
01041                             kmclipm_make_image(stored_data_cubes[ifu_nr],
01042                                                NULL,
01043                                                &stored_data_images[ifu_nr],
01044                                                &stored_noise_images[ifu_nr],
01045                                                identified_slices,
01046                                                cmethod, cpos_rej, cneg_rej,
01047                                                citer, cmax, cmin));
01048                     }
01049 
01050                     //
01051                     // apply median smoothing
01052                     //
01053 
01054                     // taking care of edges (IFUs 1-16 top/bottom, IFUs 17-24 left/right)
01055                     if (ifu_nr+1 <= 2*KMOS_IFUS_PER_DETECTOR) {
01056                         firstx = 0;
01057                         lastx = 13;
01058                         firsty = 1;
01059                         lasty = 12;
01060                     } else {
01061                         firstx = 1;
01062                         lastx= 12;
01063                         firsty = 0;
01064                         lasty = 13;
01065                     }
01066 
01067                     KMO_TRY_EXIT_IF_NULL(
01068                         tmp_img = cpl_image_duplicate(stored_data_images[ifu_nr]));
01069                     KMO_TRY_EXIT_IF_NULL(
01070                         pdata = cpl_image_get_data_float(tmp_img));
01071                     KMO_TRY_EXIT_IF_NULL(
01072                         pnoise = cpl_image_get_data_float(stored_noise_images[ifu_nr]));
01073                     nx = cpl_image_get_size_x(tmp_img);
01074                     ny = cpl_image_get_size_y(tmp_img);
01075                     KMO_TRY_CHECK_ERROR_STATE();
01076 
01077                     // median filtering
01078                     for (ix = 0; ix < nx; ix++) {
01079                         for (iy = 0; iy < ny; iy++) {
01080                             if (ix-mhalf > firstx) { xmin = ix-mhalf; } else { xmin = firstx; }
01081                             if (ix+mhalf < lastx)  { xmax = ix+mhalf; } else { xmax = lastx; }
01082                             if (iy-mhalf > firsty) { ymin = iy-mhalf; } else { ymin = firsty; }
01083                             if (iy+mhalf < lasty)  { ymax = iy+mhalf; } else { ymax = lasty; }
01084 
01085                             pdata[ix+nx*iy] = cpl_image_get_median_window(
01086                                                                 stored_data_images[ifu_nr],
01087                                                                 xmin+1, ymin+1, xmax+1, ymax+1);
01088                             KMO_TRY_CHECK_ERROR_STATE();
01089 
01090                             if (stored_noise_images[ifu_nr] != NULL) {
01091                                 boxsize = (xmax-xmin+1)*(ymax-ymin+1);
01092                                 pnoise[ix+nx*iy] /= boxsize; //sqrt(boxsize*boxsize)
01093                             }
01094 
01095                         }
01096                     }
01097 
01098                     // replace images
01099                     cpl_image_delete(stored_data_images[ifu_nr]);
01100                     stored_data_images[ifu_nr] = tmp_img;
01101                 } else {
01102                     // IFU is invalid
01103                 }
01104             } // end for (i) ifu_nr
01105         } // end for (j) det_nr
01106         cpl_vector_delete(identified_slices); identified_slices = NULL;
01107 
01108         // normalise all IFUs of a detector as a group.
01109         // Calculate mean of each IFU, add up and divide by number of successful
01110         // averaged IFUs.
01111         // Then divide all valid IFUs with mean value
01112         for (j = 0; j < nr_devices; j++) {
01113             cnt = 0;
01114             mean_data = 0;
01115             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01116                 ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i;
01117                 if (stored_data_images[ifu_nr] != NULL) {
01118                     KMO_TRY_ASSURE(cpl_image_count_rejected(stored_data_images[ifu_nr]) <
01119                                    cpl_image_get_size_x(stored_data_images[ifu_nr])*
01120                                    cpl_image_get_size_y(stored_data_images[ifu_nr]),
01121                                    CPL_ERROR_ILLEGAL_INPUT,
01122                                    "The collapsed image contains only invalid values!");
01123                     mean_data += cpl_image_get_mean(stored_data_images[ifu_nr]);
01124                     KMO_TRY_CHECK_ERROR_STATE();
01125                     cnt++;
01126                 }
01127 
01128             } // end for (i) ifu_nr
01129             mean_data /= cnt;
01130             if (mean_data != 0.0) {
01131                 for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01132                     ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i;
01133                     if (stored_data_images[ifu_nr] != NULL) {
01134                         KMO_TRY_EXIT_IF_ERROR(
01135                             cpl_image_divide_scalar(stored_data_images[ifu_nr], mean_data));
01136                     }
01137                     if (stored_noise_images[ifu_nr] != NULL) {
01138                         KMO_TRY_EXIT_IF_ERROR(
01139                             cpl_image_divide_scalar(stored_noise_images[ifu_nr], mean_data));
01140                     }
01141                 } // end for (i) ifu_nr
01142             } else {
01143                 cpl_msg_warning(cpl_func, "Data couldn't be normalised (mean = 0.0)!");
01144             }
01145         } // end for (j) det_nr
01146 
01147         //
01148         // invert data and noise
01149         //
01150         double old_val  = 0.,
01151                new_val  = 0.;
01152         for (j = 0; j < nr_devices; j++) {
01153             cnt = 0;
01154             mean_data = 0;
01155             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01156                 ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i;
01157                 if (stored_data_images[ifu_nr] != NULL) {
01158                     // invert data
01159                     KMO_TRY_EXIT_IF_NULL(
01160                         pdata = cpl_image_get_data_float(stored_data_images[ifu_nr]));
01161                     if (stored_noise_images[ifu_nr] != NULL) {
01162                         KMO_TRY_EXIT_IF_NULL(
01163                             pnoise = cpl_image_get_data_float(stored_noise_images[ifu_nr]));
01164                     }
01165                     for (ix = 0; ix < nx; ix++) {
01166                         for (iy = 0; iy < ny; iy++) {
01167                             old_val = pdata[ix+nx*iy];
01168                             pdata[ix+nx*iy] = 1. / pdata[ix+nx*iy];
01169                             if (stored_noise_images[ifu_nr] != NULL) {
01170                                 new_val = pdata[ix+nx*iy];
01171                                 pnoise[ix+nx*iy] = sqrt(pow(new_val, 2) *
01172                                                         pow(pnoise[ix+nx*iy], 2) /
01173                                                         pow(old_val, 2));
01174                             }
01175                         }
01176                     }
01177                 }
01178             } // end for (i) ifu_nr
01179         } // end for (j) det_nr
01180 
01181         // calculate qc parameters on normalised data
01182         qc_spat_unif = 0.0;
01183         cnt = 0;
01184         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01185             if (stored_data_images[i] != NULL) {
01186                 tmp_mean = cpl_image_get_mean(stored_data_images[i]);
01187                 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]);
01188 
01189                 qc_spat_unif += pow(tmp_mean-1, 2);
01190                 if (fabs(tmp_mean) > qc_max_dev) {
01191                     qc_max_dev = tmp_mean-1;
01192                     qc_max_dev_id = i+1;
01193                 }
01194                 if (fabs(tmp_stdev) > qc_max_nonunif) {
01195                     qc_max_nonunif = tmp_stdev;
01196                     qc_max_nonunif_id = i+1;
01197                 }
01198                 KMO_TRY_CHECK_ERROR_STATE();
01199                 cnt++;
01200             }
01201         }
01202         qc_spat_unif = sqrt(qc_spat_unif / cnt);
01203 
01204         //
01205         // save data
01206         //
01207 
01208         // update which IFUs are not used
01209         kmo_print_unused_ifus(unused_ifus_after, TRUE);
01210 
01211         KMO_TRY_EXIT_IF_ERROR(
01212             kmo_set_unused_ifus(unused_ifus_after, main_header,
01213                                 "kmo_illumination_flat"));
01214 
01215         cpl_msg_info("","Saving data...");
01216 
01217         KMO_TRY_EXIT_IF_ERROR(
01218             kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif,
01219                                            "[adu] uniformity of illumination correction"));
01220         KMO_TRY_EXIT_IF_ERROR(
01221             kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev,
01222                                            "[adu] max. deviation from unity"));
01223         KMO_TRY_EXIT_IF_ERROR(
01224             kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id,
01225                                         "[] IFU ID with max. dev. from unity"));
01226         KMO_TRY_EXIT_IF_ERROR(
01227             kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, qc_max_nonunif,
01228                                            "[adu] max. stdev of illumination corr."));
01229         KMO_TRY_EXIT_IF_ERROR(
01230             kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, qc_max_nonunif_id,
01231                                         "[] IFU ID with max. stdev in illum. corr."));
01232 
01233         if (!suppress_extension) {
01234             KMO_TRY_EXIT_IF_NULL(
01235                 fn_suffix = cpl_sprintf("%s", suffix));
01236         } else {
01237             KMO_TRY_EXIT_IF_NULL(
01238                 fn_suffix = cpl_sprintf("%s", ""));
01239         }
01240         KMO_TRY_EXIT_IF_ERROR(
01241             kmo_dfs_save_main_header(frameset, ILLUM_CORR_FLAT, fn_suffix, frame,
01242                                      main_header, parlist, cpl_func));
01243 
01244         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01245             cpl_propertylist_erase(stored_sub_headers[i], CRPIX3);
01246             cpl_propertylist_erase(stored_sub_headers[i], CRVAL3);
01247             cpl_propertylist_erase(stored_sub_headers[i], CDELT3);
01248             cpl_propertylist_erase(stored_sub_headers[i], CTYPE3);
01249             cpl_propertylist_erase(stored_sub_headers[i], CUNIT3);
01250             cpl_propertylist_erase(stored_sub_headers[i], CD1_3);
01251             cpl_propertylist_erase(stored_sub_headers[i], CD2_3);
01252             cpl_propertylist_erase(stored_sub_headers[i], CD3_3);
01253             cpl_propertylist_erase(stored_sub_headers[i], CD3_2);
01254             cpl_propertylist_erase(stored_sub_headers[i], CD3_1);
01255 
01256             KMO_TRY_EXIT_IF_ERROR(
01257                 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR_FLAT, fn_suffix,
01258                                    stored_sub_headers[i], 0./0.));
01259 
01260             KMO_TRY_EXIT_IF_NULL(
01261                 tmp_str = cpl_propertylist_get_string(stored_sub_headers[i], EXTNAME));
01262             KMO_TRY_EXIT_IF_ERROR(
01263                 kmo_extname_extractor(tmp_str, &fr_type, &ifu_nr, content));
01264             KMO_TRY_EXIT_IF_NULL(
01265                 extname = kmo_extname_creator(ifu_frame, ifu_nr,
01266                                               EXT_NOISE));
01267             KMO_TRY_EXIT_IF_ERROR(
01268                 kmclipm_update_property_string(stored_sub_headers[i], EXTNAME,
01269                                                extname, "FITS extension name"));
01270             cpl_free(extname); extname = NULL;
01271 
01272             KMO_TRY_EXIT_IF_ERROR(
01273                 kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR_FLAT,
01274                                    fn_suffix, stored_sub_headers[i], 0./0.));
01275         }
01276     }
01277     KMO_CATCH
01278     {
01279         KMO_CATCH_MSG();
01280         ret_val = -1;
01281     }
01282     kmo_free_fits_desc(&desc_sky);
01283     kmo_free_fits_desc(&desc_xcal);
01284     kmo_free_fits_desc(&desc_ycal);
01285     kmo_free_fits_desc(&desc_lcal);
01286     cpl_image_delete(combined_data); combined_data = NULL;
01287     cpl_image_delete(xcal); xcal = NULL;
01288     cpl_image_delete(ycal); ycal = NULL;
01289     cpl_image_delete(lcal); lcal = NULL;
01290     cpl_image_delete(img_dark); img_dark = NULL;
01291     cpl_image_delete(img_flat); img_flat = NULL;
01292     cpl_array_delete(calTimestamp); calTimestamp = NULL;
01293     cpl_free(bounds); bounds = NULL;
01294     kmo_free_unused_ifus(unused_ifus_before); unused_ifus_before = NULL;
01295     kmo_free_unused_ifus(unused_ifus_after); unused_ifus_after = NULL;
01296     cpl_free(fn_lut); fn_lut = NULL;
01297     cpl_free(suffix); suffix = NULL;
01298     cpl_free(fn_suffix); fn_suffix = NULL;
01299     cpl_free(filter); filter = NULL;
01300     if (calAngles != NULL) {
01301         cpl_vector_delete(calAngles); calAngles = NULL;
01302     }
01303     cpl_propertylist_delete(main_header); main_header = NULL;
01304     for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01305         if (stored_data_cubes != NULL) {
01306             cpl_imagelist_delete(stored_data_cubes[i]);
01307             stored_data_cubes[i] = NULL;
01308         }
01309         if (stored_data_images != NULL) {
01310             cpl_image_delete(stored_data_images[i]);
01311             stored_data_images[i] = NULL;
01312         }
01313         if (stored_noise_images != NULL) {
01314             cpl_image_delete(stored_noise_images[i]);
01315             stored_noise_images[i] = NULL;
01316         }
01317         if (stored_sub_headers != NULL) {
01318             cpl_propertylist_delete(stored_sub_headers[i]);
01319             stored_sub_headers[i] = NULL;
01320         }
01321     }
01322     cpl_free(stored_data_cubes); stored_data_cubes = NULL;
01323     cpl_free(stored_data_images); stored_data_images = NULL;
01324     cpl_free(stored_noise_images); stored_noise_images = NULL;
01325     cpl_free(stored_sub_headers); stored_sub_headers = NULL;
01326 
01327     return ret_val;
01328 }
01329