KMOS Pipeline Reference Manual  1.3.11
kmo_combine.c
00001 /* 
00002  * This file is part of the KMOS Pipeline
00003  * Copyright (C) 2002,2003 European Southern Observatory
00004  *
00005  * This program is free software; you can redistribute it and/or modify
00006  * it under the terms of the GNU General Public License as published by
00007  * the Free Software Foundation; either version 2 of the License, or
00008  * (at your option) any later version.
00009  *
00010  * This program is distributed in the hope that it will be useful,
00011  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00012  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00013  * GNU General Public License for more details.
00014  *
00015  * You should have received a copy of the GNU General Public License
00016  * along with this program; if not, write to the Free Software
00017  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00018  */
00019 
00020 #ifdef HAVE_CONFIG_H
00021 #include <config.h>
00022 #endif
00023 
00024 #include <string.h>
00025 #include <math.h>
00026 
00027 #include <cpl.h>
00028 #include <cpl_wcs.h>
00029 
00030 #include "kmo_debug.h"
00031 #include "kmo_utils.h"
00032 #include "kmo_dfs.h"
00033 #include "kmo_error.h"
00034 #include "kmo_priv_functions.h"
00035 #include "kmo_cpl_extensions.h"
00036 #include "kmo_constants.h"
00037 #include "kmo_priv_combine.h"
00038 
00039 static int kmo_combine_create(cpl_plugin *);
00040 static int kmo_combine_exec(cpl_plugin *);
00041 static int kmo_combine_destroy(cpl_plugin *);
00042 static int kmo_combine(cpl_parameterlist *, cpl_frameset *);
00043 
00044 static char kmo_combine_description[] =
00045 "This recipe shifts several exposures of an object and combines them. The diffe-\n"
00046 "rent methods to match the exposures are described below (--method parameter).\n"
00047 "The output cube is larger than the input cubes, according to the shifts to be\n"
00048 "applied. Additionally a border of NaN values is added. The WCS is the same as\n"
00049 "for the first exposure.\n"
00050 "For each spatial/spectral pixel a new value will be calculated (according the\n"
00051 "--cmethod parameter) and written into the output cube.\n"
00052 "Only exposures with equal orientation regarding the WCS can be combined (except\n"
00053 "-–method=”none”), north must point to the same direction. It is recommended to\n"
00054 "apply any rotation possibly after combining.\n"
00055 "\n"
00056 "The behavior of the selection of IFUs to combine differs for some templates and\n"
00057 "can be controlled with the parameters --name and --ifus.\n"
00058 "If the input data cubes stem from templates KMOS_spec_obs_mapping8 or\n"
00059 "KMOS_spec_obs_mapping24 all extensions from all input frames are combined into\n"
00060 "a single map by default (like in recipe kmo_sci_red). If just the area of a\n"
00061 "specific IFU should be combined, the parameter --ifus can be specified, or more\n"
00062 "easily --name.\n"
00063 "If the input data cubes stem from other templates like e.g.\n"
00064 "KMOS_spec_obs_freedither all extensions of all input frames are combined into\n"
00065 "several output frames by default. The input IFUs are grouped according their\n"
00066 "targeted object name stored in the keywords ESO OCS ARMx NAME. If just a\n"
00067 "specific object should be combined, its name can be specified with parameter\n"
00068 "--name. If arbitrary IFUs shoukd be comined, one can specify these with the\n"
00069 "parameter --ifus.\n"
00070 "\n"
00071 "The default mapping mode is done via the --name parameter, where the name of\n"
00072 "the object has to be provided. The recipe searches in all input data cubes IFUs\n"
00073 "pointing to that object.\n"
00074 "\n"
00075 "BASIC PARAMETERS:\n"
00076 "-----------------\n"
00077 "--name\n"
00078 "--ifus\n"
00079 "Since an object can be present only once per exposure and since it can be\n"
00080 "located in different IFUs for the existing exposures, there are two modes to\n"
00081 "identify the objects:\n"
00082 "   * Combine by object names (default)\n"
00083 "   In this case the object name must be provided via the --name parameter. The\n"
00084 "   object name will be searched for in all primary headers of all provided frames\n"
00085 "   in the keyword ESO OCS ARMx NAME.\n"
00086 "\n"
00087 "   * Combine by index (advanced)\n"
00088 "   In this case the --ifus parameter must be provided. The parameter must have\n"
00089 "   the same number of entries as frames are provided, e.g. \"3;1;24\" for 3 expo-\n"
00090 "   sures. The index doesn't reference the extension in the frame but the real\n"
00091 "   index of the IFU as defined in the EXTNAME keyword (e.g. 'IFU.3.DATA').\n"
00092 "\n"
00093 "--method\n"
00094 "There are following sources to get the shift parameters from:\n"
00095 "   * 'none' (default)\n"
00096 "   The cubes are directly recombined, not shifting at all. The ouput frame will\n"
00097 "   have the same dimensions as the input cubes.\n"
00098 "   If the size differs a warning will be emitted and the cubes will be aligned\n"
00099 "   to the lower left corner. If the orientation differs a warning will be emit-\n"
00100 "   ted, but the cubes are combined anyway.\n"
00101 "\n"
00102 "   * 'header'\n"
00103 "   The shifts are calculated according to the WCS information stored in the\n"
00104 "   header of every IFU. The output frame will get larger, except the object is\n"
00105 "   at the exact same position for all exposures. The size of the exposures can\n"
00106 "   differ, but the orientation must be the same for all exposures.\n"
00107 "\n"
00108 "   * 'center'\n"
00109 "   The shifts are calculated using a centering algorithm. The cube will be col-\n"
00110 "   lapsed and a 2D profile will be fitted to it to identify the centre. With \n"
00111 "   the parameter --fmethod the function to fit can be provided. The size of the\n"
00112 "   exposures can differ, but the orientation must be the same for all exposures.\n"
00113 "\n"
00114 "   * 'user'\n"
00115 "   Read the shifts from a user specified file. The path of the file must be pro-\n"
00116 "   vided using the --filename parameter. For every exposure (except the first one)\n"
00117 "   two shift values are expected per line, they have to be separated with simple\n"
00118 "   spaces. The values indicate pixel shifts and are referenced to the first\n"
00119 "   frame. The 1st value is the shift in x-direction to the left, the 2nd the\n"
00120 "   shift in y-direction upwards. The size of the exposures can differ, but the\n"
00121 "   orientation must be the same for all exposures.\n"
00122 "\n"
00123 "--cmethod\n"
00124 "Following methods of frame combination are available:\n"
00125 "   * 'ksigma' (Default)\n"
00126 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00127 "   are examined. If they deviate significantly, they will be rejected according\n"
00128 "   to the conditions:\n"
00129 "       val > mean + stdev * cpos_rej\n"
00130 "   and\n"
00131 "       val < mean - stdev * cneg_rej\n"
00132 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00133 "   parameters. In the first iteration median and percentile level are used.\n"
00134 "\n"
00135 "   * 'median'\n"
00136 "   At each pixel position the median is calculated.\n"
00137 "\n"
00138 "   * 'average'\n"
00139 "   At each pixel position the average is calculated.\n"
00140 "\n"
00141 "   * 'sum'\n"
00142 "   At each pixel position the sum is calculated.\n"
00143 "\n"
00144 "   * 'min_max'\n"
00145 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00146 "   --cmax and --cmin apply to this method.\n"
00147 "\n"
00148 "ADVANCED PARAMETERS\n"
00149 "-------------------\n"
00150 "--edge_nan\n"
00151 "Set borders of two sides of the cubes to NaN before combining them. This minimises\n"
00152 "unwanted border effects when dithering.\n"
00153 "\n"
00154 "--fmethod\n"
00155 "see --method='center'\n"
00156 "The type of function that should be fitted spatially to the collapsed image.\n"
00157 "This fit is used to create a mask to extract the spectrum of the object. Valid\n"
00158 "values are “gauss” and “moffat”.\n"
00159 "\n"
00160 "--filename\n"
00161 "see --method='user'\n"
00162 "\n"
00163 "--cpos_rej\n"
00164 "--cneg_rej\n"
00165 "--citer\n"
00166 "see --cmethod='ksigma'\n"
00167 "\n"
00168 "--cmax\n"
00169 "--cmin\n"
00170 "see --cmethod='min_max'\n"
00171 "\n"
00172 "--flux\n"
00173 "Specify if flux conservation should be applied.\n"
00174 "\n"
00175 "--suppress_extension\n"
00176 "If set to TRUE, the arbitrary filename extensions are supressed. If multiple\n"
00177 "products with the same category are produced, they will be numered consecutively\n"
00178 "starting from 0.\n"
00179 "\n"
00180 "-------------------------------------------------------------------------------\n"
00181 "  Input files:\n"
00182 "\n"
00183 "   DO                      KMOS                                                \n"
00184 "   category                Type   Explanation                  Required #Frames\n"
00185 "   --------                -----  -----------                  -------- -------\n"
00186 "   <none or any>           F3I    data frame                       Y      2-n  \n"
00187 "\n"
00188 "  Output files:\n"
00189 "\n"
00190 "   DO                      KMOS\n"
00191 "   category                Type   Explanation\n"
00192 "   --------                -----  -----------\n"
00193 "   COMBINE_<ESO PRO CATG>  F3I    Combined data cube\n"
00194 "   EXP_MASK_<ESO PRO CATG> F3I    Exposure time mask\n"
00195 "-------------------------------------------------------------------------------\n"
00196 "\n";
00197 
00214 int cpl_plugin_get_info(cpl_pluginlist *list)
00215 {
00216     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00217     cpl_plugin *plugin = &recipe->interface;
00218 
00219     cpl_plugin_init(plugin,
00220                         CPL_PLUGIN_API,
00221                         KMOS_BINARY_VERSION,
00222                         CPL_PLUGIN_TYPE_RECIPE,
00223                         "kmo_combine",
00224                         "Combine reconstructed cubes",
00225                         kmo_combine_description,
00226                         "Alex Agudo Berbel",
00227                         "usd-help@eso.org",
00228                         kmos_get_license(),
00229                         kmo_combine_create,
00230                         kmo_combine_exec,
00231                         kmo_combine_destroy);
00232 
00233     cpl_pluginlist_append(list, plugin);
00234 
00235     return 0;
00236 }
00237 
00245 static int kmo_combine_create(cpl_plugin *plugin)
00246 {
00247     cpl_recipe *recipe;
00248     cpl_parameter *p;
00249 
00250     /* Check that the plugin is part of a valid recipe */
00251     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00252         recipe = (cpl_recipe *)plugin;
00253     else
00254         return -1;
00255 
00256     /* Create the parameters list in the cpl_recipe object */
00257     recipe->parameters = cpl_parameterlist_new();
00258 
00259     /* Fill the parameters list */
00260     /* --name */
00261     p = cpl_parameter_new_value("kmos.kmo_combine.name",
00262                                 CPL_TYPE_STRING,
00263                                 "Name of the object to combine.",
00264                                 "kmos.kmo_combine",
00265                                 "");
00266     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "name");
00267     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00268     cpl_parameterlist_append(recipe->parameters, p);
00269 
00270     /* --ifus */
00271     p = cpl_parameter_new_value("kmos.kmo_combine.ifus",
00272                                 CPL_TYPE_STRING,
00273                                 "The indices of the IFUs to combine. "
00274                                 "\"ifu1;ifu2;...\"",
00275                                 "kmos.kmo_combine",
00276                                 "");
00277     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "ifus");
00278     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00279     cpl_parameterlist_append(recipe->parameters, p);
00280 
00281     /* --method */
00282     p = cpl_parameter_new_value("kmos.kmo_combine.method",
00283                                 CPL_TYPE_STRING,
00284                                 "The shifting method:   "
00285                                 "'none': no shifting, combined directly "
00286                                                                   "(default), "
00287                                 "'header': shift according to WCS, "
00288                                 "'center': centering algorithm, "
00289                                 "'user': read shifts from file",
00290                                 "kmos.kmo_combine",
00291                                 "none");
00292     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "method");
00293     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00294     cpl_parameterlist_append(recipe->parameters, p);
00295 
00296     /* --fmethod */
00297     p = cpl_parameter_new_value("kmos.kmo_combine.fmethod",
00298                                 CPL_TYPE_STRING,
00299                                 "The fitting method (applies only when "
00300                                 "method='center'):   "
00301                                 "'gauss': fit a gauss function to collapsed "
00302                                 "image (default), "
00303                                 "'moffat': fit a moffat function to collapsed"
00304                                 " image",
00305                                 "kmos.kmo_combine",
00306                                 "gauss");
00307     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "fmethod");
00308     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00309     cpl_parameterlist_append(recipe->parameters, p);
00310 
00311     /* --filename */
00312     p = cpl_parameter_new_value("kmos.kmo_combine.filename",
00313                                 CPL_TYPE_STRING,
00314                                 "The path to the file with the shift vectors."
00315                                 "(Applies only to method='user')",
00316                                 "kmos.kmo_combine",
00317                                 "");
00318     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "filename");
00319     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00320     cpl_parameterlist_append(recipe->parameters, p);
00321 
00322     /* --flux */
00323     p = cpl_parameter_new_value("kmos.kmo_combine.flux",
00324                                 CPL_TYPE_BOOL,
00325                                 "Apply flux conservation: "
00326                                 "(TRUE (apply) or "
00327                                 "FALSE (don't apply)",
00328                                 "kmos.kmo_combine",
00329                                 FALSE);
00330     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00331     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00332     cpl_parameterlist_append(recipe->parameters, p);
00333 
00334     /* --edge_nan */
00335     p = cpl_parameter_new_value("kmos.kmo_combine.edge_nan",
00336                                 CPL_TYPE_BOOL,
00337                                 "Set borders of cubes to NaN before combining them."
00338                                 "(TRUE (apply) or "
00339                                 "FALSE (don't apply)",
00340                                 "kmos.kmo_combine",
00341                                 FALSE);
00342     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "edge_nan");
00343     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00344     cpl_parameterlist_append(recipe->parameters, p);
00345 
00346     /* --suppress_extension */
00347     p = cpl_parameter_new_value("kmos.kmo_combine.suppress_extension",
00348                                 CPL_TYPE_BOOL,
00349                                 "Suppress arbitrary filename extension."
00350                                 "(TRUE (apply) or FALSE (don't apply)",
00351                                 "kmos.kmo_combine",
00352                                 FALSE);
00353     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "suppress_extension");
00354     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00355     cpl_parameterlist_append(recipe->parameters, p);
00356 
00357     return kmos_combine_pars_create(recipe->parameters,
00358                                    "kmos.kmo_combine",
00359                                    DEF_REJ_METHOD,
00360                                    FALSE);
00361 }
00362 
00368 static int kmo_combine_exec(cpl_plugin *plugin)
00369 {
00370     cpl_recipe  *recipe;
00371 
00372     /* Get the recipe out of the plugin */
00373     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00374         recipe = (cpl_recipe *)plugin;
00375     else return -1 ;
00376 
00377     return kmo_combine(recipe->parameters, recipe->frames);
00378 }
00379 
00385 static int kmo_combine_destroy(cpl_plugin *plugin)
00386 {
00387     cpl_recipe *recipe;
00388 
00389     /* Get the recipe out of the plugin */
00390     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00391         recipe = (cpl_recipe *)plugin;
00392     else return -1 ;
00393 
00394     cpl_parameterlist_delete(recipe->parameters);
00395     return 0 ;
00396 }
00397 
00412 static int kmo_combine(cpl_parameterlist *parlist, cpl_frameset *frameset)
00413 {
00414     int              ret_val             = 0,
00415                      nr_frames           = 0,
00416                      index               = 0,
00417                      data_cube_counter   = 0,
00418                      noise_cube_counter  = 0,
00419                      citer               = 0,
00420                      cmin                = 0,
00421                      cmax                = 0,
00422                      flux                = FALSE,
00423                      edge_nan            = FALSE,
00424                      name_vec_size       = 0,
00425                      found               = 0,
00426                      suppress_extension  = FALSE,
00427                      suppress_index      = 0,
00428                      i                   = 0,
00429                      j                   = 0,
00430                      ifu_nr              = 0,
00431                      nv                  = 0;
00432     double           cpos_rej            = 0.0,
00433                      cneg_rej            = 0.0;
00434     char             *tmp_str            = NULL,
00435                      *fn_combine         = NULL,
00436                      *fn_mask            = NULL,
00437                      *mapping_mode       = NULL,
00438                      *name               = NULL,
00439                      **name_vec          = NULL;
00440     const char       *method             = NULL,
00441                      *cmethod            = NULL,
00442                      *fmethod            = NULL,
00443                      *filename           = NULL,
00444                      *frame_filename     = NULL,
00445                      *ifus_txt           = NULL,
00446                      *tmp_strc           = NULL;
00447     cpl_vector       *ifus               = NULL;
00448     cpl_image        *exp_mask           = NULL;
00449     cpl_imagelist    **data_cube_list    = NULL,
00450                      **noise_cube_list   = NULL,
00451                      *cube_combined_data = NULL,
00452                      *cube_combined_noise= NULL;
00453     cpl_propertylist *main_header        = NULL,
00454                      **data_header_list  = NULL,
00455                      **noise_header_list = NULL,
00456                      *tmp_header         = NULL;
00457     cpl_frame        *frame              = NULL;
00458     cpl_size         ci                  = 0;
00459     main_fits_desc   desc;
00460     enum extrapolationType extrapol_enum = NONE_CLIPPING;
00461 
00462     KMO_TRY
00463     {
00464         /* --- check input --- */
00465         KMO_TRY_ASSURE((parlist != NULL) &&
00466                        (frameset != NULL),
00467                        CPL_ERROR_NULL_INPUT,
00468                        "Not all input data is provided!");
00469 
00470         nr_frames = cpl_frameset_get_size(frameset);
00471 
00472         KMO_TRY_ASSURE(nr_frames >= 2,
00473                        CPL_ERROR_NULL_INPUT,
00474                        "At least two frames must be provided to combine!");
00475 
00476         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_combine") == 1,
00477                        CPL_ERROR_ILLEGAL_INPUT,
00478                        "Cannot identify RAW and CALIB frames!");
00479 
00480         cpl_msg_info("", "--- Parameter setup for kmo_combine -------");
00481 
00482         KMO_TRY_EXIT_IF_NULL(
00483             method = kmo_dfs_get_parameter_string(parlist,
00484                                            "kmos.kmo_combine.method"));
00485 
00486         KMO_TRY_EXIT_IF_NULL(
00487             fmethod = kmo_dfs_get_parameter_string(parlist,
00488                                            "kmos.kmo_combine.fmethod"));
00489 
00490         KMO_TRY_ASSURE((strcmp(method, "none") == 0) ||
00491                        (strcmp(method, "header") == 0) ||
00492                        (strcmp(method, "center") == 0) ||
00493                        (strcmp(method, "user") == 0),
00494                        CPL_ERROR_ILLEGAL_INPUT,
00495                        "Following shift methods are available : 'none', "
00496                        "'header', 'center' or 'user'");
00497 
00498         if (strcmp(method, "user") == 0) {
00499             filename = kmo_dfs_get_parameter_string(parlist,
00500                                                    "kmos.kmo_combine.filename");
00501             KMO_TRY_CHECK_ERROR_STATE();
00502 
00503             KMO_TRY_ASSURE(strcmp(filename, "") != 0,
00504                            CPL_ERROR_ILLEGAL_INPUT,
00505                            "path of file with shift information must be "
00506                            "provided!");
00507 
00508             KMO_TRY_EXIT_IF_ERROR(
00509                 kmo_dfs_print_parameter_help(parlist,
00510                                              "kmos.kmo_combine.filename"));
00511         }
00512 
00513         KMO_TRY_EXIT_IF_ERROR(
00514             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_combine.method"));
00515 
00516         ifus_txt = kmo_dfs_get_parameter_string(parlist,
00517                                                   "kmos.kmo_combine.ifus");
00518         KMO_TRY_CHECK_ERROR_STATE();
00519 
00520         name = (char*)kmo_dfs_get_parameter_string(parlist, "kmos.kmo_combine.name");
00521         KMO_TRY_CHECK_ERROR_STATE();
00522 
00523         if (strcmp(ifus_txt, "") != 0) {
00524             KMO_TRY_ASSURE(strcmp(name, "") == 0,
00525                            CPL_ERROR_ILLEGAL_INPUT,
00526                            "name parameter must be NULL if IFU indices are "
00527                            "provided!");
00528 
00529             KMO_TRY_EXIT_IF_NULL(
00530                 ifus = kmo_identify_values(ifus_txt));
00531 
00532             KMO_TRY_ASSURE(cpl_vector_get_size(ifus) == nr_frames,
00533                            CPL_ERROR_ILLEGAL_INPUT,
00534                            "ifus parameter must have the same number of values "
00535                            "than frames provided ) (%lld!=%d)",
00536                            cpl_vector_get_size(ifus), nr_frames);
00537         }
00538 
00539         if (strcmp(name, "") != 0) {
00540             KMO_TRY_ASSURE(strcmp(ifus_txt, "") == 0,
00541                            CPL_ERROR_ILLEGAL_INPUT,
00542                            "ifus parameter must be NULL if name is provided!");
00543         }
00544 
00545         flux = kmo_dfs_get_parameter_bool(parlist,
00546                                           "kmos.kmo_combine.flux");
00547         KMO_TRY_CHECK_ERROR_STATE();
00548         KMO_TRY_EXIT_IF_ERROR(
00549             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_combine.flux"));
00550 
00551         KMO_TRY_ASSURE((flux == TRUE) || (flux == FALSE),
00552                        CPL_ERROR_ILLEGAL_INPUT,
00553                        "flux must be TRUE or FALSE!");
00554 
00555         edge_nan = kmo_dfs_get_parameter_bool(parlist,
00556                                           "kmos.kmo_combine.edge_nan");
00557         KMO_TRY_CHECK_ERROR_STATE();
00558         KMO_TRY_EXIT_IF_ERROR(
00559             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_combine.edge_nan"));
00560 
00561         KMO_TRY_ASSURE((edge_nan == TRUE) || (edge_nan == FALSE),
00562                        CPL_ERROR_ILLEGAL_INPUT,
00563                        "edge_nan must be TRUE or FALSE!");
00564 
00565         suppress_extension = kmo_dfs_get_parameter_bool(parlist,
00566                                           "kmos.kmo_combine.suppress_extension");
00567         KMO_TRY_CHECK_ERROR_STATE();
00568         KMO_TRY_EXIT_IF_ERROR(
00569             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_combine.suppress_extension"));
00570 
00571         KMO_TRY_ASSURE((suppress_extension == TRUE) || (suppress_extension == FALSE),
00572                        CPL_ERROR_ILLEGAL_INPUT,
00573                        "suppress_extension must be TRUE or FALSE!");
00574 
00575         KMO_TRY_EXIT_IF_ERROR(
00576             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_combine.ifus"));
00577 
00578         KMO_TRY_EXIT_IF_ERROR(
00579             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_combine.name"));
00580 
00581         KMO_TRY_EXIT_IF_ERROR(
00582             kmos_combine_pars_load(parlist,
00583                                   "kmos.kmo_combine",
00584                                   &cmethod,
00585                                   &cpos_rej,
00586                                   &cneg_rej,
00587                                   &citer,
00588                                   &cmin,
00589                                   &cmax,
00590                                   FALSE));
00591 
00592         cpl_msg_info("", "-------------------------------------------");
00593 
00594         // load data and noise
00595         KMO_TRY_EXIT_IF_NULL(
00596             data_cube_list = cpl_calloc(nr_frames*KMOS_NR_IFUS,
00597                                         sizeof(cpl_imagelist*)));
00598 
00599         KMO_TRY_EXIT_IF_NULL(
00600             data_header_list = cpl_calloc(nr_frames*KMOS_NR_IFUS,
00601                                           sizeof(cpl_propertylist*)));
00602 
00603         KMO_TRY_EXIT_IF_NULL(
00604             noise_cube_list = cpl_calloc(nr_frames*KMOS_NR_IFUS,
00605                                          sizeof(cpl_imagelist*)));
00606 
00607         KMO_TRY_EXIT_IF_NULL(
00608             noise_header_list = cpl_calloc(nr_frames*KMOS_NR_IFUS,
00609                                            sizeof(cpl_propertylist*)));
00610 
00611         //
00612         // check for mapping mode
00613         //
00614         cpl_size fs_size = cpl_frameset_get_size(frameset);
00615         KMO_TRY_CHECK_ERROR_STATE();
00616 
00617         for (ci = 0; ci < fs_size; ci++) {
00618             KMO_TRY_EXIT_IF_NULL(
00619                 frame = cpl_frameset_get_position(frameset, ci));
00620 
00621             KMO_TRY_EXIT_IF_NULL(
00622                 tmp_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00623             if (cpl_propertylist_has(tmp_header, TPL_ID)) {
00624                 KMO_TRY_EXIT_IF_NULL(
00625                     tmp_strc = cpl_propertylist_get_string(tmp_header, TPL_ID));
00626                 if (mapping_mode == NULL) {
00627                     if (strcmp(tmp_strc, MAPPING8) == 0)
00628                     {
00629                         mapping_mode = cpl_sprintf("%s", tmp_strc);
00630                     }
00631                     if (strcmp(tmp_strc, MAPPING24) == 0)
00632                     {
00633                         mapping_mode = cpl_sprintf("%s", tmp_strc);
00634                     }
00635                 } else {
00636                     if (strcmp(tmp_strc, mapping_mode) != 0)
00637                     {
00638                         cpl_msg_warning("","There are different TPL IDs present in "
00639                                            "the set of frames: %s and %s",
00640                                            tmp_strc, mapping_mode);
00641                     }
00642                 }
00643             }
00644             cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00645         }
00646 
00647         if (mapping_mode != NULL) {
00648             if ((strcmp(ifus_txt, "") == 0)  && (strcmp(name, "") == 0)) {
00649                 cpl_msg_info("","**************************************************");
00650                 cpl_msg_info("","*  A map containing all IFUs will be generated!  *");
00651                 cpl_msg_info("","**************************************************");
00652                 extrapol_enum = BCS_NATURAL;
00653             } else {
00654                 cpl_msg_info("","The frames aren't combined into a map although they originate "
00655                                 "from a mapping template. But since the name- or ifu-parameter "
00656                                 "has been specified, the default behaviour is overridden.");
00657                 cpl_free(mapping_mode); mapping_mode = NULL;
00658             }
00659         }
00660 
00661         //
00662         // create name/ifu map...
00663         //
00664         KMO_TRY_EXIT_IF_NULL(
00665             name_vec = cpl_calloc(nr_frames*KMOS_NR_IFUS, sizeof(char*)));
00666 
00667         if ((strcmp(ifus_txt, "") == 0) &&
00668             (strcmp(name, "") == 0) &&
00669             (mapping_mode == NULL))
00670         {
00671             // all available names should be combined in one go
00672             name_vec_size = 0;
00673             for (i = 0; i < nr_frames; i++) {
00674                 KMO_TRY_EXIT_IF_NULL(
00675                     tmp_str = cpl_sprintf("%d", i));
00676                 KMO_TRY_EXIT_IF_NULL(
00677                     frame = kmo_dfs_get_frame(frameset, tmp_str));
00678                 cpl_free(tmp_str); tmp_str = NULL;
00679 
00680                 for (ifu_nr = 1; ifu_nr <= KMOS_NR_IFUS; ifu_nr++) {
00681                     found = 0;
00682                     tmp_str = kmo_get_name_from_ocs_ifu(frame, ifu_nr);
00683                     KMO_TRY_CHECK_ERROR_STATE();
00684 
00685                     if (tmp_str != NULL) {
00686                         for (j = 0; j < name_vec_size; j++) {
00687                             if (strcmp(name_vec[j], tmp_str) == 0) {
00688                                 found = TRUE;
00689                                 break;
00690                             }
00691                         }
00692                         if (!found) {
00693                             name_vec[name_vec_size++] = tmp_str;
00694                         } else {
00695                             cpl_free(tmp_str); tmp_str = NULL;
00696                         }
00697                     }
00698                 }
00699             }
00700         } else {
00701             // standard behavior: either ifu_nr- or name- or mapping-case
00702             name_vec_size = 1;
00703             if (mapping_mode != NULL) {
00704                 KMO_TRY_EXIT_IF_NULL(
00705                     name_vec[0] = cpl_sprintf("mapping"));
00706             } else {
00707                 if (ifus != NULL) {
00708                     char *tmptmp = NULL;
00709 
00710                     // replace all ; with _
00711                     char *found_char = NULL;
00712                     found_char = strstr(ifus_txt, ";");
00713                     while (found_char != NULL) {
00714                         strncpy(found_char, "_", 1);
00715                         found_char = strstr(ifus_txt, ";");
00716                     }
00717 
00718                     if (strlen(ifus_txt) > 10) {
00719                         KMO_TRY_EXIT_IF_NULL(
00720                             tmptmp = kmo_shorten_ifu_string(ifus_txt));
00721                         cpl_msg_info("", "Because of lengthy ifus-parameter, filenames of products and their EXTNAME keywords will be truncated to ...ifu%s...", tmptmp);
00722                     } else {
00723                         KMO_TRY_EXIT_IF_NULL(
00724                             tmptmp = cpl_sprintf("%s", ifus_txt));
00725                     }
00726                     KMO_TRY_EXIT_IF_NULL(
00727                         name_vec[0] = cpl_sprintf("IFU%s", tmptmp));
00728                     ifus_txt = "";
00729                     cpl_free(tmptmp); tmptmp = NULL;
00730                 } else {
00731                     KMO_TRY_EXIT_IF_NULL(
00732                         name_vec[0] = cpl_sprintf("%s", name));
00733                 }
00734             }
00735         }
00736 
00737         //
00738         // load all data (and noise if existent) cubes and store them
00739         //
00740         for (nv = 0; nv < name_vec_size; nv++) {
00741             name = name_vec[nv];
00742 
00743             data_cube_counter = 0;
00744             noise_cube_counter = 0;
00745             for (i = 0; i < nr_frames; i++) {
00746                 KMO_TRY_EXIT_IF_NULL(
00747                     tmp_str = cpl_sprintf("%d", i));
00748 
00749                 KMO_TRY_EXIT_IF_NULL(
00750                     frame = kmo_dfs_get_frame(frameset, tmp_str));
00751 
00752                 KMO_TRY_EXIT_IF_NULL(
00753                     frame_filename = cpl_frame_get_filename(frame));
00754 
00755                 kmo_init_fits_desc(&desc);
00756 
00757                 desc = kmo_identify_fits_header(frame_filename);
00758                 KMO_TRY_CHECK_ERROR_STATE_MSG("Provided fits file doesn't seem to "
00759                                               "be in KMOS-format!");
00760 
00761                 KMO_TRY_ASSURE(desc.fits_type == f3i_fits,
00762                                CPL_ERROR_ILLEGAL_INPUT,
00763                                "Frame No. %d hasn't correct data type "
00764                                "(must be of type F3I)!", i+1);
00765 
00766                 if (mapping_mode != NULL) {
00767                     // we are in mapping mode
00768                     for (j = 1; j <= KMOS_NR_IFUS; j++) {
00769                         //loop over all IFUs
00770                         if (desc.sub_desc[j-1].valid_data == TRUE) {
00771                             // load data frames
00772                             override_err_msg = TRUE;
00773                             data_cube_list[data_cube_counter] =
00774                                 kmo_dfs_load_cube(frameset, tmp_str, j, FALSE);
00775                             override_err_msg = FALSE;
00776                             if (data_cube_list[data_cube_counter] == NULL) {
00777                                 // no data found for this IFU
00778                                 cpl_error_reset();
00779                             } else {
00780                                 if (edge_nan) {
00781                                     KMO_TRY_EXIT_IF_ERROR(
00782                                         kmo_edge_nan(data_cube_list[data_cube_counter], j));
00783                                 }
00784                                 KMO_TRY_EXIT_IF_NULL(
00785                                     data_header_list[data_cube_counter] =
00786                                         kmo_dfs_load_sub_header(frameset, tmp_str, j, FALSE));
00787                                 cpl_propertylist_update_string(data_header_list[data_cube_counter],
00788                                                         "ESO PRO FRNAME",
00789                                                         frame_filename);
00790                                 cpl_propertylist_update_int(data_header_list[data_cube_counter],
00791                                                         "ESO PRO IFUNR",
00792                                                         j);
00793                                 data_cube_counter++;
00794                             }
00795 
00796                             // load noise frames
00797                             override_err_msg = TRUE;
00798                             noise_cube_list[noise_cube_counter] =
00799                                 kmo_dfs_load_cube(frameset, tmp_str, j, TRUE);
00800 
00801                             override_err_msg = FALSE;
00802                             if (noise_cube_list[noise_cube_counter] == NULL) {
00803                                 // no noise found for this IFU
00804                                 cpl_error_reset();
00805                             } else {
00806                                 if (edge_nan) {
00807                                     KMO_TRY_EXIT_IF_ERROR(
00808                                         kmo_edge_nan(noise_cube_list[noise_cube_counter], j));
00809                                 }
00810                                 KMO_TRY_EXIT_IF_NULL(
00811                                     noise_header_list[noise_cube_counter] =
00812                                         kmo_dfs_load_sub_header(frameset, tmp_str, j, TRUE));
00813                                 noise_cube_counter++;
00814                             }
00815 
00816                             // check for every iteration if number of data and noise
00817                             // frames is the same
00818                             if (noise_cube_counter > 0) {
00819                                 KMO_TRY_ASSURE(data_cube_counter == noise_cube_counter,
00820                                                CPL_ERROR_ILLEGAL_INPUT,
00821                                                "Frame No. %d (%s) has no noise frame "
00822                                                "while the preceeding ones had!",
00823                                                i+1, frame_filename);
00824                             }
00825                         } // end if valid_data
00826                     }
00827                 } else {
00828                     // we are in name/ifu mode (single)
00829                     if (ifus != NULL) {
00830                         ifu_nr = cpl_vector_get(ifus, i);
00831                         KMO_TRY_CHECK_ERROR_STATE();
00832                     } else {
00833                         ifu_nr = kmo_get_index_from_ocs_name(frame, name);
00834                         KMO_TRY_CHECK_ERROR_STATE();
00835                     }
00836 
00837                     if (ifu_nr > 0) {
00838                         index = kmo_identify_index(frame_filename, ifu_nr , FALSE);
00839                         KMO_TRY_CHECK_ERROR_STATE();
00840 
00841                         if (desc.sub_desc[index-1].valid_data == TRUE) {
00842                             // load data frames
00843                             override_err_msg = TRUE;
00844                             data_cube_list[data_cube_counter] =
00845                                 kmo_dfs_load_cube(frameset, tmp_str, ifu_nr, FALSE);
00846                             override_err_msg = FALSE;
00847                             if (data_cube_list[data_cube_counter] == NULL) {
00848                                 // no data found for this IFU
00849                                 cpl_error_reset();
00850                                 if (ifus != NULL) {
00851                                     cpl_msg_warning(cpl_func, "Frame No. %d (%s) "
00852                                                     "doesn't contain IFU No. %d!", i+1,
00853                                                     frame_filename, ifu_nr);
00854                                 } else {
00855                                     cpl_msg_warning(cpl_func, "Frame No. %d (%s) "
00856                                                     "doesn't contain IFU with object "
00857                                                     "name '%s'!", i+1,
00858                                                     frame_filename, name);
00859                                 }
00860                             } else {
00861                                 if (edge_nan) {
00862                                     KMO_TRY_EXIT_IF_ERROR(
00863                                         kmo_edge_nan(data_cube_list[data_cube_counter], ifu_nr));
00864                                 }
00865 
00866                                 KMO_TRY_EXIT_IF_NULL(
00867                                     data_header_list[data_cube_counter] =
00868                                         kmo_dfs_load_sub_header(frameset, tmp_str,
00869                                                                 ifu_nr, FALSE));
00870                                 cpl_propertylist_update_string(data_header_list[data_cube_counter],
00871                                                                "ESO PRO FRNAME",
00872                                                                frame_filename);
00873                                 cpl_propertylist_update_int(data_header_list[data_cube_counter],
00874                                                             "ESO PRO IFUNR",
00875                                                             ifu_nr);
00876                                 data_cube_counter++;
00877                             }
00878 
00879                             // load noise frames
00880                             override_err_msg = TRUE;
00881                             noise_cube_list[noise_cube_counter] =
00882                                 kmo_dfs_load_cube(frameset, tmp_str, ifu_nr, TRUE);
00883                             override_err_msg = FALSE;
00884                             if (noise_cube_list[noise_cube_counter] == NULL) {
00885                                 // no noise found for this IFU
00886                                 cpl_error_reset();
00887                             } else {
00888                                 if (edge_nan) {
00889                                     KMO_TRY_EXIT_IF_ERROR(
00890                                         kmo_edge_nan(noise_cube_list[noise_cube_counter], ifu_nr));
00891                                 }
00892 
00893                                 KMO_TRY_EXIT_IF_NULL(
00894                                     noise_header_list[noise_cube_counter] =
00895                                         kmo_dfs_load_sub_header(frameset, tmp_str,
00896                                                                 ifu_nr, TRUE));
00897                                 noise_cube_counter++;
00898                             }
00899 
00900                             // check for every iteration if number of data and noise
00901                             // frames is the same
00902                             if (noise_cube_counter > 0) {
00903                                 KMO_TRY_ASSURE(data_cube_counter == noise_cube_counter,
00904                                                CPL_ERROR_ILLEGAL_INPUT,
00905                                                "Frame No. %d (%s) has no noise frame "
00906                                                "while the preceeding ones had!",
00907                                                i+1, frame_filename);
00908                             }
00909                         } // end if valid_data
00910                     } // end if (ifu_nr > 0)
00911                 }
00912 
00913                 kmo_free_fits_desc(&desc);
00914                 cpl_free(tmp_str); tmp_str = NULL;
00915             } // for i = nr_frames
00916             KMO_TRY_CHECK_ERROR_STATE();
00917 
00918             //
00919             // combine data
00920             //
00921             KMO_TRY_EXIT_IF_ERROR(
00922                 kmo_priv_combine(data_cube_list,
00923                                  noise_cube_list,
00924                                  data_header_list,
00925                                  noise_header_list,
00926                                  data_cube_counter,
00927                                  noise_cube_counter,
00928                                  name,
00929                                  ifus_txt,
00930                                  method,
00931                                  "BCS",
00932                                  fmethod,
00933                                  filename,
00934                                  cmethod,
00935                                  cpos_rej,
00936                                  cneg_rej,
00937                                  citer,
00938                                  cmin,
00939                                  cmax,
00940                                  extrapol_enum,
00941                                  flux,
00942                                  &cube_combined_data,
00943                                  &cube_combined_noise,
00944                                  &exp_mask));
00945 
00946             //
00947             // save data
00948             //
00949             /* save data and noise (if existing) */
00950             // --- load, update & save primary header ---
00951 
00952 
00953             if (!suppress_extension) {
00954                 // setup output category COMBINE + ESO PRO CATG
00955                 KMO_TRY_EXIT_IF_NULL(
00956                     main_header = kmo_dfs_load_primary_header(frameset, "0"));
00957                 KMO_TRY_EXIT_IF_NULL(
00958                     fn_combine = cpl_sprintf("%s_%s_%s",
00959                                              COMBINE,
00960                                              cpl_propertylist_get_string(main_header, CPL_DFS_PRO_CATG),
00961                                              name_vec[nv]));
00962                 KMO_TRY_EXIT_IF_NULL(
00963                     fn_mask = cpl_sprintf("%s_%s_%s",
00964                                           EXP_MASK,
00965                                           cpl_propertylist_get_string(main_header, CPL_DFS_PRO_CATG),
00966                                           name_vec[nv]));
00967                 cpl_propertylist_delete(main_header); main_header = NULL;
00968             } else {
00969                 KMO_TRY_EXIT_IF_NULL(
00970                     fn_combine = cpl_sprintf("%s_%d", COMBINE, suppress_index));
00971                 KMO_TRY_EXIT_IF_NULL(
00972                     fn_mask = cpl_sprintf("%s_%d", EXP_MASK, suppress_index++));
00973             }
00974 
00975             frame = cpl_frameset_find(frameset, NULL);
00976             KMO_TRY_EXIT_IF_ERROR(
00977                 kmo_dfs_save_main_header(frameset, fn_combine, "", frame, NULL, parlist, cpl_func));
00978             KMO_TRY_EXIT_IF_ERROR(
00979                 kmo_dfs_save_main_header(frameset, fn_mask, "", frame, NULL, parlist, cpl_func));
00980 
00981             if (data_header_list[0] != NULL) {
00982                 if (cpl_propertylist_has(data_header_list[0], "ESO PRO FRNAME")) {
00983                     cpl_propertylist_erase(data_header_list[0], "ESO PRO FRNAME");
00984                 }
00985                 if (cpl_propertylist_has(data_header_list[0], "ESO PRO IFUNR")) {
00986                     cpl_propertylist_erase(data_header_list[0], "ESO PRO IFUNR");
00987                 }
00988             }
00989             KMO_TRY_CHECK_ERROR_STATE();
00990 
00991             KMO_TRY_EXIT_IF_ERROR(
00992                 kmo_dfs_save_cube(cube_combined_data, fn_combine, "", data_header_list[0], 0./0.));
00993             KMO_TRY_EXIT_IF_ERROR(
00994                 kmo_dfs_save_cube(cube_combined_noise, fn_combine, "", noise_header_list[0], 0./0.));
00995             cpl_propertylist_erase(data_header_list[0], CRPIX3);
00996             cpl_propertylist_erase(data_header_list[0], CRPIX3);
00997             cpl_propertylist_erase(data_header_list[0], CRVAL3);
00998             cpl_propertylist_erase(data_header_list[0], CDELT3);
00999             cpl_propertylist_erase(data_header_list[0], CD1_3);
01000             cpl_propertylist_erase(data_header_list[0], CD2_3);
01001             cpl_propertylist_erase(data_header_list[0], CD3_1);
01002             cpl_propertylist_erase(data_header_list[0], CD3_2);
01003             cpl_propertylist_erase(data_header_list[0], CD3_3);
01004             cpl_propertylist_erase(data_header_list[0], CTYPE3);
01005             KMO_TRY_EXIT_IF_ERROR(
01006                 kmo_dfs_save_image(exp_mask, fn_mask, "", data_header_list[0], 0./0.));
01007 
01008             cpl_imagelist_delete(cube_combined_data); cube_combined_data = NULL;
01009             cpl_imagelist_delete(cube_combined_noise); cube_combined_noise = NULL;
01010             cpl_image_delete(exp_mask); exp_mask = NULL;
01011 
01012             if (data_cube_list != NULL) {
01013                 for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01014                     cpl_imagelist_delete(data_cube_list[i]); data_cube_list[i] = NULL;
01015                 }
01016             }
01017             if (noise_cube_list != NULL) {
01018                 for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01019                     cpl_imagelist_delete(noise_cube_list[i]); noise_cube_list[i] = NULL;
01020                 }
01021             }
01022             if (data_header_list != NULL) {
01023                 for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01024                     cpl_propertylist_delete(data_header_list[i]);
01025                     data_header_list[i] = NULL;
01026                 }
01027             }
01028             if (noise_header_list != NULL) {
01029                 for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01030                     cpl_propertylist_delete(noise_header_list[i]);
01031                     noise_header_list[i] = NULL;
01032                 }
01033             }
01034             cpl_free(fn_combine); fn_combine = NULL;
01035             cpl_free(fn_mask); fn_mask = NULL;
01036         } // end for (nv = 0; nv < name_vec_size; nv++)
01037     }
01038     KMO_CATCH
01039     {
01040         KMO_CATCH_MSG();
01041         ret_val = -1;
01042     }
01043 
01044     cpl_propertylist_delete(main_header); main_header = NULL;
01045     cpl_vector_delete(ifus); ifus = NULL;
01046     cpl_imagelist_delete(cube_combined_data); cube_combined_data = NULL;
01047     cpl_imagelist_delete(cube_combined_noise); cube_combined_noise = NULL;
01048     cpl_image_delete(exp_mask); exp_mask = NULL;
01049 
01050     if (data_cube_list != NULL) {
01051         for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01052             cpl_imagelist_delete(data_cube_list[i]); data_cube_list[i] = NULL;
01053         }
01054         cpl_free(data_cube_list); data_cube_list = NULL;
01055     }
01056 
01057     if (noise_cube_list != NULL) {
01058         for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01059             cpl_imagelist_delete(noise_cube_list[i]); noise_cube_list[i] = NULL;
01060         }
01061         cpl_free(noise_cube_list); noise_cube_list = NULL;
01062     }
01063 
01064     if (data_header_list != NULL) {
01065         for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01066             cpl_propertylist_delete(data_header_list[i]);
01067             data_header_list[i] = NULL;
01068         }
01069         cpl_free(data_header_list); data_header_list = NULL;
01070     }
01071 
01072     if (noise_header_list != NULL) {
01073         for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01074             cpl_propertylist_delete(noise_header_list[i]);
01075             noise_header_list[i] = NULL;
01076         }
01077         cpl_free(noise_header_list); noise_header_list = NULL;
01078     }
01079 
01080     if (name_vec != NULL) {
01081         for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01082             cpl_free(name_vec[i]); name_vec[i] = NULL;
01083         }
01084         cpl_free(name_vec); name_vec = NULL;
01085     }
01086     cpl_free(mapping_mode);mapping_mode = NULL;
01087 
01088     return ret_val;
01089 }
01090