KMOS Pipeline Reference Manual
1.3.11
|
00001 /* 00002 * This file is part of the KMOS Pipeline 00003 * Copyright (C) 2002,2003 European Southern Observatory 00004 * 00005 * This program is free software; you can redistribute it and/or modify 00006 * it under the terms of the GNU General Public License as published by 00007 * the Free Software Foundation; either version 2 of the License, or 00008 * (at your option) any later version. 00009 * 00010 * This program is distributed in the hope that it will be useful, 00011 * but WITHOUT ANY WARRANTY; without even the implied warranty of 00012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 00013 * GNU General Public License for more details. 00014 * 00015 * You should have received a copy of the GNU General Public License 00016 * along with this program; if not, write to the Free Software 00017 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 00018 */ 00019 00020 #ifdef HAVE_CONFIG_H 00021 #include <config.h> 00022 #endif 00023 00024 /*----------------------------------------------------------------------------- 00025 * Includes 00026 *----------------------------------------------------------------------------*/ 00027 00028 #include <math.h> 00029 #include <string.h> 00030 00031 #include <cpl.h> 00032 00033 #include "kmclipm_priv_splines.h" 00034 00035 #include "kmo_priv_reconstruct.h" 00036 #include "kmo_priv_functions.h" 00037 #include "kmo_priv_flat.h" 00038 #include "kmo_priv_wave_cal.h" 00039 #include "kmo_priv_combine.h" 00040 #include "kmo_functions.h" 00041 #include "kmo_cpl_extensions.h" 00042 #include "kmo_dfs.h" 00043 #include "kmos_pfits.h" 00044 #include "kmo_error.h" 00045 #include "kmo_constants.h" 00046 #include "kmo_debug.h" 00047 00048 /*----------------------------------------------------------------------------- 00049 * Functions prototypes 00050 *----------------------------------------------------------------------------*/ 00051 00052 static int kmos_illumination_check_inputs(cpl_frameset *, int, int *, int *, 00053 int *, int *, int *, double *) ; 00054 static cpl_table ** kmos_illumination_edge_shift_correct(cpl_image *, 00055 cpl_image *, int, const cpl_image *, int, cpl_array *, 00056 const char *, double) ; 00057 00058 static int kmos_illumination_create(cpl_plugin *); 00059 static int kmos_illumination_exec(cpl_plugin *); 00060 static int kmos_illumination_destroy(cpl_plugin *); 00061 static int kmos_illumination(cpl_parameterlist *, cpl_frameset *); 00062 00063 /*----------------------------------------------------------------------------- 00064 * Static variables 00065 *----------------------------------------------------------------------------*/ 00066 00067 static char kmos_illumination_description[] = 00068 "This recipe creates the spatial non-uniformity calibration frame needed for\n" 00069 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n" 00070 "generates the spectral calibration frame needed in this recipe. As input at\n" 00071 "least a sky, a master dark, a master flat and the spatial and spectral cali-\n" 00072 "bration frames are required.\n" 00073 "The created product, the illumination correction, can be used as input for\n" 00074 "kmo_std_star and kmo_sci_red.\n" 00075 "\n" 00076 "BASIC PARAMETERS:\n" 00077 "-----------------\n" 00078 "--imethod\n" 00079 "The interpolation method used for reconstruction.\n" 00080 "\n" 00081 "--range\n" 00082 "The spectral ranges to combine when collapsing the reconstructed cubes. e.g.\n" 00083 "\"x1_start,x1_end;x2_start,x2_end\" (microns)\n" 00084 "\n" 00085 "ADVANCED PARAMETERS\n" 00086 "-------------------\n" 00087 "--flux\n" 00088 "Specify if flux conservation should be applied.\n" 00089 "\n" 00090 "--add-all\n" 00091 "By default the first FLAT_SKY frame is omitted, since in the\n" 00092 "KMOS_spec_cal_skyflat template this is an acquisition frame to estimate the\n" 00093 "needed exposure time for the subsequent FLAT_SKY frames. If anyway all\n" 00094 "frames should be considered, set this parameter to TRUE.\n" 00095 "\n" 00096 "--neighborhoodRange\n" 00097 "Defines the range to search for neighbors during reconstruction\n" 00098 "\n" 00099 "--b_samples\n" 00100 "The number of samples in spectral direction for the reconstructed cube.\n" 00101 "Ideally this number should be greater than 2048, the detector size.\n" 00102 "\n" 00103 "--b_start\n" 00104 "--b_end\n" 00105 "Used to define manually the start and end wavelength for the reconstructed\n" 00106 "cube. By default the internally defined values are used.\n" 00107 "\n" 00108 "--cmethod\n" 00109 "Following methods of frame combination are available:\n" 00110 " * 'ksigma' (Default)\n" 00111 " An iterative sigma clipping. For each position all pixels in the spectrum\n" 00112 " are examined. If they deviate significantly, they will be rejected\n" 00113 " according to the conditions:\n" 00114 " val > mean + stdev * cpos_rej\n" 00115 " and\n" 00116 " val < mean - stdev * cneg_rej\n" 00117 " where --cpos_rej, --cneg_rej and --citer are the corresponding\n" 00118 " configuration parameters. In the first iteration median and percentile\n" 00119 " level are used.\n" 00120 "\n" 00121 " * 'median'\n" 00122 " At each pixel position the median is calculated.\n" 00123 "\n" 00124 " * 'average'\n" 00125 " At each pixel position the average is calculated.\n" 00126 "\n" 00127 " * 'sum'\n" 00128 " At each pixel position the sum is calculated.\n" 00129 "\n" 00130 " * 'min_max'\n" 00131 " The specified nb of minimum and maximum pixel values will be rejected.\n" 00132 " --cmax and --cmin apply to this method.\n" 00133 "\n" 00134 "--cpos_rej\n" 00135 "--cneg_rej\n" 00136 "--citer\n" 00137 "see --cmethod='ksigma'\n" 00138 "\n" 00139 "--cmax\n" 00140 "--cmin\n" 00141 "see --cmethod='min_max'\n" 00142 "\n" 00143 "--pix_scale\n" 00144 "Change the pixel scale [arcsec]. Default of 0.2\" results into cubes of\n" 00145 "14x14pix, a scale of 0.1\" results into cubes of 28x28pix, etc.\n" 00146 "\n" 00147 "--suppress_extension\n" 00148 "If set to TRUE, the arbitrary filename extensions are supressed. If multiple\n" 00149 "products with the same category are produced, they will be numbered\n" 00150 "consecutively starting from 0.\n" 00151 "\n" 00152 "---------------------------------------------------------------------------\n" 00153 " Input files:\n" 00154 "\n" 00155 " DO CATG Type Explanation Required #Frames\n" 00156 " -------- ----- ----------- -------- -------\n" 00157 " FLAT_SKY F2D Sky exposures Y 1-n \n" 00158 " (at least 3 frames recommended) \n" 00159 " MASTER_DARK F2D Master dark Y 1 \n" 00160 " MASTER_FLAT F2D Master flat Y 1 \n" 00161 " XCAL F2D x calibration frame Y 1 \n" 00162 " YCAL F2D y calibration frame Y 1 \n" 00163 " LCAL F2D Wavelength calib. frame Y 1 \n" 00164 " WAVE_BAND F2L Table with start-/end-wavelengths Y 1 \n" 00165 " FLAT_EDGE F2L Table with fitted slitlet edges N 0,1 \n" 00166 "\n" 00167 " Output files:\n" 00168 "\n" 00169 " DO CATG Type Explanation\n" 00170 " -------- ----- -----------\n" 00171 " ILLUM_CORR F2I Illumination calibration frame \n" 00172 " If FLAT_EDGE is provided: \n" 00173 " SKYFLAT_EDGE F2L Frame containing parameters of fitted \n" 00174 " slitlets of all IFUs of all detectors\n" 00175 "---------------------------------------------------------------------------\n" 00176 "\n"; 00177 00178 /*----------------------------------------------------------------------------- 00179 * Functions code 00180 *----------------------------------------------------------------------------*/ 00181 00182 /*----------------------------------------------------------------------------*/ 00187 /*----------------------------------------------------------------------------*/ 00188 00191 /*----------------------------------------------------------------------------*/ 00200 /*----------------------------------------------------------------------------*/ 00201 int cpl_plugin_get_info(cpl_pluginlist *list) 00202 { 00203 cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe); 00204 cpl_plugin *plugin = &recipe->interface; 00205 00206 cpl_plugin_init(plugin, 00207 CPL_PLUGIN_API, 00208 KMOS_BINARY_VERSION, 00209 CPL_PLUGIN_TYPE_RECIPE, 00210 "kmos_illumination", 00211 "Create a frame to correct spatial non-uniformity of flatfield", 00212 kmos_illumination_description, 00213 "Alex Agudo Berbel, Yves Jung", 00214 "usd-help@eso.org", 00215 kmos_get_license(), 00216 kmos_illumination_create, 00217 kmos_illumination_exec, 00218 kmos_illumination_destroy); 00219 cpl_pluginlist_append(list, plugin); 00220 00221 return 0; 00222 } 00223 00224 /*----------------------------------------------------------------------------*/ 00232 /*----------------------------------------------------------------------------*/ 00233 static int kmos_illumination_create(cpl_plugin *plugin) 00234 { 00235 cpl_recipe *recipe; 00236 cpl_parameter *p; 00237 00238 /* Check that the plugin is part of a valid recipe */ 00239 if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 00240 recipe = (cpl_recipe *)plugin; 00241 else 00242 return -1; 00243 00244 /* Create the parameters list in the cpl_recipe object */ 00245 recipe->parameters = cpl_parameterlist_new(); 00246 00247 /* Fill the parameters list */ 00248 /* --imethod */ 00249 p = cpl_parameter_new_value("kmos.kmos_illumination.imethod", 00250 CPL_TYPE_STRING, 00251 "Method to use for interpolation: " 00252 "[\"NN\" (nearest neighbour), " 00253 "\"lwNN\" (linear weighted nearest neighbor), " 00254 "\"swNN\" (square weighted nearest neighbor), " 00255 "\"MS\" (Modified Shepard's method), " 00256 "\"CS\" (Cubic spline)]", 00257 "kmos.kmos_illumination", "CS"); 00258 cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod"); 00259 cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); 00260 cpl_parameterlist_append(recipe->parameters, p); 00261 00262 /* --neighborhoodRange */ 00263 p = cpl_parameter_new_value("kmos.kmos_illumination.neighborhoodRange", 00264 CPL_TYPE_DOUBLE, "Range (pixels) to search for neighbors", 00265 "kmos.kmos_illumination", 1.001); 00266 cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange"); 00267 cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); 00268 cpl_parameterlist_append(recipe->parameters, p); 00269 00270 /* --range */ 00271 p = cpl_parameter_new_value("kmos.kmos_illumination.range", 00272 CPL_TYPE_STRING, 00273 "The spectral ranges to combine when collapsing the reconstructed cubes. e.g. " "\"x1_start,x1_end;x2_start,x2_end\" (microns)", 00274 "kmos.kmos_illumination", ""); 00275 cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "range"); 00276 cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); 00277 cpl_parameterlist_append(recipe->parameters, p); 00278 00279 /* --flux */ 00280 p = cpl_parameter_new_value("kmos.kmos_illumination.flux", 00281 CPL_TYPE_BOOL, "TRUE: Apply flux conservation. FALSE: otherwise", 00282 "kmos.kmos_illumination", FALSE); 00283 cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux"); 00284 cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); 00285 cpl_parameterlist_append(recipe->parameters, p); 00286 00287 /* --add-all */ 00288 p = cpl_parameter_new_value("kmos.kmos_illumination.add-all", CPL_TYPE_BOOL, 00289 "FALSE: omit 1st FLAT_SKY frame (acquisition), " 00290 "TRUE: don't perform any checks, add them all", 00291 "kmos.kmos_illumination", FALSE); 00292 cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "add-all"); 00293 cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); 00294 cpl_parameterlist_append(recipe->parameters, p); 00295 00296 /* --pix_scale */ 00297 p = cpl_parameter_new_value("kmos.kmos_illumination.pix_scale", 00298 CPL_TYPE_DOUBLE, 00299 "Change the pixel scale [arcsec]. " 00300 "Default of 0.2\" results into cubes of 14x14pix, " 00301 "a scale of 0.1\" results into cubes of 28x28pix, etc.", 00302 "kmos.kmos_illumination", KMOS_PIX_RESOLUTION); 00303 cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pix_scale"); 00304 cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); 00305 cpl_parameterlist_append(recipe->parameters, p); 00306 00307 /* --suppress_extension */ 00308 p = cpl_parameter_new_value("kmos.kmos_illumination.suppress_extension", 00309 CPL_TYPE_BOOL, 00310 "Suppress filename extension. (TRUE (apply) or FALSE (don't apply)", 00311 "kmos.kmos_illumination", FALSE); 00312 cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "suppress_extension"); 00313 cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); 00314 cpl_parameterlist_append(recipe->parameters, p); 00315 00316 /* Add parameters for band-definition */ 00317 kmos_band_pars_create(recipe->parameters, "kmos.kmos_illumination"); 00318 00319 /* Add parameters for combining */ 00320 kmos_combine_pars_create(recipe->parameters, "kmos.kmos_illumination", 00321 DEF_REJ_METHOD, FALSE); 00322 00323 /* --detector */ 00324 p = cpl_parameter_new_value("kmos.kmos_illumination.detector", 00325 CPL_TYPE_INT, "Only reduce the specified detector", 00326 "kmos.kmos_illumination", 0); 00327 cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "det"); 00328 cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); 00329 cpl_parameterlist_append(recipe->parameters, p); 00330 00331 return 0 ; 00332 } 00333 00334 /*----------------------------------------------------------------------------*/ 00340 /*----------------------------------------------------------------------------*/ 00341 static int kmos_illumination_exec(cpl_plugin *plugin) 00342 { 00343 cpl_recipe *recipe; 00344 00345 /* Get the recipe out of the plugin */ 00346 if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 00347 recipe = (cpl_recipe *)plugin; 00348 else return -1; 00349 00350 return kmos_illumination(recipe->parameters, recipe->frames); 00351 } 00352 00353 /*----------------------------------------------------------------------------*/ 00359 /*----------------------------------------------------------------------------*/ 00360 static int kmos_illumination_destroy(cpl_plugin *plugin) 00361 { 00362 cpl_recipe *recipe; 00363 00364 /* Get the recipe out of the plugin */ 00365 if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 00366 recipe = (cpl_recipe *)plugin; 00367 else return -1 ; 00368 00369 cpl_parameterlist_delete(recipe->parameters); 00370 return 0 ; 00371 } 00372 00373 /*----------------------------------------------------------------------------*/ 00386 /*----------------------------------------------------------------------------*/ 00387 static int kmos_illumination( 00388 cpl_parameterlist * parlist, 00389 cpl_frameset * frameset) 00390 { 00391 const cpl_parameter * par ; 00392 const char * method ; 00393 const char * cmethod ; 00394 const char * ranges_txt ; 00395 int next, nx, ny, flux, add_all_sky, cmax, cmin, citer, 00396 suppress_extension, reduce_det, dark_has_noise, 00397 flat_has_noise, 00398 process_noise, cnt, has_flat_edge ; 00399 double neighborhoodRange, pix_scale, cpos_rej, cneg_rej, 00400 rotangle, rotangle_found ; 00401 char * suffix ; 00402 char * fn_lut ; 00403 char * fn_suffix ; 00404 char * extname ; 00405 cpl_array ** unused_ifus ; 00406 const int * punused_ifus ; 00407 cpl_frame * frame ; 00408 cpl_frameset * frameset_sky ; 00409 char * filter ; 00410 char * keyword ; 00411 cpl_vector * ranges ; 00412 gridDefinition gd; 00413 cpl_propertylist * main_header ; 00414 cpl_propertylist * tmp_header ; 00415 int * bounds ; 00416 cpl_array * calTimestamp ; 00417 cpl_imagelist ** stored_data_cubes ; 00418 cpl_imagelist ** stored_noise_cubes ; 00419 cpl_image ** stored_data_images ; 00420 cpl_image ** stored_noise_images ; 00421 cpl_propertylist ** stored_sub_data_headers ; 00422 cpl_propertylist ** stored_sub_noise_headers ; 00423 cpl_table *** edge_table_sky ; 00424 cpl_vector * calAngles ; 00425 cpl_imagelist * detector_in ; 00426 cpl_image * img_in ; 00427 cpl_image * combined_data ; 00428 cpl_image * combined_noise ; 00429 cpl_image * xcal ; 00430 cpl_image * ycal ; 00431 cpl_image * lcal ; 00432 cpl_image * bad_pix_mask ; 00433 float * pbad_pix_mask ; 00434 cpl_image * img_dark ; 00435 cpl_image * img_dark_noise ; 00436 cpl_image * img_flat ; 00437 cpl_image * img_flat_noise ; 00438 cpl_table * band_table ; 00439 cpl_propertylist * sub_header ; 00440 cpl_imagelist * cube_data ; 00441 cpl_imagelist * cube_noise ; 00442 cpl_image * data_ifu ; 00443 cpl_image * noise_ifu ; 00444 cpl_vector * identified_slices ; 00445 double ifu_crpix, ifu_crval, ifu_cdelt, mean_data, 00446 qc_spat_unif, qc_max_dev, qc_max_nonunif, 00447 tmp_stdev, tmp_mean ; 00448 int x, y, det_nr, i, j, ifu_nr, qc_max_dev_id, 00449 qc_max_nonunif_id ; 00450 00451 /* Check entries */ 00452 if (parlist == NULL || frameset == NULL) { 00453 cpl_msg_error(__func__, "Null Inputs") ; 00454 cpl_error_set(__func__, CPL_ERROR_NULL_INPUT) ; 00455 return -1 ; 00456 } 00457 00458 /* Get parameters */ 00459 par = cpl_parameterlist_find_const(parlist, 00460 "kmos.kmos_illumination.imethod") ; 00461 method = cpl_parameter_get_string(par); 00462 par = cpl_parameterlist_find_const(parlist, 00463 "kmos.kmos_illumination.neighborhoodRange"); 00464 neighborhoodRange = cpl_parameter_get_double(par) ; 00465 par = cpl_parameterlist_find_const(parlist, "kmos.kmos_illumination.range"); 00466 ranges_txt = cpl_parameter_get_string(par) ; 00467 par = cpl_parameterlist_find_const(parlist, "kmos.kmos_illumination.flux"); 00468 flux = cpl_parameter_get_bool(par); 00469 par = cpl_parameterlist_find_const(parlist, 00470 "kmos.kmos_illumination.add-all"); 00471 add_all_sky = cpl_parameter_get_bool(par) ; 00472 par = cpl_parameterlist_find_const(parlist, 00473 "kmos.kmos_illumination.pix_scale"); 00474 pix_scale = cpl_parameter_get_double(par) ; 00475 par = cpl_parameterlist_find_const(parlist, 00476 "kmos.kmos_illumination.suppress_extension"); 00477 suppress_extension = cpl_parameter_get_bool(par) ; 00478 par = cpl_parameterlist_find_const(parlist, 00479 "kmos.kmos_illumination.detector"); 00480 reduce_det = cpl_parameter_get_int(par); 00481 00482 kmos_band_pars_load(parlist, "kmos.kmos_illumination"); 00483 kmos_combine_pars_load(parlist, "kmos.kmos_illumination", &cmethod, 00484 &cpos_rej, &cneg_rej, &citer, &cmin, &cmax, FALSE); 00485 00486 /* Check Parameters */ 00487 if (strcmp(method, "NN") && strcmp(method, "lwNN") && strcmp(method, "swNN") 00488 && strcmp(method, "MS") && strcmp(method, "CS")) { 00489 cpl_msg_error(__func__, 00490 "method must be \"NN\", \"lwNN\", \"swNN\", \"MS\" or \"CS\"") ; 00491 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_INPUT) ; 00492 return -1 ; 00493 } 00494 if (neighborhoodRange <= 0.0) { 00495 cpl_msg_error(__func__, "neighborhoodRange must be > 0") ; 00496 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_INPUT) ; 00497 return -1 ; 00498 } 00499 if (pix_scale < 0.01 || pix_scale > 0.4) { 00500 cpl_msg_error(__func__, 00501 "pix_scale must be in [0.01,0.4] -> 7x7 to 280x280 pixels") ; 00502 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_INPUT) ; 00503 return -1 ; 00504 } 00505 if (reduce_det < 0 || reduce_det > 3) { 00506 cpl_msg_error(__func__, "detector must be in [1,3]") ; 00507 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_INPUT) ; 00508 return -1 ; 00509 } 00510 if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) { 00511 cpl_msg_warning(cpl_func, "1 input FLAT -> cmethod changed to average"); 00512 cmethod = "average"; 00513 } 00514 00515 /* Identify the RAW and CALIB frames in the input frameset */ 00516 if (kmo_dfs_set_groups(frameset, "kmos_illumination") != 1) { 00517 cpl_msg_error(__func__, "Cannot identify RAW and CALIB frames") ; 00518 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_INPUT) ; 00519 return -1 ; 00520 } 00521 00522 /* Check the inputs consistency */ 00523 if (kmos_illumination_check_inputs(frameset, add_all_sky, &dark_has_noise, 00524 &flat_has_noise, &next, &nx, &ny, &rotangle) != 1) { 00525 cpl_msg_error(__func__, "Input frameset is not consistent") ; 00526 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_INPUT) ; 00527 return -1 ; 00528 } 00529 00530 /* Instrument setup */ 00531 suffix = kmo_dfs_get_suffix(kmo_dfs_get_frame(frameset, XCAL), TRUE, FALSE); 00532 cpl_msg_info(__func__, "Detected instrument setup: %s", suffix+1); 00533 00534 /* Check which IFUs are active for all frames */ 00535 unused_ifus = kmo_get_unused_ifus(frameset, 0, 0); 00536 kmo_print_unused_ifus(unused_ifus, FALSE); 00537 00538 has_flat_edge = cpl_frameset_count_tags(frameset, FLAT_EDGE); 00539 00540 /* Decide here if noise is propagated */ 00541 if (cpl_frameset_count_tags(frameset, FLAT_SKY) >= 2 && 00542 dark_has_noise && flat_has_noise) process_noise = 1 ; 00543 else process_noise = 0 ; 00544 00545 /* Load the FLAT_SKY frames in a frameset */ 00546 frameset_sky = cpl_frameset_new(); 00547 frame = kmo_dfs_get_frame(frameset, FLAT_SKY); 00548 if (add_all_sky) { 00549 cpl_msg_info(__func__, "Use all FLAT_SKY frames"); 00550 } else { 00551 /* Omit the first frame */ 00552 cpl_msg_info(__func__, "Use all FLAT_SKY frames but the first"); 00553 frame = kmo_dfs_get_frame(frameset, NULL); 00554 } 00555 while (frame != NULL) { 00556 cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)); 00557 frame = kmo_dfs_get_frame(frameset, NULL); 00558 } 00559 00560 /* Load first file primary header */ 00561 frame = kmo_dfs_get_frame(frameset_sky, FLAT_SKY); 00562 if (frame == NULL) { 00563 cpl_free(suffix) ; 00564 kmo_free_unused_ifus(unused_ifus); 00565 cpl_frameset_delete(frameset_sky) ; 00566 cpl_msg_error(__func__, "Missing FLAT_SKY in input") ; 00567 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_INPUT) ; 00568 return -1 ; 00569 } 00570 00571 main_header = kmo_dfs_load_primary_header(frameset_sky, FLAT_SKY); 00572 00573 /* Set default band-specific ranges for collapsing */ 00574 if (!strcmp(ranges_txt, "")) { 00575 keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX,1,IFU_GRATID_POSTFIX); 00576 filter = cpl_sprintf("%s", 00577 cpl_propertylist_get_string(main_header,keyword)); 00578 cpl_free(keyword); 00579 if (strcmp(filter, "IZ") == 0) ranges_txt = "0.81,1.05"; 00580 else if (strcmp(filter, "YJ") == 0) ranges_txt = "1.025,1.3"; 00581 else if (strcmp(filter, "H") == 0) ranges_txt = "1.5,1.7"; 00582 else if (strcmp(filter, "K") == 0) ranges_txt = "2.1,2.35"; 00583 else if (strcmp(filter, "HK") == 0) ranges_txt = "1.5,1.7;2.1,2.35"; 00584 else { 00585 cpl_msg_error(__func__, "Filter %s not supported", filter) ; 00586 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_INPUT) ; 00587 cpl_free(suffix) ; 00588 kmo_free_unused_ifus(unused_ifus); 00589 cpl_frameset_delete(frameset_sky) ; 00590 cpl_propertylist_delete(main_header); 00591 cpl_free(filter) ; 00592 return -1 ; 00593 } 00594 cpl_free(filter) ; 00595 } 00596 cpl_msg_info(__func__, "Spectral range to collapse: %s um", ranges_txt); 00597 00598 /* Set grid definition, wl start/end points will be set in the loop */ 00599 kmclipm_setup_grid(&gd, method, neighborhoodRange, pix_scale, 0.); 00600 00601 // create filename for LUT 00602 fn_lut = cpl_sprintf("%s%s", "lut", suffix); 00603 00604 // extract bounds 00605 tmp_header = kmo_dfs_load_primary_header(frameset, XCAL); 00606 bounds = kmclipm_extract_bounds(tmp_header); 00607 cpl_propertylist_delete(tmp_header); 00608 00609 /* Get timestamps of xcal, ycal & lcal */ 00610 calTimestamp = kmo_get_timestamps( 00611 kmo_dfs_get_frame(frameset, XCAL), 00612 kmo_dfs_get_frame(frameset, YCAL), 00613 kmo_dfs_get_frame(frameset, LCAL)) ; 00614 00615 /* Create holders for reconstructed data, noise cubes and headers */ 00616 stored_data_cubes=(cpl_imagelist**)cpl_calloc(next*KMOS_IFUS_PER_DETECTOR, 00617 sizeof(cpl_imagelist*)); 00618 stored_noise_cubes=(cpl_imagelist**)cpl_calloc(next*KMOS_IFUS_PER_DETECTOR, 00619 sizeof(cpl_imagelist*)); 00620 stored_data_images = (cpl_image**)cpl_calloc(next*KMOS_IFUS_PER_DETECTOR, 00621 sizeof(cpl_image*)); 00622 stored_noise_images = (cpl_image**)cpl_calloc(next*KMOS_IFUS_PER_DETECTOR, 00623 sizeof(cpl_image*)); 00624 stored_sub_data_headers = (cpl_propertylist**)cpl_calloc( 00625 next*KMOS_IFUS_PER_DETECTOR, sizeof(cpl_propertylist*)); 00626 stored_sub_noise_headers = (cpl_propertylist**)cpl_calloc( 00627 next*KMOS_IFUS_PER_DETECTOR, sizeof(cpl_propertylist*)); 00628 if (has_flat_edge) edge_table_sky = (cpl_table***)cpl_calloc( 00629 KMOS_NR_DETECTORS, sizeof(cpl_table**)); 00630 calAngles = cpl_vector_new(3); 00631 00632 /* Loop through all detectors */ 00633 for (det_nr = 1; det_nr <= next; det_nr++) { 00634 00635 /* Compute only one detector */ 00636 if (reduce_det != 0 && det_nr != reduce_det) continue ; 00637 00638 cpl_msg_info(__func__, "Processing detector No. %d", det_nr); 00639 cpl_msg_indent_more() ; 00640 00641 detector_in = cpl_imagelist_new(); 00642 00643 /* Load all images of this detector */ 00644 img_in = kmo_dfs_load_image(frameset_sky, FLAT_SKY, det_nr, FALSE, 00645 TRUE,NULL); 00646 cnt = 0; 00647 while (img_in != NULL) { 00648 cpl_imagelist_set(detector_in, img_in, cnt); 00649 00650 /* load same extension of next FLAT_SKY frame*/ 00651 img_in = kmo_dfs_load_image(frameset_sky, NULL, det_nr, FALSE, 00652 TRUE, NULL); 00653 cnt++; 00654 } 00655 00656 /* Combine images (data only) and create noise (stdev of data) */ 00657 cpl_msg_info(__func__, "Combining frames"); 00658 if (process_noise) { 00659 kmos_combine_frames(detector_in, cmethod, cpos_rej, cneg_rej, 00660 citer, cmax, cmin, &combined_data, &combined_noise, -1.0); 00661 } else { 00662 kmos_combine_frames(detector_in, cmethod, cpos_rej, cneg_rej, 00663 citer, cmax, cmin, &combined_data, NULL, -1.0); 00664 } 00665 cpl_imagelist_delete(detector_in); 00666 00667 /* Check if combination succesfull */ 00668 if (cpl_error_get_code() != CPL_ERROR_NONE) { 00669 cpl_free(suffix) ; 00670 kmo_free_unused_ifus(unused_ifus); 00671 cpl_frameset_delete(frameset_sky) ; 00672 cpl_propertylist_delete(main_header); 00673 cpl_free(fn_lut) ; 00674 cpl_free(bounds) ; 00675 cpl_array_delete(calTimestamp); 00676 cpl_free(stored_data_cubes); 00677 cpl_free(stored_noise_cubes); 00678 cpl_free(stored_data_images); 00679 cpl_free(stored_noise_images); 00680 cpl_free(stored_sub_data_headers); 00681 cpl_free(stored_sub_noise_headers); 00682 if (has_flat_edge) cpl_free(edge_table_sky) ; 00683 cpl_vector_delete(calAngles) ; 00684 cpl_msg_error(__func__, "Combination failed") ; 00685 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_INPUT) ; 00686 cpl_msg_indent_less() ; 00687 return -1 ; 00688 } 00689 00690 /* 00691 cpl_image_save(combined_data, "combined_image.fits", 00692 CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE) ; 00693 */ 00694 00695 if (kmclipm_omit_warning_one_slice > 10) 00696 kmclipm_omit_warning_one_slice = FALSE; 00697 00698 /* Load calibration files */ 00699 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle, 00700 FALSE, NULL, &rotangle_found, -1, 0, 0); 00701 cpl_vector_set(calAngles, 0, rotangle_found); 00702 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle, 00703 FALSE, NULL, &rotangle_found, -1, 0, 0); 00704 cpl_vector_set(calAngles, 1, rotangle_found); 00705 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle, 00706 FALSE, NULL, &rotangle_found, -1, 0, 0); 00707 cpl_vector_set(calAngles, 2, rotangle_found); 00708 00709 /* Load bad pixel mask from XCAL and set NaNs to 0 other values to 1 */ 00710 bad_pix_mask = cpl_image_duplicate(xcal); 00711 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask); 00712 for (x = 0; x < nx; x++) { 00713 for (y = 0; y < ny; y++) { 00714 if (isnan(pbad_pix_mask[x+nx*y])) { 00715 pbad_pix_mask[x+nx*y] = 0.; 00716 } else { 00717 pbad_pix_mask[x+nx*y] = 1.; 00718 } 00719 } 00720 } 00721 00722 /* Compute SKYFLAT_EDGE */ 00723 if (has_flat_edge) { 00724 edge_table_sky[det_nr-1] = kmos_illumination_edge_shift_correct( 00725 combined_data, combined_noise, process_noise, bad_pix_mask, 00726 det_nr, unused_ifus[det_nr-1], 00727 cpl_frame_get_filename(kmo_dfs_get_frame(frameset, 00728 FLAT_EDGE)), rotangle); 00729 if (edge_table_sky[det_nr-1] == NULL) { 00730 cpl_free(suffix) ; 00731 kmo_free_unused_ifus(unused_ifus); 00732 cpl_frameset_delete(frameset_sky) ; 00733 cpl_propertylist_delete(main_header); 00734 cpl_free(fn_lut) ; 00735 cpl_free(bounds) ; 00736 cpl_array_delete(calTimestamp); 00737 cpl_free(stored_data_cubes); 00738 cpl_free(stored_noise_cubes); 00739 cpl_free(stored_data_images); 00740 cpl_free(stored_noise_images); 00741 cpl_free(stored_sub_data_headers); 00742 cpl_free(stored_sub_noise_headers); 00743 if (has_flat_edge) cpl_free(edge_table_sky) ; 00744 cpl_image_delete(bad_pix_mask); 00745 cpl_image_delete(combined_data); 00746 if (process_noise) { 00747 cpl_image_delete(combined_noise); 00748 } 00749 cpl_image_delete(xcal); 00750 cpl_image_delete(ycal); 00751 cpl_image_delete(lcal); 00752 cpl_vector_delete(calAngles) ; 00753 cpl_msg_error(__func__, "Edge Shift Correction failed") ; 00754 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_INPUT) ; 00755 cpl_msg_indent_less() ; 00756 return -1 ; 00757 } 00758 } 00759 00760 /* Reconstruct */ 00761 /* Load MASTER_DARK and MASTER_FLAT */ 00762 img_dark = kmo_dfs_load_image(frameset, MASTER_DARK, det_nr, FALSE, 00763 FALSE, NULL); 00764 if (process_noise) { 00765 img_dark_noise = kmo_dfs_load_image(frameset, MASTER_DARK, det_nr, 00766 TRUE, FALSE, NULL); 00767 } 00768 img_flat = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, FALSE, 00769 rotangle, FALSE, NULL, &rotangle_found, -1, 0, 0); 00770 if (process_noise) { 00771 img_flat_noise = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, 00772 det_nr, TRUE, rotangle, FALSE, NULL, &rotangle_found, -1, 00773 0, 0); 00774 } 00775 00776 /* ESO INS FILTi ID */ 00777 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr, 00778 IFU_FILTID_POSTFIX); 00779 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0); 00780 kmclipm_setup_grid_band_lcal(&gd, 00781 cpl_propertylist_get_string(main_header, keyword), band_table); 00782 cpl_free(keyword); 00783 cpl_table_delete(band_table); 00784 00785 cpl_msg_info(__func__, "Reconstructing cubes"); 00786 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) { 00787 // update sub-header 00788 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1; 00789 00790 // load raw image and sub-header 00791 sub_header = kmo_dfs_load_sub_header(frameset_sky, FLAT_SKY, 00792 det_nr, FALSE); 00793 00794 punused_ifus = cpl_array_get_data_int_const(unused_ifus[det_nr-1]); 00795 00796 /* Check if IFU is valid */ 00797 keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr, 00798 IFU_VALID_POSTFIX); 00799 cpl_propertylist_get_string(main_header, keyword); 00800 cpl_free(keyword); 00801 00802 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) && 00803 (bounds[2*(ifu_nr-1)] != -1) && 00804 (bounds[2*(ifu_nr-1)+1] != -1) && (punused_ifus[j] == 0)) { 00805 // IFU is valid 00806 cpl_error_reset(); 00807 00808 // calculate WCS 00809 kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd); 00810 00811 // reconstruct data 00812 kmo_reconstruct_sci_image(ifu_nr, bounds[2*(ifu_nr-1)], 00813 bounds[2*(ifu_nr-1)+1], combined_data, combined_noise, 00814 img_dark, img_dark_noise, img_flat, img_flat_noise, 00815 xcal, ycal, lcal, &gd, calTimestamp, calAngles, 00816 fn_lut, &cube_data, &cube_noise, flux, 0, 00817 NULL, NULL, NULL); 00818 } else { 00819 // IFU is invalid 00820 cpl_error_reset(); 00821 } 00822 00823 /* Save output */ 00824 extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA); 00825 00826 kmclipm_update_property_string(sub_header, EXTNAME, extname, 00827 "FITS extension name"); 00828 cpl_free(extname); 00829 00830 // store cube and sub header into array for later 00831 stored_data_cubes[ifu_nr - 1] = cube_data; 00832 stored_sub_data_headers[ifu_nr - 1] = sub_header; 00833 00834 if (process_noise) { 00835 sub_header=cpl_propertylist_duplicate( 00836 stored_sub_data_headers[ifu_nr - 1]); 00837 extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_NOISE); 00838 kmclipm_update_property_string(sub_header, EXTNAME, extname, 00839 "FITS extension name"); 00840 cpl_free(extname); 00841 00842 stored_noise_cubes[ifu_nr - 1] = cube_noise; 00843 stored_sub_noise_headers[ifu_nr - 1] = sub_header; 00844 } 00845 cpl_image_delete(data_ifu); data_ifu = NULL; 00846 cpl_image_delete(noise_ifu); noise_ifu = NULL; 00847 cube_data = NULL; 00848 cube_noise = NULL; 00849 } 00850 00851 /* Free memory */ 00852 cpl_image_delete(combined_data); 00853 cpl_image_delete(xcal); 00854 cpl_image_delete(ycal); 00855 cpl_image_delete(lcal); 00856 cpl_image_delete(img_dark); 00857 cpl_image_delete(img_flat); 00858 cpl_image_delete(bad_pix_mask); 00859 if (process_noise) { 00860 cpl_image_delete(combined_noise); 00861 cpl_image_delete(img_dark_noise); 00862 cpl_image_delete(img_flat_noise); 00863 } 00864 cpl_msg_indent_less() ; 00865 } 00866 cpl_vector_delete(calAngles) ; 00867 cpl_free(fn_lut) ; 00868 cpl_free(bounds); 00869 cpl_array_delete(calTimestamp); 00870 00871 /* 00872 cpl_frameset_delete(frameset_sky) ; 00873 cpl_free(suffix) ; 00874 kmo_free_unused_ifus(unused_ifus); 00875 cpl_propertylist_delete(main_header); 00876 cpl_free(stored_data_cubes); 00877 cpl_free(stored_noise_cubes); 00878 cpl_free(stored_data_images); 00879 cpl_free(stored_noise_images); 00880 cpl_free(stored_sub_data_headers); 00881 cpl_free(stored_sub_noise_headers); 00882 if (has_flat_edge) cpl_free(edge_table_sky) ; 00883 */ 00884 00885 ranges = kmo_identify_ranges(ranges_txt); 00886 00887 /* Collapse cubes using rejection */ 00888 cpl_msg_info(__func__, "Collapse cubes"); 00889 for (det_nr = 1; det_nr <= next; det_nr++) { 00890 00891 /* Compute only one detector */ 00892 if (reduce_det != 0 && det_nr != reduce_det) continue ; 00893 00894 cpl_msg_info(__func__, "Processing detector No. %d", det_nr); 00895 cpl_msg_indent_more() ; 00896 00897 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) { 00898 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1; 00899 00900 punused_ifus=cpl_array_get_data_int_const( 00901 unused_ifus[det_nr-1]); 00902 if (punused_ifus[j] == 0) { 00903 if (stored_sub_data_headers[ifu_nr-1] != NULL) { 00904 // IFU is valid 00905 ifu_crpix = cpl_propertylist_get_double( 00906 stored_sub_data_headers[ifu_nr-1], CRPIX3); 00907 ifu_crval = cpl_propertylist_get_double( 00908 stored_sub_data_headers[ifu_nr-1], CRVAL3); 00909 ifu_cdelt = cpl_propertylist_get_double( 00910 stored_sub_data_headers[ifu_nr-1], CDELT3); 00911 identified_slices = kmo_identify_slices(ranges, ifu_crpix, 00912 ifu_crval, ifu_cdelt, gd.l.dim); 00913 } 00914 00915 if (stored_data_cubes[ifu_nr-1] != NULL) { 00916 kmclipm_make_image(stored_data_cubes[ifu_nr-1], 00917 stored_noise_cubes[ifu_nr-1], 00918 &stored_data_images[ifu_nr-1], 00919 &stored_noise_images[ifu_nr-1], identified_slices, 00920 cmethod, cpos_rej, cneg_rej, citer, cmax, cmin); 00921 } 00922 cpl_vector_delete(identified_slices); 00923 } else { 00924 // IFU is invalid 00925 } 00926 } 00927 cpl_msg_indent_less() ; 00928 } 00929 cpl_vector_delete(ranges); 00930 for (i = 0; i < next * KMOS_IFUS_PER_DETECTOR; i++) { 00931 if (stored_data_cubes != NULL) { 00932 cpl_imagelist_delete(stored_data_cubes[i]); 00933 } 00934 if (stored_noise_cubes != NULL) { 00935 cpl_imagelist_delete(stored_noise_cubes[i]); 00936 } 00937 } 00938 cpl_free(stored_data_cubes); 00939 cpl_free(stored_noise_cubes); 00940 00941 // normalise all IFUs of a detector as a group. 00942 // Calculate mean of each IFU, add up and divide by number of successful 00943 // averaged IFUs. 00944 // Then divide all valid IFUs with mean value 00945 for (j = 0; j < next; j++) { 00946 00947 /* Compute only one detector */ 00948 if (reduce_det != 0 && j+1 != reduce_det) continue ; 00949 00950 cnt = 0; 00951 mean_data = 0; 00952 for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) { 00953 ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i; 00954 if (stored_data_images[ifu_nr] != NULL) { 00955 if (cpl_image_count_rejected(stored_data_images[ifu_nr]) >= 00956 cpl_image_get_size_x(stored_data_images[ifu_nr])* 00957 cpl_image_get_size_y(stored_data_images[ifu_nr])) { 00958 /* TODO - Deallocate */ 00959 cpl_msg_error(__func__, 00960 "The collapsed, dark-subtracted image contains " 00961 "only invalid values! Probably the provided " 00962 "FLAT_SKY frames are exactly the same as the " 00963 "frames used for MASTER_DARK calculation."); 00964 cpl_error_set(__func__, CPL_ERROR_ILLEGAL_INPUT) ; 00965 return -1 ; 00966 } 00967 mean_data += cpl_image_get_mean(stored_data_images[ifu_nr]); 00968 cnt++; 00969 } 00970 } 00971 mean_data /= cnt; 00972 00973 if (mean_data != 0.0) { 00974 for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) { 00975 ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i; 00976 if (stored_data_images[ifu_nr] != NULL) { 00977 cpl_image_divide_scalar(stored_data_images[ifu_nr], 00978 mean_data); 00979 } 00980 } 00981 } else { 00982 cpl_msg_warning(__func__, 00983 "Data cannot be normalised (mean=0.0)"); 00984 } 00985 00986 if (process_noise) { 00987 if (mean_data != 0.0) { 00988 for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) { 00989 ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i; 00990 if (stored_noise_images[ifu_nr] != NULL) { 00991 cpl_image_divide_scalar(stored_noise_images[ifu_nr], 00992 mean_data); 00993 } 00994 } 00995 } else { 00996 cpl_msg_warning(__func__, 00997 "Noise cannot be normalised (mean=0.0)"); 00998 } 00999 } 01000 } 01001 01002 /* Compute qc parameters on normalised data */ 01003 qc_spat_unif = 0.0; 01004 qc_max_nonunif_id = 0 ; 01005 qc_max_dev_id = 0 ; 01006 cnt = 0; 01007 for (i = 0; i < next * KMOS_IFUS_PER_DETECTOR; i++) { 01008 if (stored_data_images[i] != NULL) { 01009 tmp_mean = cpl_image_get_mean(stored_data_images[i]); 01010 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]); 01011 01012 qc_spat_unif += pow(tmp_mean-1, 2); 01013 if (fabs(tmp_mean) > qc_max_dev) { 01014 qc_max_dev = tmp_mean-1; 01015 qc_max_dev_id = i+1; 01016 } 01017 if (fabs(tmp_stdev) > qc_max_nonunif) { 01018 qc_max_nonunif = tmp_stdev; 01019 qc_max_nonunif_id = i+1; 01020 } 01021 cnt++; 01022 } 01023 } 01024 qc_spat_unif = sqrt(qc_spat_unif / cnt); 01025 01026 /* Udate which IFUs are not used */ 01027 kmo_print_unused_ifus(unused_ifus, TRUE); 01028 kmo_set_unused_ifus(unused_ifus, main_header, "kmos_illumination"); 01029 kmo_free_unused_ifus(unused_ifus); 01030 01031 cpl_msg_info(__func__, "Save data"); 01032 kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif, 01033 "[adu] uniformity of illumination correction"); 01034 kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev, 01035 "[adu] max. deviation from unity"); 01036 kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id, 01037 "[] IFU ID with max. dev. from unity"); 01038 kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, 01039 qc_max_nonunif, "[adu] max. stdev of illumination corr."); 01040 kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, 01041 qc_max_nonunif_id, "[] IFU ID with max. stdev in illum. corr."); 01042 01043 if (!suppress_extension) fn_suffix = cpl_sprintf("%s", suffix); 01044 else fn_suffix = cpl_sprintf("%s", ""); 01045 cpl_free(suffix) ; 01046 01047 kmo_dfs_save_main_header(frameset, ILLUM_CORR, fn_suffix, frame, 01048 main_header, parlist, cpl_func); 01049 if (has_flat_edge) { 01050 kmo_dfs_save_main_header(frameset, SKYFLAT_EDGE, fn_suffix, frame, 01051 main_header, parlist, cpl_func); 01052 } 01053 cpl_propertylist_delete(main_header); 01054 01055 for (i = 0; i < next * KMOS_IFUS_PER_DETECTOR; i++) { 01056 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR, fn_suffix, 01057 stored_sub_data_headers[i], 0./0.); 01058 if (process_noise) { 01059 kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR, fn_suffix, 01060 stored_sub_noise_headers[i], 0./0.); 01061 } 01062 } 01063 for (i = 0; i < next * KMOS_IFUS_PER_DETECTOR; i++) { 01064 if (stored_data_images != NULL) { 01065 cpl_image_delete(stored_data_images[i]); 01066 } 01067 if (stored_noise_images != NULL) { 01068 cpl_image_delete(stored_noise_images[i]); 01069 } 01070 } 01071 cpl_free(stored_data_images); 01072 cpl_free(stored_noise_images); 01073 01074 for (det_nr = 1; det_nr <= next; det_nr++) { 01075 01076 /* Compute only one detector */ 01077 if (reduce_det != 0 && det_nr != reduce_det) continue ; 01078 01079 for (ifu_nr = 0; ifu_nr < KMOS_IFUS_PER_DETECTOR; ifu_nr++) { 01080 kmclipm_update_property_int( 01081 stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr], 01082 CAL_IFU_NR, ifu_nr+1+(det_nr-1)*KMOS_IFUS_PER_DETECTOR, 01083 "IFU Number {1..24}"); 01084 kmclipm_update_property_double( 01085 stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr], 01086 CAL_ROTANGLE, rotangle_found, 01087 "[deg] Rotator relative to nasmyth"); 01088 if (has_flat_edge) { 01089 // save edge-parameters as product 01090 kmo_dfs_save_table(edge_table_sky[det_nr-1][ifu_nr], 01091 SKYFLAT_EDGE, fn_suffix, 01092 stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr]); 01093 } 01094 } 01095 } 01096 cpl_frameset_delete(frameset_sky) ; 01097 cpl_free(fn_suffix); 01098 01099 for (i = 0; i < next * KMOS_IFUS_PER_DETECTOR; i++) { 01100 if (stored_sub_data_headers != NULL) { 01101 cpl_propertylist_delete(stored_sub_data_headers[i]); 01102 } 01103 if (stored_sub_noise_headers != NULL) { 01104 cpl_propertylist_delete(stored_sub_noise_headers[i]); 01105 } 01106 } 01107 cpl_free(stored_sub_data_headers); 01108 cpl_free(stored_sub_noise_headers); 01109 if (has_flat_edge) { 01110 for (i = 0; i < KMOS_NR_DETECTORS; i++) { 01111 if (edge_table_sky[i] != NULL) { 01112 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) { 01113 cpl_table_delete(edge_table_sky[i][j]); 01114 } 01115 cpl_free(edge_table_sky[i]); 01116 } 01117 } 01118 cpl_free(edge_table_sky); 01119 } 01120 return 0 ; 01121 } 01122 01125 /*----------------------------------------------------------------------------*/ 01138 /*----------------------------------------------------------------------------*/ 01139 static int kmos_illumination_check_inputs( 01140 cpl_frameset * frameset, 01141 int add_all_sky, 01142 int * dark_has_noise, 01143 int * flat_has_noise, 01144 int * next, 01145 int * nx, 01146 int * ny, 01147 double * rotangle) 01148 { 01149 cpl_frame * frame ; 01150 cpl_propertylist * main_header ; 01151 cpl_propertylist * tmp_header ; 01152 cpl_propertylist * eh ; 01153 char * keyword ; 01154 const char * filter_id ; 01155 const char * filter_id_l ; 01156 double exptime, exptime_cur, rotangle_loc, tmp_rotangle ; 01157 cpl_error_code err ; 01158 int naxis1, naxis2, naxis1_cur, naxis2_cur, 01159 n_ext, n_ext_cur, i ; 01160 01161 /* Check Entries */ 01162 if (nx == NULL || ny == NULL || frameset == NULL || dark_has_noise == NULL 01163 || flat_has_noise == NULL || next == NULL || rotangle == NULL) 01164 return -1; 01165 01166 /* Check Exptime consistency */ 01167 frame = kmo_dfs_get_frame(frameset, FLAT_SKY); 01168 /* Skip first file */ 01169 if (!add_all_sky) frame = kmo_dfs_get_frame(frameset, NULL); 01170 01171 /* Get first exptime */ 01172 main_header = cpl_propertylist_load(cpl_frame_get_filename(frame), 0); 01173 exptime = kmos_pfits_get_exptime(main_header); 01174 cpl_propertylist_delete(main_header); 01175 01176 /* Get second frame and the next */ 01177 frame = kmo_dfs_get_frame(frameset, NULL); 01178 while (frame != NULL) { 01179 /* Get exptime */ 01180 main_header = cpl_propertylist_load(cpl_frame_get_filename(frame), 0); 01181 exptime_cur = kmos_pfits_get_exptime(main_header); 01182 cpl_propertylist_delete(main_header); 01183 01184 if (fabs(exptime-exptime_cur) > 0.01) { 01185 cpl_msg_warning(__func__, "EXPTIME is not consistent") ; 01186 return 0 ; 01187 } 01188 frame = kmo_dfs_get_frame(frameset, NULL); 01189 } 01190 01191 /* Check frames numbers */ 01192 if (cpl_frameset_count_tags(frameset, FLAT_SKY) < 3) { 01193 cpl_msg_warning(cpl_func, "3 or more FLAT_SKY frames is wished"); 01194 } 01195 if (cpl_frameset_count_tags(frameset, MASTER_DARK) != 1) { 01196 cpl_msg_warning(__func__, "Need 1 MASTER_DARK") ; 01197 return 0 ; 01198 } 01199 if (cpl_frameset_count_tags(frameset, MASTER_FLAT) != 1) { 01200 cpl_msg_warning(__func__, "Need 1 MASTER_FLAT") ; 01201 return 0 ; 01202 } 01203 if (cpl_frameset_count_tags(frameset, XCAL) != 1) { 01204 cpl_msg_warning(__func__, "Need 1 XCAL") ; 01205 return 0 ; 01206 } 01207 if (cpl_frameset_count_tags(frameset, YCAL) != 1) { 01208 cpl_msg_warning(__func__, "Need 1 YCAL") ; 01209 return 0 ; 01210 } 01211 if (cpl_frameset_count_tags(frameset, LCAL) != 1) { 01212 cpl_msg_warning(__func__, "Need 1 LCAL") ; 01213 return 0 ; 01214 } 01215 if (cpl_frameset_count_tags(frameset, WAVE_BAND) != 1) { 01216 cpl_msg_warning(__func__, "Need 1 WAVE_BAND") ; 01217 return 0 ; 01218 } 01219 01220 /* filter_id, grating_id and rotator offset match all detectors */ 01221 err = CPL_ERROR_NONE ; 01222 err += kmo_check_frameset_setup(frameset, FLAT_SKY, TRUE, FALSE, TRUE); 01223 err += kmo_check_frame_setup(frameset, FLAT_SKY, XCAL, TRUE, FALSE, TRUE); 01224 err += kmo_check_frame_setup(frameset, XCAL, YCAL, TRUE, FALSE, TRUE); 01225 err += kmo_check_frame_setup(frameset, XCAL, LCAL, TRUE, FALSE, TRUE); 01226 err += kmo_check_frame_setup(frameset, XCAL, MASTER_FLAT, TRUE, FALSE,TRUE); 01227 err += kmo_check_frame_setup_md5_xycal(frameset); 01228 err += kmo_check_frame_setup_md5(frameset); 01229 if (err != CPL_ERROR_NONE) { 01230 cpl_msg_warning(__func__, "Frames are inconsistent") ; 01231 return 0 ; 01232 } 01233 01234 /* Check MASTER_DARK */ 01235 frame = kmo_dfs_get_frame(frameset, MASTER_DARK); 01236 n_ext = cpl_frame_get_nextensions(frame); 01237 if (n_ext != 2*KMOS_NR_DETECTORS) { 01238 cpl_msg_warning(__func__, "MASTER_DARK must have 6 extensions") ; 01239 return 0 ; 01240 } 01241 eh = cpl_propertylist_load(cpl_frame_get_filename(frame), 1); 01242 naxis1 = kmos_pfits_get_naxis1(eh) ; 01243 naxis2 = kmos_pfits_get_naxis2(eh) ; 01244 cpl_propertylist_delete(eh) ; 01245 01246 /* Check MASTER_FLAT */ 01247 frame = kmo_dfs_get_frame(frameset, MASTER_FLAT); 01248 n_ext = cpl_frame_get_nextensions(frame); 01249 if (n_ext % (2*KMOS_NR_DETECTORS) != 0) { 01250 cpl_msg_warning(__func__, "MASTER_FLAT must have 6*n extensions") ; 01251 return 0 ; 01252 } 01253 01254 /* Check XCAL */ 01255 frame = kmo_dfs_get_frame(frameset, XCAL) ; 01256 n_ext = cpl_frame_get_nextensions(frame); 01257 if (n_ext % KMOS_NR_DETECTORS != 0) { 01258 cpl_msg_warning(__func__, "XCAL must have 3*n extensions") ; 01259 return 0 ; 01260 } 01261 eh = cpl_propertylist_load(cpl_frame_get_filename(frame), 1); 01262 naxis1_cur = kmos_pfits_get_naxis1(eh) ; 01263 naxis2_cur = kmos_pfits_get_naxis2(eh) ; 01264 cpl_propertylist_delete(eh) ; 01265 if (naxis1_cur != naxis1 || naxis2_cur != naxis2) { 01266 cpl_msg_warning(__func__, "XCAL and MASTER_DARK sizes differ") ; 01267 return 0 ; 01268 } 01269 01270 /* Check YCAL */ 01271 frame = kmo_dfs_get_frame(frameset, YCAL); 01272 n_ext_cur = cpl_frame_get_nextensions(frame); 01273 if (n_ext_cur != n_ext) { 01274 cpl_msg_warning(__func__, "XCAL and YCAL nb of extensions differ") ; 01275 return 0 ; 01276 } 01277 eh = cpl_propertylist_load(cpl_frame_get_filename(frame), 1); 01278 naxis1_cur = kmos_pfits_get_naxis1(eh) ; 01279 naxis2_cur = kmos_pfits_get_naxis2(eh) ; 01280 cpl_propertylist_delete(eh) ; 01281 if (naxis1_cur != naxis1 || naxis2_cur != naxis2) { 01282 cpl_msg_warning(__func__, "YCAL and MASTER_DARK sizes differ") ; 01283 return 0 ; 01284 } 01285 01286 /* Check LCAL */ 01287 frame = kmo_dfs_get_frame(frameset, LCAL); 01288 n_ext_cur = cpl_frame_get_nextensions(frame); 01289 if (n_ext_cur != n_ext) { 01290 cpl_msg_warning(__func__, "XCAL and LCAL nb of extensions differ") ; 01291 return 0 ; 01292 } 01293 eh = cpl_propertylist_load(cpl_frame_get_filename(frame), 1); 01294 naxis1_cur = kmos_pfits_get_naxis1(eh) ; 01295 naxis2_cur = kmos_pfits_get_naxis2(eh) ; 01296 cpl_propertylist_delete(eh) ; 01297 if (naxis1_cur != naxis1 || naxis2_cur != naxis2) { 01298 cpl_msg_warning(__func__, "LCAL and MASTER_DARK sizes differ") ; 01299 return 0 ; 01300 } 01301 01302 tmp_header = kmo_dfs_load_primary_header(frameset, LCAL); 01303 01304 /* Check FLAT_SKY */ 01305 frame = kmo_dfs_get_frame(frameset, FLAT_SKY); 01306 main_header = cpl_propertylist_load(cpl_frame_get_filename(frame), 0); 01307 rotangle_loc = kmos_pfits_get_rotangle(main_header) ; 01308 cpl_propertylist_delete(main_header); 01309 kmclipm_strip_angle(&rotangle_loc); 01310 01311 while (frame != NULL) { 01312 n_ext = cpl_frame_get_nextensions(frame); 01313 if (n_ext != KMOS_NR_DETECTORS) { 01314 cpl_msg_warning(__func__, "FLAT_SKY has wrong nb of estensions") ; 01315 return 0 ; 01316 } 01317 eh = cpl_propertylist_load(cpl_frame_get_filename(frame), 1); 01318 naxis1_cur = kmos_pfits_get_naxis1(eh) ; 01319 naxis2_cur = kmos_pfits_get_naxis2(eh) ; 01320 cpl_propertylist_delete(eh) ; 01321 if (naxis1_cur != naxis1 || naxis2_cur != naxis2) { 01322 cpl_msg_warning(__func__, "FLAT_SKY and MASTER_DARK sizes differ") ; 01323 return 0 ; 01324 } 01325 01326 /* Check Lamps */ 01327 main_header = cpl_propertylist_load(cpl_frame_get_filename(frame), 0); 01328 if (kmo_check_lamp(main_header, INS_LAMP1_ST) != FALSE || 01329 kmo_check_lamp(main_header, INS_LAMP2_ST) != FALSE || 01330 kmo_check_lamp(main_header, INS_LAMP3_ST) != FALSE || 01331 kmo_check_lamp(main_header, INS_LAMP4_ST) != FALSE) { 01332 cpl_msg_warning(__func__, "Some FLAT_SKY lamps are ON") ; 01333 cpl_propertylist_delete(main_header) ; 01334 return 0 ; 01335 } 01336 01337 /* Check filters */ 01338 for (i = 1; i <= KMOS_NR_DETECTORS; i++) { 01339 /* ESO INS FILTi ID */ 01340 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, 01341 IFU_FILTID_POSTFIX); 01342 filter_id = cpl_propertylist_get_string(main_header, keyword); 01343 filter_id_l = cpl_propertylist_get_string(tmp_header, keyword); 01344 cpl_free(keyword); 01345 01346 if (strcmp(filter_id, "IZ") && strcmp(filter_id, "YJ") && 01347 strcmp(filter_id, "H") && strcmp(filter_id, "K") && 01348 strcmp(filter_id, "HK")) { 01349 cpl_msg_warning(__func__, 01350 "Filter ID must be 'IZ', 'YJ', 'H', 'K' or 'HK' ") ; 01351 cpl_propertylist_delete(main_header) ; 01352 cpl_propertylist_delete(tmp_header) ; 01353 return 0 ; 01354 } 01355 01356 if (strcmp(filter_id, filter_id_l)) { 01357 cpl_msg_warning(__func__, 01358 "Filter IDs in FLAT_SKY and LCAL don't match") ; 01359 cpl_propertylist_delete(main_header) ; 01360 cpl_propertylist_delete(tmp_header) ; 01361 return 0 ; 01362 } 01363 01364 /* ESO INS GRATi ID */ 01365 keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, 01366 IFU_GRATID_POSTFIX); 01367 filter_id = cpl_propertylist_get_string(main_header, keyword); 01368 filter_id_l = cpl_propertylist_get_string(tmp_header, keyword); 01369 cpl_free(keyword); 01370 01371 if (strcmp(filter_id, "IZ") && strcmp(filter_id, "YJ") && 01372 strcmp(filter_id, "H") && strcmp(filter_id, "K") && 01373 strcmp(filter_id, "HK")) { 01374 cpl_msg_warning(__func__, 01375 "Grating ID must be 'IZ', 'YJ', 'H', 'K' or 'HK' ") ; 01376 cpl_propertylist_delete(main_header) ; 01377 cpl_propertylist_delete(tmp_header) ; 01378 return 0 ; 01379 } 01380 if (strcmp(filter_id, filter_id_l)) { 01381 cpl_msg_warning(__func__, 01382 "Grating IDs in FLAT_SKY and LCAL don't match") ; 01383 cpl_propertylist_delete(main_header) ; 01384 cpl_propertylist_delete(tmp_header) ; 01385 return 0 ; 01386 } 01387 01388 tmp_rotangle = kmos_pfits_get_rotangle(main_header) ; 01389 kmclipm_strip_angle(&tmp_rotangle); 01390 01391 if (fabs(rotangle_loc - tmp_rotangle) > 10.0 && 01392 fabs(rotangle_loc - tmp_rotangle) < 360.-10.) { 01393 cpl_msg_warning(__func__, 01394 "OCS ROT NAANGLE of sky flats differ too much: %f %f", 01395 rotangle_loc, tmp_rotangle); 01396 cpl_propertylist_delete(main_header) ; 01397 cpl_propertylist_delete(tmp_header) ; 01398 return 0 ; 01399 } 01400 } 01401 cpl_propertylist_delete(main_header); 01402 01403 // get next FLAT_SKY frame 01404 frame = kmo_dfs_get_frame(frameset, NULL); 01405 } 01406 cpl_propertylist_delete(tmp_header); 01407 01408 /* Return */ 01409 *nx = naxis1 ; 01410 *ny = naxis2 ; 01411 *next = n_ext ; 01412 *rotangle = rotangle_loc ; 01413 /* TODO */ 01414 *dark_has_noise = 1 ; 01415 *flat_has_noise = 1 ; 01416 return 1 ; 01417 } 01418 01419 /*----------------------------------------------------------------------------*/ 01425 /*----------------------------------------------------------------------------*/ 01426 static cpl_table ** kmos_illumination_edge_shift_correct( 01427 cpl_image * combined_data, 01428 cpl_image * combined_noise, 01429 int process_noise, 01430 const cpl_image * bad_pix_mask, 01431 int det_nr, 01432 cpl_array * unused_ifus, 01433 const char * flat_edge_filename, 01434 double rotangle) 01435 { 01436 int middle_row ; 01437 cpl_vector ** slitlet_ids = NULL ; 01438 cpl_matrix ** edgepars = NULL ; 01439 cpl_table ** edges ; 01440 cpl_vector * shift_vec ; 01441 const int * punused_ifus ; 01442 cpl_table * edge_table_flat ; 01443 cpl_vector * edge_vec ; 01444 kmclipm_vector * kv ; 01445 float * pcombined_data ; 01446 float * pcombined_noise ; 01447 double * array_in ; 01448 double * array_out ; 01449 double tmp_rotangle, flatval, skyval, shift_val ; 01450 int ifu_nr, i, nx, ny, ix, iy, edgeNr ; 01451 01452 /* Check Entries */ 01453 01454 /* Initialise */ 01455 middle_row = 1024 ; 01456 01457 /* Get edge-edgepars from FLAT_SKY */ 01458 kmos_calc_edgepars(combined_data, unused_ifus, bad_pix_mask, det_nr, 01459 &slitlet_ids, &edgepars); 01460 if (cpl_error_get_code() != CPL_ERROR_NONE) { 01461 cpl_msg_error(__func__, "Cannot compute edges parameters") ; 01462 return NULL ; 01463 } 01464 01465 /* Copy edgepars to table for saving later on */ 01466 edges = kmo_edgepars_to_table(slitlet_ids, edgepars); 01467 if (edgepars != NULL) { 01468 for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) 01469 cpl_matrix_delete(edgepars[i]); 01470 cpl_free(edgepars); 01471 } 01472 if (slitlet_ids != NULL) { 01473 for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) 01474 cpl_vector_delete(slitlet_ids[i]); 01475 cpl_free(slitlet_ids); 01476 } 01477 01478 /* Correlate FLAT_EDGE and SKYFLAT_EDGE */ 01479 shift_vec = cpl_vector_new(KMOS_IFUS_PER_DETECTOR); 01480 for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) { 01481 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + i + 1; 01482 punused_ifus = cpl_array_get_data_int_const(unused_ifus); 01483 if (punused_ifus[i] == 0) { 01484 edge_table_flat = kmclipm_cal_table_load(flat_edge_filename, 01485 ifu_nr, rotangle, 0, &tmp_rotangle); 01486 /* Shift values for each IFU by comparing edge parameters */ 01487 if (edge_table_flat != NULL) { 01488 edge_vec = cpl_vector_new(2*KMOS_SLITLET_X); 01489 for (edgeNr = 0; edgeNr < 2*KMOS_SLITLET_X; edgeNr++) { 01490 flatval = kmo_calc_fitted_slitlet_edge(edge_table_flat, 01491 edgeNr, middle_row); 01492 skyval = kmo_calc_fitted_slitlet_edge(edges[i], edgeNr, 01493 middle_row); 01494 cpl_vector_set(edge_vec, edgeNr, flatval-skyval); 01495 } 01496 cpl_table_delete(edge_table_flat); 01497 01498 /* Reject deviating edge-differences */ 01499 kv = kmclipm_vector_create(edge_vec); 01500 kmclipm_reject_deviant(kv, 3, 3, NULL, NULL); 01501 01502 /* Set shift value for each IFU */ 01503 cpl_vector_set(shift_vec, i, 01504 kmclipm_vector_get_median(kv, KMCLIPM_ARITHMETIC)); 01505 kmclipm_vector_delete(kv); 01506 } else { 01507 cpl_vector_set(shift_vec, i, 0.0) ; 01508 } 01509 } else { 01510 cpl_vector_set(shift_vec, i, 0.0) ; 01511 } 01512 } 01513 01514 /* Take median of all IFU-shift-values */ 01515 shift_val = -cpl_vector_get_median(shift_vec); 01516 cpl_vector_delete(shift_vec); 01517 01518 cpl_msg_info(__func__, "Shift detector %d by %g pixels", det_nr, shift_val); 01519 nx = cpl_image_get_size_x(combined_data), 01520 ny = cpl_image_get_size_x(combined_data), 01521 pcombined_data = cpl_image_get_data_float(combined_data) ; 01522 if (process_noise) { 01523 pcombined_noise = cpl_image_get_data_float(combined_noise); 01524 } 01525 01526 array_in = cpl_calloc(nx, sizeof(double)) ; 01527 /* Apply shift - Cubic spline */ 01528 for (iy = 0; iy < ny; iy++) { 01529 for (ix = 0; ix < nx; ix++) array_in[ix] = pcombined_data[ix+iy*nx]; 01530 array_out = cubicspline_reg_reg(nx, 0., 1., array_in, nx, shift_val, 01531 1.0, NATURAL); 01532 for (ix = 0; ix < nx; ix++) pcombined_data[ix+iy*nx] = array_out[ix]; 01533 cpl_free(array_out); 01534 01535 if (process_noise) { 01536 for (ix = 0; ix < nx; ix++) array_in[ix]=pcombined_noise[ix+iy*nx]; 01537 array_out = cubicspline_reg_reg(nx, 0., 1., array_in, nx, shift_val, 01538 1.0, NATURAL); 01539 for (ix = 0; ix < nx; ix++) pcombined_noise[ix+iy*nx]=array_out[ix]; 01540 cpl_free(array_out); 01541 } 01542 } 01543 cpl_free(array_in); 01544 return edges ; 01545 } 01546 01547