vircam_standard_process.c

00001 /* $Id: vircam_standard_process.c,v 1.40 2010/12/09 13:20:49 jim Exp $
00002  *
00003  * This file is part of the VIRCAM Pipeline
00004  * Copyright (C) 2005 Cambridge Astronomy Survey Unit
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: jim $
00023  * $Date: 2010/12/09 13:20:49 $
00024  * $Revision: 1.40 $
00025  * $Name: v1-1-0 $
00026  */
00027 
00028 /* Includes */
00029 
00030 #ifdef HAVE_CONFIG_H
00031 #include <config.h>
00032 #endif
00033 
00034 #include <stdio.h>
00035 #include <math.h>
00036 #include <string.h>
00037 #include <cpl.h>
00038 
00039 #include "vircam_utils.h"
00040 #include "vircam_mask.h"
00041 #include "vircam_pfits.h"
00042 #include "vircam_dfs.h"
00043 #include "vircam_mods.h"
00044 #include "vircam_fits.h"
00045 #include "vircam_tfits.h"
00046 #include "vircam_jmp_utils.h"
00047 #include "vircam_paf.h"
00048 
00049 /* Function prototypes */
00050 
00051 static int vircam_standard_process_create(cpl_plugin *);
00052 static int vircam_standard_process_exec(cpl_plugin *);
00053 static int vircam_standard_process_destroy(cpl_plugin *);
00054 static int vircam_standard_process(cpl_parameterlist *,cpl_frameset *);
00055 static cpl_propertylist *vircam_standard_process_dummyqc(int type);
00056 
00057 
00058 static char vircam_standard_process_description[] =
00059 "vircam_standard_process -- VIRCAM standard field processing recipe.\n\n"
00060 "Process a complete pawprint for standard fields in VIRCAM data.\n"
00061 "Remove instrumental signature, interleave microstep sequences (if done),\n"
00062 "combine jitters, photometrically and astrometrically calibrate the pawprint\n"
00063 "image. Photometric calibration can be done with 2mass and 1 other source\n\n"
00064 "The program accepts the following files in the SOF:\n\n"
00065 "    Tag                   Description\n"
00066 "    -----------------------------------------------------------------------\n"
00067 "    %-21s A list of raw science images\n"
00068 "    %-21s A master dark frame\n"
00069 "    %-21s A master twilight flat frame\n"
00070 "    %-21s A channel table\n"
00071 "    %-21s A photometric calibration table\n"
00072 "    %-21s A readnoise/gain file\n"
00073 "    %-21s A master confidence map or\n"
00074 "    %-21s A master bad pixel mask\n"
00075 "    %-21s A master 2mass catalogue index\n"
00076 "    %-21s A second reference catalogue index\n"
00077 "All of the above are required\n"
00078 "\n";
00079 
00223 /* Function code */
00224 
00225 /*---------------------------------------------------------------------------*/
00233 /*---------------------------------------------------------------------------*/
00234 
00235 int cpl_plugin_get_info(cpl_pluginlist *list) {
00236     cpl_recipe  *recipe = cpl_calloc(1,sizeof(*recipe));
00237     cpl_plugin  *plugin = &recipe->interface;
00238     char alldesc[SZ_ALLDESC];
00239     (void)snprintf(alldesc,SZ_ALLDESC,vircam_standard_process_description,
00240                    VIRCAM_STD_OBJECT_RAW,VIRCAM_CAL_DARK,
00241                    VIRCAM_CAL_TWILIGHT_FLAT,VIRCAM_CAL_CHANTAB,
00242                    VIRCAM_CAL_PHOTTAB,VIRCAM_CAL_READGAINFILE,VIRCAM_CAL_CONF,
00243                    VIRCAM_CAL_BPM,VIRCAM_CAL_2MASS,VIRCAM_CAL_REFCAT);
00244 
00245     cpl_plugin_init(plugin,
00246                     CPL_PLUGIN_API,
00247                     VIRCAM_BINARY_VERSION,
00248                     CPL_PLUGIN_TYPE_RECIPE,
00249                     "vircam_standard_process",
00250                     "VIRCAM standard field processing recipe",
00251                     alldesc,
00252                     "Jim Lewis",
00253                     "jrl@ast.cam.ac.uk",
00254                     vircam_get_license(),
00255                     vircam_standard_process_create,
00256                     vircam_standard_process_exec,
00257                     vircam_standard_process_destroy);
00258 
00259     cpl_pluginlist_append(list,plugin);
00260 
00261     return(0);
00262 }
00263 
00264 /*---------------------------------------------------------------------------*/
00273 /*---------------------------------------------------------------------------*/
00274 
00275 static int vircam_standard_process_create(cpl_plugin *plugin) {
00276     cpl_recipe      *recipe;
00277     cpl_parameter   *p;
00278 
00279     /* Get the recipe out of the plugin */
00280 
00281     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00282         recipe = (cpl_recipe *)plugin;
00283     else 
00284         return(-1);
00285 
00286     /* Create the parameters list in the cpl_recipe object */
00287 
00288     recipe->parameters = cpl_parameterlist_new();
00289 
00290     /* Fill in the minimum object size */
00291 
00292     p = cpl_parameter_new_value("vircam.vircam_standard_process.ipix",
00293                                 CPL_TYPE_INT,
00294                                 "Minimum pixel area for each detected object",
00295                                 "vircam.vircam_standard_process",5);
00296     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ipix");
00297     cpl_parameterlist_append(recipe->parameters,p);
00298 
00299     /* Fill in the detection threshold parameter */
00300 
00301     p = cpl_parameter_new_value("vircam.vircam_standard_process.thresh",
00302                                 CPL_TYPE_DOUBLE,
00303                                 "Detection threshold in sigma above sky",
00304                                 "vircam.vircam_standard_process",2.0);
00305     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"thr");
00306     cpl_parameterlist_append(recipe->parameters,p);
00307 
00308     /* Fill in flag to use deblending software or not */
00309 
00310     p = cpl_parameter_new_value("vircam.vircam_standard_process.icrowd",
00311                                 CPL_TYPE_BOOL,"Use deblending?",
00312                                 "vircam.vircam_standard_process",1);
00313     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"icrowd");
00314     cpl_parameterlist_append(recipe->parameters,p);
00315 
00316     /* Fill in core radius */
00317 
00318     p = cpl_parameter_new_value("vircam.vircam_standard_process.rcore",
00319                                 CPL_TYPE_DOUBLE,"Value of Rcore in pixels",
00320                                 "vircam.vircam_standard_process",3.0);
00321     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"rcore");
00322     cpl_parameterlist_append(recipe->parameters,p);
00323 
00324     /* Fill in background smoothing box size */
00325 
00326     p = cpl_parameter_new_value("vircam.vircam_standard_process.nbsize",
00327                                 CPL_TYPE_INT,"Background smoothing box size",
00328                                 "vircam.vircam_standard_process",64);
00329     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"nb");
00330     cpl_parameterlist_append(recipe->parameters,p);
00331 
00332     /* Fill in flag to use save the output catalogue */
00333 
00334     p = cpl_parameter_new_value("vircam.vircam_standard_process.savecat",
00335                                 CPL_TYPE_BOOL,"Save catalogue?",
00336                                 "vircam.vircam_standard_process",1);
00337     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"savecat");
00338     cpl_parameterlist_append(recipe->parameters,p);
00339 
00340 
00341     /* Fill in flag to destripe the images */
00342 
00343     p = cpl_parameter_new_value("vircam.vircam_standard_process.destripe",
00344                                 CPL_TYPE_BOOL,"Destripe images?",
00345                                 "vircam.vircam_standard_process",1);
00346     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"destripe");
00347     cpl_parameterlist_append(recipe->parameters,p);
00348 
00349     /* Fill in flag to correct sky background */
00350 
00351     p = cpl_parameter_new_value("vircam.vircam_standard_process.skycor",
00352                                 CPL_TYPE_BOOL,"Do sky correction?",
00353                                 "vircam.vircam_standard_process",1);
00354     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"skycor");
00355     cpl_parameterlist_append(recipe->parameters,p);
00356 
00357     /* Fill in flag to save simple images */
00358 
00359     p = cpl_parameter_new_value("vircam.vircam_standard_process.savesimple",
00360                                 CPL_TYPE_BOOL,"Save simple images?",
00361                                 "vircam.vircam_standard_process",0);
00362     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"savesimple");
00363     cpl_parameterlist_append(recipe->parameters,p);
00364 
00365     /* Extension number of input frames to use */
00366 
00367     p = cpl_parameter_new_range("vircam.vircam_standard_process.extenum",
00368                                 CPL_TYPE_INT,
00369                                 "Extension number to be done, 0 == all",
00370                                 "vircam.vircam_standard_process",
00371                                 1,0,16);
00372     cpl_parameter_set_alias(p,CPL_PARAMETER_MODE_CLI,"ext");
00373     cpl_parameterlist_append(recipe->parameters,p);
00374         
00375     /* Get out of here */
00376 
00377     return(0);
00378 }
00379     
00380     
00381 /*---------------------------------------------------------------------------*/
00387 /*---------------------------------------------------------------------------*/
00388 
00389 static int vircam_standard_process_exec(cpl_plugin *plugin) {
00390     cpl_recipe  *recipe;
00391 
00392     /* Get the recipe out of the plugin */
00393 
00394     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00395         recipe = (cpl_recipe *)plugin;
00396     else 
00397         return(-1);
00398 
00399     return(vircam_standard_process(recipe->parameters,recipe->frames));
00400 }
00401                                 
00402 /*---------------------------------------------------------------------------*/
00408 /*---------------------------------------------------------------------------*/
00409 
00410 static int vircam_standard_process_destroy(cpl_plugin *plugin) {
00411     cpl_recipe *recipe ;
00412 
00413     /* Get the recipe out of the plugin */
00414 
00415     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE)
00416         recipe = (cpl_recipe *)plugin;
00417     else 
00418         return(-1);
00419 
00420     cpl_parameterlist_delete(recipe->parameters);
00421     return(0);
00422 }
00423 
00424 /*---------------------------------------------------------------------------*/
00431 /*---------------------------------------------------------------------------*/
00432 
00433 static int vircam_standard_process(cpl_parameterlist *parlist, 
00434                                    cpl_frameset *framelist) {
00435     const char *fctid="vircam_standard_process";
00436     cpl_parameter *p;
00437     int nlab,jst,jfn,status,j,i,retval,nusteps,isconf,live,ndit;
00438     float readnoise,gain,gaincor_fac;
00439     vir_fits *ff;
00440     cpl_propertylist *ehu,*pp;
00441     cpl_frame *catindex;
00442 
00443     /* Check validity of input frameset */
00444 
00445     if (framelist == NULL || cpl_frameset_get_size(framelist) <= 0) {
00446         cpl_msg_error(fctid,"Input framelist NULL or has no input data\n");
00447         return(-1);
00448     }
00449 
00450     /* Check the files in the frameset */
00451 
00452     if (vircam_frameset_fexists(framelist) != VIR_OK) {
00453         cpl_msg_error(fctid,"Input frameset is missing files. Check SOF");
00454         return(-1);
00455     }
00456 
00457     /* Initialise some things */
00458 
00459     vircam_jmp_init();
00460     (void)strncpy(vircam_recipename,fctid,VIRCAM_PATHSZ);
00461     (void)snprintf(vircam_recipepaf,VIRCAM_PATHSZ,"VIRCAM/%s",fctid);
00462     recflag = RECSTD;
00463 
00464     /* Get the parameters */
00465 
00466     p = cpl_parameterlist_find(parlist,
00467                                "vircam.vircam_standard_process.ipix");
00468     vircam_jmp_config.ipix = cpl_parameter_get_int(p);
00469     p = cpl_parameterlist_find(parlist,
00470                                "vircam.vircam_standard_process.thresh");
00471     vircam_jmp_config.threshold = (float)cpl_parameter_get_double(p);
00472     p = cpl_parameterlist_find(parlist,
00473                                "vircam.vircam_standard_process.icrowd");
00474     vircam_jmp_config.icrowd = cpl_parameter_get_bool(p);
00475     p = cpl_parameterlist_find(parlist,
00476                                "vircam.vircam_standard_process.rcore");
00477     vircam_jmp_config.rcore = (float)cpl_parameter_get_double(p);
00478     p = cpl_parameterlist_find(parlist,
00479                                "vircam.vircam_standard_process.nbsize");
00480     vircam_jmp_config.nbsize = cpl_parameter_get_int(p);
00481     p = cpl_parameterlist_find(parlist,
00482                                "vircam.vircam_standard_process.savecat");
00483     vircam_jmp_config.savecat = cpl_parameter_get_bool(p);
00484     p = cpl_parameterlist_find(parlist,
00485                                "vircam.vircam_standard_process.destripe");
00486     vircam_jmp_config.destripe = cpl_parameter_get_bool(p);
00487     p = cpl_parameterlist_find(parlist,
00488                                "vircam.vircam_standard_process.skycor");
00489     vircam_jmp_config.skycor = cpl_parameter_get_bool(p);
00490     p = cpl_parameterlist_find(parlist,
00491                                "vircam.vircam_standard_process.savesimple");
00492     vircam_jmp_config.savesimple = cpl_parameter_get_bool(p);
00493     p = cpl_parameterlist_find(parlist,
00494                                "vircam.vircam_standard_process.extenum");
00495     vircam_jmp_config.extenum = cpl_parameter_get_int(p);
00496 
00497     /* Sort out raw from calib frames */
00498 
00499     if (vircam_dfs_set_groups(framelist) != VIR_OK) {
00500         cpl_msg_error(fctid,"Cannot identify RAW and CALIB frames");
00501         vircam_jmp_tidy(0);
00502         return(-1);
00503     }
00504 
00505     /* Label the input frames */
00506 
00507     if ((ps.labels = cpl_frameset_labelise(framelist,vircam_compare_tags,
00508                                            &nlab)) == NULL) {
00509         cpl_msg_error(fctid,"Cannot labelise the input frames");
00510         vircam_jmp_tidy(0);
00511         return(-1);
00512     }
00513 
00514     /* Get the input science frames */
00515 
00516     if ((ps.science_frames = 
00517          vircam_frameset_subgroup(framelist,ps.labels,nlab,
00518                                   VIRCAM_STD_OBJECT_RAW)) == NULL) {
00519         cpl_msg_error(fctid,"No science images to process!");
00520         vircam_jmp_tidy(0);
00521         return(-1);
00522     }
00523 
00524     /* Check to see if there is a master dark frame */
00525 
00526     if ((ps.master_dark = 
00527          vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00528                                     VIRCAM_CAL_DARK)) == NULL) {
00529         cpl_msg_error(fctid,"No master dark found");
00530         vircam_jmp_tidy(0);
00531         return(-1);
00532     }
00533         
00534     /* Check to see if there is a master twilight flat frame */
00535 
00536     if ((ps.master_twilight_flat = 
00537          vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00538                                     VIRCAM_CAL_TWILIGHT_FLAT)) == NULL) {
00539         cpl_msg_error(fctid,"No master twilight flat found");
00540         vircam_jmp_tidy(0);
00541         return(-1);
00542     }
00543         
00544     /* Get the gain corrections */
00545 
00546     status = VIR_OK;
00547     if (vircam_gaincor_calc(ps.master_twilight_flat,&i,&(ps.gaincors),
00548                             &status) != VIR_OK) {
00549         cpl_msg_error(fctid,"Error calculating gain corrections");
00550         vircam_jmp_tidy(0);
00551         return(-1);
00552     }
00553         
00554     /* Check to see if there is a readgain file */
00555 
00556     if ((ps.readgain_file = 
00557          vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00558                                     VIRCAM_CAL_READGAINFILE)) == NULL) {
00559         cpl_msg_error(fctid,"No master readnoise/gain file found");
00560         vircam_jmp_tidy(0);
00561         return(-1);
00562     }
00563         
00564     /* Check to see if there is a master confidence map. If there isn't
00565        then look for a bad pixel mask that can be converted into a 
00566        confidence map (in an emergency) */
00567 
00568     isconf = 1;
00569     if ((ps.master_conf = 
00570          vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00571                                     VIRCAM_CAL_CONF)) == NULL) {
00572         isconf = 0;
00573         if ((ps.master_conf = 
00574              vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00575                                         VIRCAM_CAL_BPM)) == NULL) {
00576             cpl_msg_error(fctid,"No master confidence map found");
00577             vircam_jmp_tidy(0);
00578             return(-1);
00579         }
00580     }
00581     ps.mask = vircam_mask_define(framelist,ps.labels,nlab);
00582         
00583     /* Check to see if there is a channel table */
00584 
00585     if ((ps.chantab = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00586                                                  VIRCAM_CAL_CHANTAB)) == NULL) {
00587         cpl_msg_error(fctid,"No channel table found");
00588         vircam_jmp_tidy(0);
00589         return(-1);
00590     }
00591 
00592     /* Check to see if there is a photometric table */
00593 
00594     if ((ps.phottab = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00595                                                  VIRCAM_CAL_PHOTTAB)) == NULL) {
00596         cpl_msg_error(fctid,"No photometric table found");
00597         vircam_jmp_tidy(0);
00598         return(-1);
00599     }
00600     if ((ps.tphottab = cpl_table_load(cpl_frame_get_filename(ps.phottab),1,0)) == NULL) {
00601         cpl_msg_error(fctid,"Unable to load photometric table");
00602         vircam_jmp_tidy(0);
00603         return(-1);
00604     }
00605 
00606     /* Is the 2mass index file specified? */
00607 
00608     if ((catindex = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00609                                                VIRCAM_CAL_2MASS)) == NULL) {
00610         cpl_msg_info(fctid,"No 2MASS index found -- cannot continue");
00611         vircam_jmp_tidy(0);
00612         return(-1);
00613     }
00614     
00615     /* Get catalogue parameters */
00616 
00617     if (vircam_catpars(catindex,&(ps.catpath),&(ps.catname)) == VIR_FATAL) {
00618         vircam_jmp_tidy(0);
00619         cpl_frame_delete(catindex);
00620         return(-1);
00621     }
00622     cpl_frame_delete(catindex);
00623 
00624     /* Is there a second photometric catalogue index file specified? */
00625 
00626     if ((catindex = vircam_frameset_subgroup_1(framelist,ps.labels,nlab,
00627                                                VIRCAM_CAL_REFCAT)) != NULL) {
00628     
00629         /* Get catalogue parameters */
00630 
00631         if (vircam_catpars(catindex,&(ps.catpath2),&(ps.catname2)) == VIR_FATAL) {
00632             vircam_jmp_tidy(0);
00633             cpl_frame_delete(catindex);
00634             return(-1);
00635         }
00636         cpl_frame_delete(catindex);
00637     }
00638 
00639     /* Get the number of DITs */
00640 
00641     pp = cpl_propertylist_load(cpl_frame_get_filename(cpl_frameset_get_frame(ps.science_frames,0)),0);
00642     if (vircam_pfits_get_ndit(pp,&ndit) != VIR_OK) {
00643         cpl_msg_error(fctid,"No value for NDIT available");
00644         freepropertylist(pp);
00645         vircam_jmp_tidy(0);
00646         return(-1);
00647     }
00648     cpl_propertylist_delete(pp);
00649 
00650     /* Now, how many image extensions do we want to do? If the extension
00651        number is zero, then we loop for all possible extensions. If it
00652        isn't then we just do the extension specified */
00653 
00654     vircam_exten_range(vircam_jmp_config.extenum,
00655                        (const cpl_frame *)cpl_frameset_get_frame(ps.science_frames,0),
00656                        &jst,&jfn);
00657     if (jst == -1 || jfn == -1) {
00658         cpl_msg_error(fctid,"Unable to continue");
00659         vircam_jmp_tidy(0);
00660         return(-1);
00661     }
00662 
00663     /* Now loop for all the extensions... */
00664 
00665     status = VIR_OK;
00666     for (j = jst; j <= jfn; j++) {
00667         isfirst = (j == jst);
00668         gaincor_fac = (ps.gaincors)[j-1];
00669 
00670         /* Load up the calibration frames into vir_fits/vir_tfits structures 
00671            It is a fatal error if any one of them can't load properly */
00672         
00673         ps.fdark = vircam_fits_load(ps.master_dark,CPL_TYPE_FLOAT,j);
00674         if (ps.fdark == NULL) {
00675             cpl_msg_error(fctid,"Error loading master dark %s[%d]\n%s",
00676                           cpl_frame_get_filename(ps.master_dark),j,
00677                           cpl_error_get_message());
00678             vircam_jmp_tidy(0);
00679             return(-1);
00680         }
00681         ps.fflat = vircam_fits_load(ps.master_twilight_flat,CPL_TYPE_FLOAT,j);
00682         if (ps.fflat == NULL) {
00683             cpl_msg_error(fctid,"Error loading master flat %s[%d]\n%s",
00684                           cpl_frame_get_filename(ps.master_twilight_flat),j,
00685                           cpl_error_get_message());
00686             vircam_jmp_tidy(0);
00687             return(-1);
00688         }
00689         ps.fconf = vircam_fits_load(ps.master_conf,CPL_TYPE_INT,j);
00690         if (ps.fconf == NULL) {
00691             cpl_msg_error(fctid,"Error loading master conf %s[%d]\n%s",
00692                           cpl_frame_get_filename(ps.master_conf),j,
00693                           cpl_error_get_message());
00694             vircam_jmp_tidy(0);
00695             return(-1);
00696         }
00697         if (! isconf) 
00698             vircam_jmp_bpm2conf();
00699         if (vircam_mask_load(ps.mask,j,
00700                              cpl_image_get_size_x(vircam_fits_get_image(ps.fconf)),
00701                              cpl_image_get_size_y(vircam_fits_get_image(ps.fconf))) != VIR_OK) {
00702             cpl_msg_error(fctid,"Error loading mask from master conf %s[%d]\n%s",
00703                           cpl_frame_get_filename(ps.master_conf),j,
00704                           cpl_error_get_message());
00705             vircam_jmp_tidy(0);
00706             return(-1);
00707         }
00708         ps.fchantab = vircam_tfits_load(ps.chantab,j);
00709         if (ps.fchantab == NULL) {
00710             cpl_msg_error(fctid,"Error loading channel table %s[%d]\n%s",
00711                           cpl_frame_get_filename(ps.chantab),j,
00712                           cpl_error_get_message());
00713             vircam_jmp_tidy(0);
00714             return(-1);
00715         }
00716 
00717         /* Load up the vir_fits structures for the science images */
00718 
00719         ps.nscience = cpl_frameset_get_size(ps.science_frames);
00720         ps.sci_fits = vircam_fits_load_list(ps.science_frames,CPL_TYPE_FLOAT,j);
00721         if (ps.sci_fits == NULL) {
00722             cpl_msg_error(fctid,"Error loading science frames extension %d: %s",
00723                           j,cpl_error_get_message());
00724             vircam_jmp_tidy(0);
00725             return(-1);
00726         }
00727 
00728         /* Set up something for the PAFs */
00729 
00730         ps.phupaf = vircam_paf_phu_items(vircam_fits_get_phu(ps.sci_fits[0]));
00731 
00732         /* Loop through and mark the frames where the header says the detector
00733            wasn't live */
00734 
00735         for (i = 0; i < ps.nscience; i++) {
00736             ff = ps.sci_fits[i];
00737             vircam_pfits_get_detlive(vircam_fits_get_ehu(ff),&live);
00738             if (! live) 
00739                 vircam_fits_set_error(ff,VIR_FATAL);
00740             if (vircam_check_crval(vircam_fits_get_phu(ff),
00741                                    vircam_fits_get_ehu(ff)) != VIR_OK) {
00742                 cpl_msg_error(fctid,"Unable to correct CRVAL in %s",
00743                               vircam_fits_get_fullname(ff));
00744                 vircam_jmp_tidy(0);
00745                 return(-1);
00746             }
00747         }
00748 
00749         /* Get the readnoise and gain estimate for this extension */
00750 
00751         vircam_jmp_get_readnoise_gain(j,&readnoise,&gain);
00752 
00753         /* Loop for all the science frames and do the 2d corrections */
00754 
00755         cpl_msg_info(fctid,"Doing stage1 corrections on %s",
00756                      vircam_fits_get_extname(ps.sci_fits[0]));
00757         for (i = 0; i < ps.nscience; i++) {
00758             ff = ps.sci_fits[i];
00759             cpl_propertylist_update_float(vircam_fits_get_ehu(ff),"READNOIS",
00760                                           readnoise);
00761             cpl_propertylist_set_comment(vircam_fits_get_ehu(ff),"READNOIS",
00762                                          "[e-] Readnoise used in processing");
00763             cpl_propertylist_update_float(vircam_fits_get_ehu(ff),"GAIN",gain);
00764             cpl_propertylist_set_comment(vircam_fits_get_ehu(ff),"GAIN",
00765                                          "[e-/adu] Gain used in processing");
00766             if (vircam_fits_get_status(ff) == VIR_FATAL) {
00767                 cpl_msg_info(fctid,"Detector is flagged dead in %s",
00768                              vircam_fits_get_fullname(ff));
00769                 continue;
00770             }
00771             status = VIR_OK;
00772             (void)vircam_darkcor(ff,ps.fdark,1.0,&status);
00773             (void)vircam_lincor(ff,ps.fchantab,1,ndit,&status);
00774             (void)vircam_nditcor(ff,ndit,&status);
00775             (void)vircam_flatcor(ff,ps.fflat,&status);
00776             (void)vircam_gaincor(ff,gaincor_fac,&status);
00777             vircam_fits_set_error(ff,status);
00778         }
00779 
00780         /* Do a simple sky correction if requested */
00781 
00782         if (vircam_jmp_config.skycor) {
00783             cpl_msg_info(fctid,"Doing sky correction");
00784             vircam_jmp_skycor();
00785         } else {
00786             for (i = 0; i < ps.nscience; i++) {
00787                 pp = vircam_fits_get_ehu(ps.sci_fits[i]);
00788                 cpl_propertylist_update_string(pp,"ESO DRS SKYCOR","none");
00789                 cpl_propertylist_update_comment(pp,"ESO DRS SKYCOR",
00790                                                 "Sky correction method");
00791             }
00792         }
00793 
00794         /* Do destripe now */
00795 
00796         if (vircam_jmp_config.destripe) {
00797             cpl_msg_info(fctid,"Doing destripe");
00798             for (i = 0; i < ps.nscience; i++) {
00799                 ff = ps.sci_fits[i];
00800                 (void)vircam_destripe(ff,ps.mask,&status);
00801                 vircam_fits_set_error(ff,status);
00802             }           
00803         }
00804 
00805         /* Calculate the illumination correction */
00806 
00807         cpl_msg_info(fctid,"Doing illumination correction");
00808         (void)strcpy(current_cat,ps.catname);
00809         (void)strcpy(current_catpath,ps.catpath);
00810         vircam_jmp_illum();     
00811         
00812         /* Save the simple images */ 
00813 
00814         if (vircam_jmp_config.savesimple) {
00815             cpl_msg_info(fctid,"Saving simple images");
00816             if (vircam_jmp_save_simple(framelist,parlist) != 0) {
00817                 vircam_jmp_tidy(0);
00818                 return(-1);
00819             }
00820         }
00821         vircam_mask_clear(ps.mask);
00822 
00823         /* Save the illumination correction table */
00824 
00825         cpl_msg_info(fctid,"Saving illumination correction table");
00826         dummyqc = vircam_standard_process_dummyqc(3);
00827         if (vircam_jmp_save_illum(framelist,parlist) != 0) {
00828             vircam_jmp_tidy(0);
00829             freepropertylist(dummyqc);
00830             return(-1);
00831         }
00832         freepropertylist(dummyqc);
00833 
00834         /* Look at the first frame in the list and see if the number of
00835            microsteps is greater than 1. If so, then we'll have to go
00836            through interleaving. */
00837 
00838         retval = vircam_pfits_get_nusteps(vircam_fits_get_phu(ps.sci_fits[0]),
00839                                           &nusteps);
00840         if (retval != VIR_OK) {
00841             cpl_msg_warning(fctid,"Unable to get nusteps from header.\nAssuming no microstepping");
00842             nusteps = 1;
00843         }
00844         
00845         /* If the number of microsteps is 1 then copy over the good frames
00846            into a fits list for dithering. */
00847 
00848         ps.ndith = 0;
00849         if (nusteps < 4) {
00850             if (nusteps == 1) 
00851                 cpl_msg_info(fctid,"No interleaving will be done");
00852             else
00853                 cpl_msg_warning(fctid,"Illegal number of microsteps: %d\nNo interleaving will be done",nusteps);
00854             ps.dith_input = cpl_malloc(ps.nscience*sizeof(vir_fits *));
00855             ps.dithc_input = cpl_malloc(sizeof(vir_fits *));
00856             for (i = 0; i < ps.nscience; i++) {
00857                 if (vircam_fits_get_status(ps.sci_fits[i]) == VIR_OK) 
00858                     ps.dith_input[ps.ndith++] = ps.sci_fits[i];
00859             }
00860             ps.dithc_input[0] = ps.fconf;
00861             ps.ndithc = 1;
00862             interlv = 0;
00863 
00864         /* If the number of microsteps is more than 1, then we need
00865            to do interleaving. The interleaving routine define
00866            ps.dith_input. */
00867 
00868         } else {
00869             cpl_msg_info(fctid,"Interleaving");
00870             vircam_jmp_interleave();
00871             cpl_msg_info(fctid,"Saving superframe images");
00872             if (vircam_jmp_save_super(framelist,parlist) != 0) {
00873                 vircam_jmp_tidy(0);
00874                 return(-1);
00875             }
00876             interlv = 1;
00877         }
00878 
00879         /* Work out the jitter offsets and the stack the jitter frame */
00880 
00881         cpl_msg_info(fctid,"Working out jitter offsets");
00882         vircam_jmp_dither_offsets();
00883         cpl_msg_info(fctid,"Stacking jittered frame");
00884         vircam_jmp_dither_images();
00885 
00886         /* Do a catalogue generation */
00887 
00888         cpl_msg_info(fctid,"Doing object extraction");
00889         vircam_jmp_catalogue();
00890 
00891         /* Create a matched standards table */
00892 
00893         cpl_msg_info(fctid,"Matching objects with 2mass standards");
00894         vircam_jmp_matched_stds();
00895 
00896         /* Do a WCS fit for the dithered image */
00897 
00898         cpl_msg_info(fctid,"Fitting a WCS");
00899         vircam_jmp_wcsfit();
00900 
00901         /* Finally do the photometric zeropoint fit */
00902 
00903         cpl_msg_info(fctid,"Doing 2mass photometric zeropoint calculation");
00904         vircam_jmp_photcal();
00905         if (vircam_fits_get_status(ps.stack_frame) == VIR_OK) {
00906             ehu = vircam_fits_get_ehu(ps.stack_frame);
00907             cpl_propertylist_update_float(ehu,"ESO QC ZPT_2MASS",
00908                                           cpl_propertylist_get_float(ehu,"ESO QC MAGZPT"));
00909             cpl_propertylist_set_comment(ehu,"ESO QC ZPT_2MASS",
00910                                          cpl_propertylist_get_comment(ehu,"ESO QC MAGZPT"));
00911         }
00912 
00913         /* Redo the photometric zeropoint with another catalogue if we
00914            have one */
00915 
00916         if (ps.catpath2 != NULL) {
00917             freetable(ps.matchstds);
00918             (void)strcpy(current_cat,ps.catname2);
00919             (void)strcpy(current_catpath,ps.catpath2);
00920             cpl_msg_info(fctid,"Matching objects with %s standards",
00921                          ps.catname2);
00922             vircam_jmp_matched_stds();
00923             cpl_msg_info(fctid,"Doing %s photometric zeropoint calculation",
00924                          ps.catname2);
00925             vircam_jmp_photcal();
00926             if (vircam_fits_get_status(ps.stack_frame) == VIR_OK) {
00927                 ehu = vircam_fits_get_ehu(ps.stack_frame);
00928                 cpl_propertylist_update_float(ehu,"ESO QC ZPT_STDS",
00929                                               cpl_propertylist_get_float(ehu,"ESO QC MAGZPT"));
00930                 cpl_propertylist_set_comment(ehu,"ESO QC ZPT_STDS",
00931                                              cpl_propertylist_get_comment(ehu,"ESO QC MAGZPT"));
00932                 cpl_propertylist_update_string(ehu,"ESO QC ZPT_STDS_CAT",
00933                                                ps.catname2);
00934                 cpl_propertylist_set_comment(ehu,"ESO QC ZPT_STDS_CAT",
00935                                              "Catalogue used in zeropoint calc");
00936             }
00937         } else {
00938             if (vircam_fits_get_status(ps.stack_frame) == VIR_OK) {
00939                 ehu = vircam_fits_get_ehu(ps.stack_frame);
00940                 cpl_propertylist_update_float(ehu,"ESO QC ZPT_STDS",0.0);
00941                 cpl_propertylist_set_comment(ehu,"ESO QC ZPT_STDS",
00942                                              "No secondary standard calib");
00943                 cpl_propertylist_update_string(ehu,"ESO QC ZPT_STDS_CAT","");
00944                 cpl_propertylist_set_comment(ehu,"ESO QC ZPT_STDS_CAT",
00945                                              "No secondary standard catalogue");
00946             }
00947         }
00948 
00949         /* Save the dithered images */
00950 
00951         cpl_msg_info(fctid,"Saving stacked images");    
00952         dummyqc = vircam_standard_process_dummyqc(1);
00953         if (vircam_jmp_save_stack(framelist,parlist) != 0) {
00954             vircam_jmp_tidy(0);
00955             freepropertylist(dummyqc);
00956             return(-1);
00957         }
00958         freepropertylist(dummyqc);
00959         if (vircam_jmp_config.savecat) {
00960             cpl_msg_info(fctid,"Saving stacked image catalogues");
00961             dummyqc = vircam_standard_process_dummyqc(2);
00962             if (vircam_jmp_save_catalogue(framelist,parlist) != 0) {
00963                 vircam_jmp_tidy(0);
00964                 freepropertylist(dummyqc);
00965                 return(-1);
00966             }
00967             freepropertylist(dummyqc);
00968         }
00969 
00970         /* Clean up on aisle 12! */
00971 
00972         vircam_jmp_tidy(1);
00973     }
00974 
00975     /* Final cleanup */
00976 
00977     vircam_jmp_tidy(0);
00978     return(0);
00979 }
00980 
00981 static cpl_propertylist *vircam_standard_process_dummyqc(int type) {
00982     cpl_propertylist *p;
00983 
00984     /* Get an empty property list */
00985 
00986     p = cpl_propertylist_new();
00987 
00988     /* Now switch for the various products */
00989 
00990     switch (type) {
00991 
00992     /* Stack images */
00993 
00994     case 1:
00995         cpl_propertylist_update_double(p,"ESO QC WCS_DCRVAL1",0.0);
00996         cpl_propertylist_set_comment(p,"ESO QC WCS_DCRVAL1",
00997                                      "[deg] change in crval1");
00998         cpl_propertylist_update_double(p,"ESO QC WCS_DCRVAL2",0.0);
00999         cpl_propertylist_set_comment(p,"ESO QC WCS_DCRVAL2",
01000                                      "[deg] change in crval2");
01001         cpl_propertylist_update_double(p,"ESO QC WCS_DTHETA",0.0);
01002         cpl_propertylist_set_comment(p,"ESO QC WCS_DTHETA",
01003                                      "[deg] change in rotation");
01004         cpl_propertylist_update_double(p,"ESO QC WCS_SCALE",0.0);
01005         cpl_propertylist_set_comment(p,"ESO QC WCS_SCALE",
01006                                      "[arcsec] mean plate scale");
01007         cpl_propertylist_update_double(p,"ESO QC WCS_SHEAR",0.0);
01008         cpl_propertylist_set_comment(p,"ESO QC WCS_SHEAR",
01009                                      "[deg] abs(xrot) - abs(yrot)");
01010         cpl_propertylist_update_double(p,"ESO QC WCS_RMS",0.0);
01011         cpl_propertylist_set_comment(p,"ESO QC WCS_RMS",
01012                                      "[arcsec] Average error in WCS fit");
01013         cpl_propertylist_update_float(p,"ESO QC MAGZPT",0.0);
01014         cpl_propertylist_set_comment(p,"ESO QC MAGZPT",
01015                                      "[mag] photometric zeropoint");
01016         cpl_propertylist_update_float(p,"ESO QC MAGZERR",0.0);
01017         cpl_propertylist_set_comment(p,"ESO QC MAGZERR",
01018                                      "[mag] photometric zeropoint error");
01019         cpl_propertylist_update_int(p,"ESO QC MAGNZPT",0);
01020         cpl_propertylist_set_comment(p,"ESO QC MAGNZPT",
01021                                      "number of stars in magzpt calc");
01022         cpl_propertylist_update_int(p,"ESO QC MAGNCUT",0);
01023         cpl_propertylist_set_comment(p,"ESO QC MAGNCUT",
01024                                      "number of cut from magzpt calc");
01025         cpl_propertylist_update_float(p,"ESO QC LIMITING_MAG",0.0);
01026         cpl_propertylist_set_comment(p,"ESO QC LIMITING_MAG",
01027                                      "[mag] 5 sigma limiting mag");
01028         cpl_propertylist_update_float(p,"ESO QC ZPT_2MASS",0.0);
01029         cpl_propertylist_set_comment(p,"ESO QC ZPT_2MASS",
01030                                      "[mag] photometric zeropoint");
01031         cpl_propertylist_update_float(p,"ESO QC ZPT_STDS",0.0);
01032         cpl_propertylist_set_comment(p,"ESO QC ZPT_STDS",
01033                                      "[mag] photometric zeropoint");
01034         cpl_propertylist_update_string(p,"ESO QC ZPT_STDS_CAT","");
01035         cpl_propertylist_set_comment(p,"ESO QC ZPT_STDS_CAT",
01036                                      "Catalogue used in zeropoint calc");
01037         break;
01038         
01039     /* Catalogues */
01040 
01041     case 2:
01042         cpl_propertylist_update_float(p,"ESO QC SATURATION",0.0);
01043         cpl_propertylist_set_comment(p,"ESO QC SATURATION",
01044                                      "[adu] Saturation level");
01045         cpl_propertylist_update_float(p,"ESO QC MEAN_SKY",0.0);
01046         cpl_propertylist_set_comment(p,"ESO QC MEAN_SKY",
01047                                      "[adu] Median sky brightness");
01048         cpl_propertylist_update_float(p,"ESO QC SKY_NOISE",0.0);
01049         cpl_propertylist_set_comment(p,"ESO QC SKY_NOISE",
01050                                      "[adu] Pixel noise at sky level");
01051         cpl_propertylist_update_float(p,"ESO QC IMAGE_SIZE",0.0);
01052         cpl_propertylist_set_comment(p,"ESO QC IMAGE_SIZE",
01053                                      "[pixels] Average FWHM of stellar objects");
01054         cpl_propertylist_update_float(p,"ESO QC ELLIPTICITY",0.0);
01055         cpl_propertylist_set_comment(p,"ESO QC ELLIPTICITY",
01056                                      "Average stellar ellipticity (1-b/a)");
01057         cpl_propertylist_update_float(p,"ESO QC APERTURE_CORR",0.0);
01058         cpl_propertylist_set_comment(p,"ESO QC APERTURE_CORR",
01059                                      "Stellar ap-corr 1x core flux");
01060         cpl_propertylist_update_int(p,"ESO QC NOISE_OBJ",0);
01061         cpl_propertylist_set_comment(p,"ESO QC NOISE_OBJ",
01062                                      "Number of noise objects");
01063         break;
01064 
01065     /* Illumination tables */
01066 
01067     case 3:
01068         cpl_propertylist_update_float(p,"ESO QC ILLUMCOR_RMS",0.0);
01069         cpl_propertylist_set_comment(p,"ESO QC ILLUMCOR_RMS",
01070                                      "RMS of illumination correction map");
01071         break;
01072     default:
01073         break;
01074     }
01075 
01076     /* Get out of here */
01077 
01078     return(p);
01079 }
01080 
01083 /*
01084 
01085 $Log: vircam_standard_process.c,v $
01086 Revision 1.40  2010/12/09 13:20:49  jim
01087 Modified some comments
01088 
01089 Revision 1.39  2010/09/13 11:43:13  jim
01090 Added flag DRS SKYCOR if background correction isn't being done
01091 
01092 Revision 1.38  2010/09/09 12:16:00  jim
01093 Added new QC parameter MAGNCUT
01094 
01095 Revision 1.37  2010/06/03 11:31:06  jim
01096 Modified so that deblending is done by default
01097 
01098 Revision 1.36  2010/03/21 06:47:42  jim
01099 Added code to trap for problems in saving products
01100 
01101 Revision 1.35  2010/03/12 10:44:07  lbilbao
01102 Added missing header inclusion.
01103 
01104 Revision 1.34  2010/02/08 16:35:26  jim
01105 Moved the definition of propertylist ps.phupaf to the recipes so that
01106 this doesn't fail when we don't save the simple images
01107 
01108 Revision 1.33  2010/01/31 18:56:35  jim
01109 Now optionally saves the simple images. Also included missing ndit correction
01110 
01111 Revision 1.32  2009/12/11 06:52:56  jim
01112 Minor mods to documentation
01113 
01114 Revision 1.31  2009/09/09 09:50:21  jim
01115 Modified to try and get headers right
01116 
01117 Revision 1.30  2009/07/03 12:30:04  jim
01118 Default value of rcore is now 3
01119 
01120 Revision 1.29  2009/02/04 09:23:29  jim
01121 Moved destriping to after sky subtraction
01122 
01123 Revision 1.28  2008/12/08 06:38:41  jim
01124 Added trap for crval==0 error. Also added a trap for illegal 2x1 microstep
01125 sequences
01126 
01127 Revision 1.27  2008/11/27 09:14:47  jim
01128 Fixed PRO CATG keyword values for interleaved confidence maps in docs
01129 
01130 Revision 1.26  2008/11/25 18:55:59  jim
01131 took out extra sky stuff
01132 
01133 Revision 1.25  2008/11/25 12:03:03  jim
01134 Now allows for masked object sky estimation
01135 
01136 Revision 1.24  2008/11/25 06:18:30  jim
01137 Changed a reference to VIRCAM_CAL_2MASS to VIRCAM_CAL_REFCAT.
01138 
01139 Revision 1.23  2008/10/01 04:59:13  jim
01140 Added call to vircam_frameset_fexists to check input frameset
01141 
01142 Revision 1.22  2008/05/06 12:15:21  jim
01143 Changed to use new version of vircam_catpars
01144 
01145 Revision 1.21  2007/11/26 09:59:06  jim
01146 Recipe now takes ndit into account when doing linearity correction
01147 
01148 Revision 1.20  2007/10/25 18:39:22  jim
01149 Altered to remove some lint messages
01150 
01151 Revision 1.19  2007/10/19 06:55:06  jim
01152 Modifications made to use new method for directing the recipes to the
01153 standard catalogues using the sof
01154 
01155 Revision 1.18  2007/07/09 13:21:56  jim
01156 Modified to use new version of vircam_exten_range
01157 
01158 Revision 1.17  2007/06/13 08:11:27  jim
01159 Modified docs to reflect changes in DFS tags
01160 
01161 Revision 1.16  2007/05/08 21:31:16  jim
01162 fixed typo
01163 
01164 Revision 1.15  2007/05/08 10:42:44  jim
01165 Added gain correction
01166 
01167 Revision 1.14  2007/05/02 12:53:11  jim
01168 typo fixes in docs
01169 
01170 Revision 1.13  2007/04/26 13:09:40  jim
01171 fixed typos
01172 
01173 Revision 1.12  2007/04/04 16:05:59  jim
01174 Modified to make paf information a bit more correct
01175 
01176 Revision 1.11  2007/04/04 10:36:18  jim
01177 Modified to use new dfs tags
01178 
01179 Revision 1.10  2007/03/29 12:19:39  jim
01180 Little changes to improve documentation
01181 
01182 Revision 1.9  2007/03/14 14:49:13  jim
01183 Fixed problem with missing paf files in jmp recipes if detlive = F. Also
01184 fixed problem where extra dummy products were being created
01185 
01186 Revision 1.8  2007/03/06 12:00:48  jim
01187 Fixed stupid typo in header
01188 
01189 Revision 1.7  2007/03/02 12:37:51  jim
01190 Fixed small memory leak
01191 
01192 Revision 1.6  2007/03/01 12:41:49  jim
01193 Modified slightly after code checking
01194 
01195 Revision 1.5  2007/02/07 10:12:40  jim
01196 Removed calls to vircam_ndit_correct as this is now no longer necessary
01197 
01198 Revision 1.4  2006/12/19 13:32:03  jim
01199 Images that are flagged as dead detectors now generate an INFO rather than
01200 an ERROR message
01201 
01202 Revision 1.3  2006/11/29 12:28:45  jim
01203 Modified so that the correct recipe names would appear in the headers of
01204 data products
01205 
01206 Revision 1.2  2006/11/28 20:57:43  jim
01207 Added illumination correction section
01208 
01209 Revision 1.1  2006/11/27 12:15:43  jim
01210 Initial entry
01211 
01212 
01213 */

Generated on 7 Feb 2011 for VIRCAM Pipeline by  doxygen 1.6.1