# This TCL script runs a set of high-level TCL procedures which
#   take a set of raw images and process them into a set of 
#   calibrated star lists.
#
# Returns
#   0               if all goes well
#   1               if there's a problem
#   
# MWR 5/21/2000
#
# Moved the "collate" step to its own, second loop over all file names.
#   This is necessary, since we have no guarantee that all images
#   in a given field will have been processed in time to be collated
#   in the first loop.
# MWR 2/9/2001
#
# Moved the "setup" step to the very first place in execution,
#   so that the directory names in .param files are modified
#   BEFORE they are used by "init_master_dark", etc.
# MWR 3/1/2001
#
# Changed the error-handling behavior.  When some of the main
#   processing subroutines return with an error, instead of simply
#   calling "return" (which halts program execution), use "continue"
#   so that the program moves onto the next target image.
#   The idea is not to allow a few cloudy images or dawn images
#   to prevent the complete reduction of the good images.
# MWR 5/22/2002
# 
# Added a segment to handle the new "sky" procedure.
# MWR 2/8/2003
#
# Added lines to handle the new "version" param file and TCL source code
#   file, which as its own namespace.
# MWR 4/12/2003
#
# Now, when performing the "do_refcat" step, we first try to run 
#   the Perl script "refcat.pl"; if it fails, we try the slower
#   but equivalent TCL script.
#   We also make minor changes when initializing the refcat stuff,
#   since we now have two separate catalogs, not just one.
# MWR 4/18/2003
#
# Moved the "check_compatibility" procedure from the "setup.tcl"
#   script into this file.
# MWR 4/22/2003
#
# Added a step in the "do_photom" section which creates the big
#   concatenation of all param files.  This also requires importing
#   two functions from a new private namespace for Photom.
# Also, added a pair of lines printed to stdout which give the starting
#   and ending time of program execution.  
# MWR 4/24/2003
#
# Added segment to call the "init_skyparams" function before the
#   main processing loop.
# MWR 5/14/2003



global debug
set debug 2


if { $debug > 0 } {
  set datestr [exec date]
  puts "starting pipeline execution $datestr"
}


source param.tcl
source proc_file_list.tcl
source setup.tcl
source make_list.tcl
source refcat.tcl
source make_dark.tcl
source make_flat.tcl
source ccdproc.tcl
source sky.tcl
source stars.tcl
source astrom.tcl
source collate.tcl
source photom.tcl
source thumbnail.tcl
source version.tcl


set param_file "markiv_driver.param"

set myname [get_param $param_file myname]
set do_setup     [get_param $param_file do_setup]
set do_make_list [get_param $param_file do_make_list]
set do_refcat    [get_param $param_file do_refcat]
set do_make_dark [get_param $param_file do_make_dark]
set do_make_flat [get_param $param_file do_make_flat]
set do_ccdproc   [get_param $param_file do_ccdproc]
set do_sky       [get_param $param_file do_sky]
set do_stars     [get_param $param_file do_stars]
set do_astrom    [get_param $param_file do_astrom]
set do_collate   [get_param $param_file do_collate]
set do_photom    [get_param $param_file do_photom]
set do_thumbnail [get_param $param_file do_thumbnail]


# read in names of the param files controlling subroutines
set setup_param    [get_param $param_file setup_param]
set version_param  [get_param $param_file version_param]
set list_param     [get_param $param_file list_param]
set refcat_param   [get_param $param_file refcat_param]
set dark_param     [get_param $param_file dark_param]
set flat_param     [get_param $param_file flat_param]
set ccdproc_param  [get_param $param_file ccdproc_param]
set sky_param      [get_param $param_file sky_param]    
set stars_param    [get_param $param_file stars_param]
set astrom_param   [get_param $param_file astrom_param]
set collate_param  [get_param $param_file collate_param]
set photom_param   [get_param $param_file photom_param]
set thumbnail_param [get_param $param_file thumbnail_param]

#
# Replace default values in param files with those listed in the
#   "setup.param" file (which user has presumably edited).
# 
if { $do_setup == 1 } {
  if { [setup $setup_param] != 0 } {
    puts stderr "$myname: setup returns with error"
    return 1
  }
}


# Set environment variables which we need to run the XVista programs
if { [set_sym_table $setup_param] != 0 } {
  puts stderr "$myname: set_sym_table returns with error"
  return 1
}


# Squirrel away values of several directories, which we'll
#   use later to call external programs.
#   The namespace "Directory", defined in "setup.tcl", contains
#   the routines to set/get directory values.
#
namespace import -force Directory::*
if { [init_directory $setup_param] != 0 } {
  puts stderr "$myname: init_directory returns with error"
  return 1
}

# The namespace Version contains a pair of simple procedures,
#   one to set the version number by reading a param file,
#   and a second to return the version number.
#   These are used at the start of the pipeline, when creating
#   the make_list.out file.
namespace import -force Version::*
if { [init_version $version_param] != 0 } {
  puts stderr "$myname: init_version returns with error"
  return 1
}

# make a few checks to ensure that the user has installed compatible 
#   versions of the software packages used by the pipeline
if { [check_compatibility $param_file ] != 0 } {
  puts stderr "setup: check_compatibility returns with error"
  return 1
}



# We get the name of a file which WILL hold the list of images in this run,
#   together with their filters, exposure times, etc.
# 
# The "make_list" procedure will create this file.
# The namespace Proc_File_List, defined in "proc_file_list.tcl",
#   contains procedures which act on the file list.
#   We must call "init_list_name" so that those procedures know
#   which file contains the information they use
#   We must also call the "init_list_fields" procedure to set up 
#   the format of this file.
#   And we need to call "init_skip_bad" so that we know whether we
#   should discard bad frames, or keep processing them.
#   
set list_dir  [get_param $list_param output_dir]
set list_file [get_param $list_param imagelist_name]
set list_file $list_dir/$list_file
namespace import -force Proc_File_List::*
if { [init_list_name $list_file] != 0 } {
  puts stderr "$myname: init_list_name returns with error"
  return 1
}
if { [init_list_fields $list_param] != 0 } {
  puts stderr "$myname: init_list_fields returns with error"
  return 1
} 
set skip_bad [get_param $setup_param skip_bad]
if { [init_skip_bad $skip_bad] != 0 } {
  puts stderr "$myname: init_skip_bad returns with error"
  return 1
}


# The namespace Refcat, defined in "refcat.tcl",
#   contains procedures to set the name of the mini_catalog
#   file, and to retrieve that name.
#   This code is used in "refcat.tcl", "astrom.tcl", "photom.tcl"
#
set astrom_mini_catalog [get_param $refcat_param "astrom_mini_catalog"]
set photom_mini_catalog [get_param $refcat_param "photom_mini_catalog"]
namespace import -force Refcat::*
if { [init_astrom_refcat $astrom_mini_catalog] != 0 } {
  puts stderr "$myname: init_astrom_refcat returns with error"
  return 1
}
if { [init_photom_refcat $photom_mini_catalog] != 0 } {
  puts stderr "$myname: init_photom_refcat returns with error"
  return 1
}


# initialize stuff needed by the "master dark" routines
#   in "make_dark.tcl"
namespace import -force Make_Dark::*
if { [init_master_dark $dark_param] != 0 } {
  puts stderr "$myname: init_master_dark returns with error"
  return 1
}

# initialize stuff needed by the "master flat" routines
#   in "make_flat.tcl"
namespace import -force Make_Flat::*
if { [init_master_flat $flat_param] != 0 } {
  puts stderr "$myname: init_master_flat returns with error"
  return 1
}

# initialize stuff needed by the "ccdproc" routines
#   in "ccdproc.tcl"
namespace import -force Ccdproc::*
if { [init_sub_dark_info $ccdproc_param] != 0 } {
  puts stderr "$myname: init_sub_dark_info returns with error"
  return 1
}


# initialize stuff needed by the "sky" routines
#   in "sky.tcl" and "stars.tcl"
namespace import -force Sky::*
if { [init_skyparams $sky_param] != 0 } {
  puts stderr "$myname: init_skyparams returns with error"
  return 1
}


# initialize stuff needed by the "photom" routines, defined
#   in the "photom.tcl" file
namespace import -force Photom::*





  # make a list of images, with filter, exposure time, etc.
  # 
  if { $do_make_list == 1 } {
    if { [make_list $list_param] != 0 } {
      puts stderr "$myname: make_list returns with error"
      return 1
    }
  }


  # create a small reference catalog of stars, based on a much larger
  #   catalog.  We'll use the small catalog for both astrometry and
  #   photometry.
  # 
  if { $do_refcat == 1 } {
    # first, we try to run the Perl script (which is fast)
    set cmd "exec perl refcat.pl $refcat_param"
    if { $debug > 0 } {
      puts "markiv_driver: about to eval ..$cmd.."
    }
    if { [catch { set ret [eval $cmd] } ] != 0 } {
      puts stderr "$myname: refcat.pl Perl script failed"
      puts stderr "$myname: refcat.pl Perl script failed, so try TCL script"
      if { [refcat $refcat_param] != 0 } {
        puts stderr "$myname: refcat returns with error"
        return 1
      }
    }
  }


  # create master dark images
  if { $do_make_dark == 1 } {
    if { [make_dark $dark_param] != 0 } {
      puts stderr "$myname: make_dark returns with error"
      return 1
    }
  }
  

  # create master flatfield images
  # 
  if { $do_make_flat == 1 } {
    if { [make_flat $flat_param] != 0 } {
      puts stderr "$myname: make_flat returns with error"
      return 1
    }
  }
 
  
  # If necessary, loop over all the "target" images to clean
  #   them up, find stars, do astrometry
  # 
  set enter_loop [expr ($do_ccdproc > 0) || ($do_sky > 0) || \
                       ($do_stars > 0) || ($do_astrom > 0) ]
  if { $enter_loop == 1 } {


    if { [get_list_info target_list "name"] != 0 } {
      puts stderr "$myname: get_list_info fails to find target names"
      return 1
    }
  
    foreach target_file $target_list {
      if { $debug > 1 } {
        puts "$myname: loop 1: target file is $target_file"
      }

      # "clean" the raw image 
      # 
      if { [image_include_check $target_file] == 1 } {
        if { $do_ccdproc == 1 } {
          if { [ccdproc $ccdproc_param $target_file]  != 0 } {
            puts stderr "$myname: ccdproc returns with error"
            continue
          }
        }
      }
  
      # measure the sky value and, if desired, subtract it
      #
      if { [image_include_check $target_file] == 1 } {
        if { $do_sky == 1 } {
          if { [sky $sky_param $target_file] != 0 } {
            puts stderr "$myname: sky returns with error on $target_file"
            continue
          }
        }
      }

  
      # detect and measure stars in the image
      # 
      if { [image_include_check $target_file] == 1 } {
        if { $do_stars == 1 } {
          if { [stars $stars_param $target_file] != 0 } {
            puts stderr "$myname: stars returns with error on $target_file"
            continue
          }
        }
      }
  
      
      # calibrate the astrometry of stars in the image
      #
      if { [image_include_check $target_file] == 1 } {
        if { $do_astrom == 1 } {
          if { [astrom $astrom_param $target_file] != 0 } {
            puts stderr "$myname: astrom returns with error"
            continue
          }
        }
      }
     
   
    }
  }
  #  end of first loop (clean, stars, astrometry) over each image


  # Now (if necessary), we make a SECOND pass over all the files,
  #   collating the results from all passbands together.  This step
  #   requires its own loop, because we need to create ALL output
  #   files with star lists before we can be guaranteed that 
  #   we have the matching lists for each field.
  # 
  set enter_loop [expr ($do_collate > 0) ]
  if { $enter_loop == 1 } {

    if { [get_list_info target_list "name"] != 0 } {
      puts stderr "$myname: get_list_info fails to find target names"
      return 1
    }
  
    foreach target_file $target_list {
      if { $debug > 1 } {
        puts "$myname: loop 2: target file is $target_file"
      }
  
      # gather together the raw measurements in each passband
      # 
      if { [image_include_check $target_file] == 1 } {
        if { $do_collate == 1 } {
          if { [collate $collate_param $target_file] != 0 } {
            puts stderr "$myname: collate returns with error"
            continue
          }
        }
      }

    }
  }
  # end of second (collating) loop over each image
 

  # Now, after we've reduced data from all the images, we are ready
  #   to perform a photometric solution for the entire set.
  if { $do_photom == 1 } {
    if { [photom $photom_param] != 0 } {
      puts stderr "$myname: photom returns with error"
      return 1
    }

    # At this point, we are ready to create a big file which contains 
    #   all the contents of all the param files, so that we can 
    #   reconstruct the reduction of this dataset at some later date.
    #   Since the "photom" procedure is responsible for setting 
    #   the stem of this output file's name (it's the same name
    #   used in the .cal file), we must wait until after 
    #   it has run to do this step.
    if { [concat_all_params $param_file] != 0 } {
      puts stderr "$myname: concat_all_params returns with error"
      return 1
    }
  }


 

  # display images on the screen
  # 
  if { $do_thumbnail == 1 } {
    if { [thumbnail $thumbnail_param] != 0 } {
      puts stderr "$myname: thumbnail returns with error"
      return 1
    }
  }



# end of pipeline
if { $debug > 0 } {
  set datestr [exec date]
  puts "ending pipeline execution $datestr" 
}
return 0