# This script "collates" the data in different passbands -- that is,
#   it puts together in one place the raw data gathered in different
#   passbands of the same field.  It requires that there already exist
#   instrumental photometry in BOTH V and I, and calibrated astrometry
#   in both passbands.
#
# If only one passband has been reduced so far, the "collate" procedure
#   prints a warning message and does nothing
#   
#      
# Removed "input_dir" from parameters, since we expect all input
#   (generated by earlier steps of pipeline) to be in the same
#   directory we use for output.  Hence, "output_dir" suffices.
# Also, we no longer read airmass as an input -- we always calculate
#   it from JD and location.
# -- MWR 1/25/2001
#
# Added "latitude" and "longitude" parameters, which we pass to the
#   "collate" executable as command-line arguments
# -- MWR 1/28/2001
#
# Modified "matching_image_name" to use the new convention for Mark IV
#   image names, as of Data Disk 17.  Ex:  hvra1992373.fits
#   Also modified "create_collate_filename" so that it produces
#   collated data files for each image pair with names like "Mhra1992799.clt"
# -- MWR 4/16/2001
#
# Modify "run_collate" so that it removes any output file if the 
#   "collate_exe" program fails.  An empty output file can cause
#   problems for the photom stage of the pipeline.
# -- MWR 5/15/2003
#
# Added option for "collate_orphans", if user wants to keep all stars,
#   even those detected in only a single passband.
# -- MWR 3/26/2005


if { [catch { set debug } ] != 0 } {
  set debug 2
}

# contains source for the "get_param" proc
source param.tcl



#############################################################################
# Given a param file with input values, and the name of one reduced
#   image for which we have already done photometry, 
#   
#       - read and parse the values in the param file
#
#       - check to make sure that all the needed information
#            (.coo, .pht and .ast files for both V and I)
#            already exists
#
#       - find all stars which appear in both V and I, and write them
#            to an output file
#            
#       
# Returns
#    0              if all goes well
#    1              if there's an error
#
proc collate { param_file this_image } {
  global debug

  if { $debug > 0 } {
    puts "entering collate for image $this_image"
  }

  if { [file exists $param_file] != 1 } {
    puts "collate: can't find param file $param_file"
    return 1
  }

  set output_dir       [get_param $param_file "output_dir"]
  set latitude         [get_param $param_file "latitude"]
  set longitude        [get_param $param_file "longitude"]
  set collate_matchrad [get_param $param_file "collate_matchrad"]
  set collate_exe      [get_param $param_file "collate_exe"] 
  set collate_orphans  [get_param $param_file "collate_orphans"]

  # Get the name of the directory in which the "collate" program lives,
  #   and prepend it to the executable's name
  set photom_dir [get_directory "photom_dir"]
  set collate_exe $photom_dir/$collate_exe

  # get information about the image
  if { [single_image_info $this_image image_info_list] != 0 } {
    puts stderr "collate: single_image_info fails for image $this_image"
   return 1
  }

  # if this is not an "object" image, don't process it!  Just return
  #   with code 0, but do print a warning message
  set type [get_image_value $image_info_list "type"]
  if { [string compare $type "object"] != 0 } {
    if { $debug > 0 } {
      puts "collate: image $this_image is not 'object', so skip it"
    }
    return 0
  }

  # get the Julian Date, and exposure time for the image -- 
  #   we'll need them later
  set jd [get_image_value $image_info_list "jd"]
  set exptime [get_image_value $image_info_list "exposure"]
  if { $debug > 1 } {
    puts "collate: image $this_image has JD $jd exptime $exptime"
  }

  # figure out the name of the "matching" image -- that is, the image
  #   taken with the other camera at the same time as this one.
  #   Each V-band image has a matching I-band image, and vice versa.
  if { [matching_image_name $this_image vband_image iband_image] != 0 } {
    puts stderr "collate: matching_image_name fails for $this_image"
    return 1
  }

  # If this image is not in the "primary" filter -- usually V --
  #   then we want to quit at this stage, because we'll deal with
  #   this image as a "secondary" image when processing its corresponding
  #   "primary" image.
  # We can tell is this is a "primary" image by looking to see if it
  #   matches the name of the "vband_image" 
  if { [string compare $this_image $vband_image] == 0 } {
    if { $debug > 1 } {
      puts "collate: $this_image is a primary image, we continue with it "
    }
  } else {
    if { $debug > 1 } {
      puts "collate: $this_image is a secondary image, we quit on it "
    }
    return 0
  }

  # verify that all the required files exist
  #
  # Note that we do print an error message, but return 0, if this
  #    procedure fails.  There may have been an error in matching
  #    one set of detected stars to the reference catalog, and we
  #    don't want that to bring the entire pipeline to a halt.
   if { [verify_files_exist $output_dir $vband_image $iband_image \
                           v_coo v_ast i_coo i_ast] != 0 } {
    puts stderr "collate: verify_files_exist fails for $this_image"
    return 0
  }

  # create the name of the output, collated datafile which will contain
  #    info in both V-band and I-band for this field
  if { [create_collate_filename $output_dir $vband_image \
                                           collate_outfile] != 0 } {
    puts stderr "collate: create_collate_filename fails for $this_image"
    return 1
  }
  if { $debug > 1 } {
    puts "collate: new filename $collate_outfile"
  }

  # run the Unix command which collages and collates the contents
  #    of the V-band and I-band data files, to create a single
  #    data file for the field
  if { [run_collate $collate_exe $output_dir $v_coo $v_ast $i_coo $i_ast \
                        $collate_matchrad $jd $latitude $longitude $exptime \
                        $collate_orphans $collate_outfile] != 0 } {
    puts stderr "collate: run_collate fails for $this_image"
    return 1
  }
  if { $debug > 1 } {
    puts "collate: run_collate returns OK "
  }


  

  return 0
}



#############################################################################
# PROCEDURE: matching_image_name
# 
# DESCRIPTION: Given the name of an image, figure out the name of the 
#              corresponding image in the other passband.  We use
#              the convention that files have names like this:
#              
#     ---- this is the new convention (Data Disk 17 and later)
#                 hvra1992737.fits
#
#           where 
#                 h                 means 'Observatory h', or Tom's house
#            
#                  v                is the filter name (other choice is 'i')
#
#                   r               means 'raw FITS file' (which it isn't)
#
#                    a              reserved for future use
#
#                     1992          modified Julian Date = JD - 2450000
#
#                         737       fractional JD
#
#
#     ---- this is the old convention (Data Disk 16 and earlier)
#                 H3R1659.840       
#           where
#                  3                is the code for V, "4" is the code for I
#
#                    1659           is the modified Julian Date:
#                                          full JD = 1659 + 2,450,000
#
#                         840       is the fractional Julian Date of exposure
#                                          so in this case, the exposure
#                                          was taken at JD
#                                               2,451,659.840
#     ----------------------------------------------------------------------
# 
# We assume that every V-band image has a matching I-band image, and vice
#   versa.  
#
# After we've figured out the matching image name, we place the
#   V-band image name in second arg, and I-band image name in third arg.
#   If no matching image is found, both are set to the null value {}.
#   
# RETURNS:
#    0              if all goes well
#    1              if there's a problem
#
proc matching_image_name { this_image vband_image iband_image } {
  upvar $vband_image v_image
  upvar $iband_image i_image
  global debug
  
  if { $debug > 0 } {
    puts "entering matching_image_name"
  }

  set match_image {}

  # we demand that the image name have the proper format, and scan it
  #   for the several pieces
  if { [scan $this_image "%c%c%s" observatory_code filter stuff ] != 3 } {
    puts stderr "matching_image_name: bad format in name of $this_image"
    return 1
  }
  # convert the characters from their decimal equivalents (which are
  #   returned by the TCL scan command) to ordinary characters;
  #   i.e. convert "104" back to "h"
  set observatory_code [format "%c" $observatory_code]
  set filter [format "%c" $filter]

  # check to see that the pieces make sense
  if { [string match "*.fits" $stuff] != 1 } {
    puts stderr "matching_image_name: image $this_image doesn't end in .fits"
    return 1
  }
  if { [string compare $filter "v"] == 0 } {
    set other_filter "i"
  } elseif { [string compare $filter "i"] == 0 } {
    set other_filter "v"
  } else {
    puts stderr "matching_image_name: bad filter $filter in $this_image"
    return 1
  }

  # create the two image names
  set v_image [format "%sv%s" $observatory_code $stuff]
  set i_image [format "%si%s" $observatory_code $stuff]

  if { $debug > 1 } {
    puts "matching_image_name: V-band $v_image, I-band $i_image"
  }

  return 0
}


##############################################################################
# PROCEDURE: verify_files_exist
# 
# DESCRIPTION: Given the names of the V-band and I-band files for this field,
#              make sure that two data files exist for each image:
#                      
#                     .coo             stellar position and shape info
#                     .ast             raw mag and calibrated (RA, Dec)
#                     
# If one or more of the required files doesn't exist, print an error message
#   and return.  
#   
# If all the files do exist, place their names into the 6 final args.
# 
# RETURNS:
#    0          if all goes well
#    1          if not
#
proc verify_files_exist { input_dir v_band i_band \
                                        v_band_coo v_band_ast \
                                        i_band_coo i_band_ast } {
  upvar $v_band_coo v_coo $v_band_ast v_ast
  upvar $i_band_coo i_coo $i_band_ast i_ast
  global debug


  if { $debug > 0 } {
    puts "entering verify_files_exist"
  } else {
    puts "in verify_files_exist, debug is $debug not > 0"
  }

  # create the file names
  set v_coo [format "%s.coo" $v_band]
  set v_ast [format "%s.ast" $v_band]
  set i_coo [format "%s.coo" $i_band]
  set i_ast [format "%s.ast" $i_band]

  # now verify that they all exist
  set failed 0
  set flist [list $v_coo $v_ast $i_coo $i_ast]
  set index 0
  while { $index < [llength $flist] } {
    set fullname $input_dir/[lindex $flist $index]
    if { [file exists $fullname] != 1 } {
      set failed 1
      puts stderr "verify_files_exist: file $fullname not found"
    }
    incr index
  }

  return $failed
}


#############################################################################
# PROCEDURE: create_collate_filename
# 
# DESCRIPTION: given an output directory name, and the name of a V-band image, 
#              create a name for the
#              datafile which will contain collated information from both
#              V-band and I-band datafiles for this field.
#
#              We expect a name like this:

# ------------------ new filename convention, Disk Set 17 and later ------
#                     hvra1992737.fits
#                     
#              and from it, we create an output name like this:
#                     /my_out_dir/Mhra1992737.clt
#
# ------------------ old filename convention, Disk Set 16 and earlier -----
#                     H3R1659.840
#                     
#              and from it, we create an output name like this:
#                     /my_out_dir/M1659.840
# -------------------------------------------------------------------------
#
#              where the "M" stands for "collate", and the remainder of
#              the name encodes the Julian Date.
#
#              Place the filename into the second arg.
#              
# RETURNS:
#    0          if all goes well
#    1          if not
#
proc create_collate_filename { output_dir vband_image collate_name } {
  upvar $collate_name m_name
  global debug

  if { $debug > 0 } { 
    puts "entering create_collate_filename"
  }

  if { [scan $vband_image "%c%c%s" observatory_code filter stuff ] != 3 } {
    puts stderr "create_collate_filename: bad V-band name $vband_image"
    return 1
  }
  set observatory_code [format "%c" $observatory_code]
  set filter [format "%c" $filter]
  if { [string compare $filter "v"] != 0 } {
    puts stderr "create_collate_filename: no 'v' in V-band name $vband_image?"
    return 1
  }

  # strip off the ".fits" at the end of the 'stuff' section of the file name
  if { [string match "*.fits" $stuff] != 1 } {
    puts stderr "create_collate_filename: no '.fits' at end of $vband_image?"
    return 1
  }
  set len [string length $stuff]
  set stripped_stuff [string range $stuff 0 [expr $len-6]]

  set m_name [format "%s/M%s%s.clt" $output_dir \
                               $observatory_code $stripped_stuff]
  if { $debug > 1 } {
    puts "create_collate_filename: output name is $m_name"
  }

  return 0
}
      



#############################################################################
# PROCEDURE: run_collate
# 
# DESCRIPTION: run the Unix command which collates data from the V-band
#              and I-band files.  The command will create a single
#              data file, including only stars which appear in both
#              the V-band and I-band images.
#   
#    Arguments are:
#
#              collate_exe           full path name of executable program 
#                                          which does the collating
#    
#              input_dir             directory containing input files
#
#              v_coo                 .coo file for V-band: star shape info
#              v_ast                 .ast file for V-band: instrumental mags
#                                           and (RA, Dec)
#
#              i_coo                 .coo file for I-band
#              i_ast                 .ast file for I-band
#              
#              matchrad              radius (in arcsec) within which a V-band
#                                          star is considered to match 
#                                          an I-band star
#
#              jd                    Julian Date of the exposures (which we
#                                          assume are simultaneous)
#               
#              latitude              latitude of observatory, decimal degrees
#
#              longitude             longitude of observatory, decimal degrees
#                                          West of Greenwich
#                                          
#              exptime               exposure time (in seconds) for all
#                                          images in this set.  Must be the
#                                          same for all images in the set!
#
#              orphans               value of the "orphans=" command-line arg
#
#              outfile               name of file into which to write the 
#                                          collated output
#                                          
#                                           
#              
# RETURNS:
#    0            if all goes well
#    1            if not
# 
proc run_collate { collate_exe input_dir v_coo v_ast i_coo i_ast matchrad \
                          jd latitude longitude exptime orphans outfile } {
  global debug

  if { $debug > 0 } {
    puts "entering run_collate "
  }

#
# This code may be resurrected if we try to handle a more general case
# than fixed V,I image pairs.
#
if { 0 } {
  # we need to figure out which image(s) form "corresponding" groups;
  #   that is, which images were taken of the same field at the same time.
  #   Pick one passband as "primary", and search for corresponding 
  #   images in each other passband
  if { [get_list_info filter_list "filter"] != 0 } {
    puts stderr "run_collate: get_list_info fails"
    return 1
  }
  if { $debug > 2 } {
    puts "filter_list is $filter_list"
  }
  if { [llength $filter_list] < 2 } {
    puts stderr "run_collate: fewer than 2 filters in list $filter_list"
    puts stderr "             nothing to collate, returning with error"
    return 1
  }
  set primary_filter [lindex $filter_list 0]
  if { $debug > 2 } {
    puts "primary_filter is $primary_filter"
  }

  # make a list of all the images in the primary filter
  set param_list [list [list "filter" $primary_filter ]]
  set primary_list [select_input_list $param_list]
  if { [llength $primary_list] == 0 } {
    puts stderr "run_collate: primary list empty?"
    return 1
  }
  if { $debug > 2 } {
    puts "primary_list is $primary_list"
  }

  # okay, now we walk through the list of primary images, looking for 
  #   corresponding images in the other passbands
  foreach primary_image $primary_list {
    if { $debug > 0 } {
      puts "next primary image is $primary_image"
      puts "hit return to continue" ; set aa [gets stdin]
    }

    if { [single_image_info $primary_image info_list] != 0 } {
      puts stderr "run_collate: single_image_info fails for $primary_image"
      return 1
    }
    set primary_ra [get_image_value $info_list "ra"]
    set primary_dec [get_image_value $info_list "dec"]
    set primary_jd [get_image_value $info_list "jd"]
    if { $debug > 2 } {
      puts "primary ra, dec are $primary_ra, $primary_dec"
      puts "primary_jd is $primary_jd"
    }

    # now, we find all files which share these values for RA, Dec, JD
    set param_list [list [list "ra"  $primary_ra] \
                         [list "dec" $primary_dec] \
			 [list "jd"  $primary_jd]  ]
    set other_list [select_input_list $param_list]
    if { $debug > 2 } {
      puts "other_list is $other_list"
    }

    # we need to remove the entry for the primary image in this list,
    #   so we create "truly_other_list", which is free of the primary_image
    set truly_other_list {}
    set truly_other_filters {}
    foreach img $other_list {
      if { [string compare $img $primary_image] != 0 } {
        set truly_other_list [concat $truly_other_list $img]

        # set element of truly_other_filters here ....
        if { [single_image_info $img other_info_list] != 0 } {
          puts stderr "run_collate: single_image_info fails for $img"
          return 1
        }
        set other_filter [get_image_value $other_info_list "filter"]
        set truly_other_filters [concat $truly_other_filters $other_filter]
      }
    }
    if { $debug > 2 } {
      puts "truly_other_list is $truly_other_list"
      puts "truly_other_filters is $truly_other_filters"
    }
  }
}

  #
  # okay -- now have 
  #       a) the primary image
  #       b) names for all the corresponding images in other filters
  # so, we're ready to create a command line for the "collate"
  #   program, which will combine the star lists for all corresponding
  #   images into a single collated list
  #
  set cmd "exec $collate_exe passband=V $input_dir/$v_coo $input_dir/$v_ast "
  set cmd "$cmd         passband=I $input_dir/$i_coo $input_dir/$i_ast "
  set cmd "$cmd         matchrad=$matchrad jd=$jd "
  set cmd "$cmd         lat=$latitude long=$longitude "
  set cmd "$cmd         exptime=$exptime orphans=$orphans outfile=$outfile "
  if { $debug > 2 } {
    puts "cmd is $cmd"
  }

  if { [catch { set retval [eval $cmd] }] != 0 } {
    puts stderr "run_collate: error executing $cmd"
  
    # if the collate command failed, it might leave behind an empty file.
    #   That can cause problems later.  Let's remove any output file
    #   now, so that won't happen.
    exec /bin/rm -f $outfile

    return 1
  }

  return 0
}