#!/usr/bin/perl
#
# Get game-by-game boxscores for one team for an entire season
#

$debug = 1;

$start_year = 1997;
$end_year = 2008;
$team = "BOS";
$url_base_a = "http://www.baseball-reference.com/teams/";
$url_base_b = "2000-schedule-scores.shtml";
$url_box_base = "http://www.baseball-reference.com/boxes/";
$tempfile = "./get_stats.tmp";
$tempfile2 = "./get_stats.2.tmp";
$tempfile3 = "./get_stats.3.tmp";


for ($year = $start_year; $year <= $end_year; $year++) {

   $outdir = sprintf "./boxes/%s_%04d", $team, $year;
   if (-d $outdir) {
     # okay, already exists, do nothing
     if ($debug > 0) {
       printf " subdir ..%s.. already exists \n", $outdir;
     }
   }
   else {
     # create the needed subdirectory 
     if ($debug > 0) {
       printf " creating subdir ..%s.. \n", $outdir;
     }
     mkdir($outdir);
   }


if (1 == 1) {
   $outfile = sprintf "%s/gamelist.html", $outdir;
   # now, we get an HTML file which has links to the individual game
   #    box score files
   $cmd = "curl ";
   $cmd .= " -o $outfile ";
   $url = sprintf "%s%s/%s", $url_base_a, $team, $url_base_b;
   $cmd .= " $url ";
   if ($debug > 0) {
     printf "cmd is ..$cmd.. \n", $cmd;
   }
   $ret = `$cmd`;
}

if (1 == 1) {
  # next, we dump the HTML into plain text for easier parsing 
  $cmd = "lynx --dump -width=500 $outfile > $tempfile";
  $ret = exec_cmd($cmd);

  # we read through the text file to get a list of all the 
  #    internal URL references, which we will put into an 
  #    array "url_array"
  $num_url_ref = 0;
  open(TEMPFILE, "$tempfile") || die("can't open file $tempfile");
  while (<TEMPFILE>) {
    $line = " " . $_;
    @words = split(/\s+/, $line);
    if ($#words == 2) {
      $ref_index = $words[1];
      $ref_index =~ s/\.//;
      if ($ref_index > 0) {
        if (($words[2] =~ /file/) || ($words[2] =~ /http/)) {
          $url_index_array[$num_url_ref] = $ref_index;
          $url_value_array[$num_url_ref] = $words[2];
          if ($debug > 0) {
            printf "  url index %4d is ..%s.. \n", 
                   $url_index_array[$num_url_ref],
                   $url_value_array[$num_url_ref];
          }
          $num_url_ref++;
          
        }
      }
    }
  } 
  close(TEMPFILE);

  # now we make a second pass reading the text version of the 
  #     file, looking for lines which contain a reference to a boxscore
  if ($debug > 0) {
    printf " about to make second pass through $tempfile \n";
  }
  open(TEMPFILE, "$tempfile") || die("can't open file $tempfile");
  while (<TEMPFILE>) {
    $line = " " . $_;
    chomp($line);
    @words = split(/\s+/, $line);
    if ($line =~ /boxscore/) {
      if ($debug > 0) {
        printf " got a line with boxscore .. \n";
        printf " %s \n", $line;
      }
      for ($i = 0; $i < $#words; $i++) {
        if ($words[$i] =~ /boxscore/) {
          $box_string = " " . $words[$i];
          if ($debug > 0) {
            printf " boxstring is ..%s.. \n", $box_string;
          }
          $box_string =~ s/\[/ /;
          $box_string =~ s/\]/ /;
          @box_words = split(/\s+/, $box_string);
          if ($#box_words == 2) {
            if ($box_words[1] > 0) {
              # we have a reference to a boxscore.  Track down 
              #     the real URL for this boxscore, using the reference number
              $found = 0;
              for ($iref = 0; $iref < $num_url_ref; $iref++) {
                if ($url_index_array[$iref] == $box_words[1]) {
                  # found it!
                  $found = 1;
                  last;
                }
              }
              if ($found == 0) {
                printf STDERR " error, can't find ref for boxscore in line \n";
                printf STDERR " %s\n", $line;
                printf STDERR " skipping, will continue\n";
              }
              else {
                if ($debug > 0) { 
                  printf "  found boxscore url ..%s.. \n", 
                           $url_value_array[$iref];
                } 
                # now put together the entire URL for the boxscore
                #    using the $url_box_base and the end of this string
                $head = $url_box_base;
                $tail = $url_value_array[$iref];
                $tail =~ s/file:\/\/localhost\/boxes//;
                $full_url = sprintf "%s/%s", $head, $tail;
                if ($debug > 0) {
                  printf " head ..%s.. tail ..%s.. full ..%s.. \n",
                        $head, $tail, $full_url;
                }
                # and download the file
                $cmd = "curl ";
                $cmd .= " -o $tempfile2 ";
                $cmd .= " $full_url ";
                exec_cmd($cmd);

                # we can get the date of this game from the URL
                $_ = $full_url;
                /[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]\.shtml/;
                $date_str = $&;
                $date_str =~ s/\.shtml//;
                $year = substr($date_str, 0, 4);
                $month = substr($date_str, 4, 2);
                $day = substr($date_str, 6, 2);
                $game_in_day = substr($date_str, 8, 1);
                if ($debug > 0) {
                  printf " date_str is ..%s..  %04d %02d %02d %01d \n", 
                           $date_str, $year, $month, $day, $game_in_day;
                }
                $filename = sprintf "box_%04d_%02d_%02d_%01d.dat",
                           $year, $month, $day, $game_in_day;
                
                # now we reduce the file into a smaller version 
                #    stripped of everything except the box score
                #    in plain text
                if (strip_box($tempfile2, $tempfile3) != 0) {
                  printf STDERR "strip_box fails on url $full_url \n";
                  exit(1);
                }

                # and move the output, clean box score to the 
                #    proper subdirectory
                $cmd = sprintf "mv $tempfile3 %s/%s ", $outdir, $filename;
                $ret = exec_cmd($cmd);
              }
            }
          }
          else {
            if ($debug > 0) { 
              printf " box_words has %d elems \n", $#box_words;
            }
          }
        }
      }
    }
    else {
      if ($debug > 0) {
        printf " ignoring line ..%s.. \n", $line;
      }
    }
  } 
  close(TEMPFILE);

  
  }

}



exit 0;



##############################################################################
# PROCEDURE: exec_cmd
#
# DESCRIPTION: Execute the given shell command line.  If the $debug flag
#              is set, we print to stdout the command line, and
#              also print out the result string.
#
# RETURNS:
#              the result of the command
#
#
sub exec_cmd {

  my($cmd, $ret);

  $cmd = $_[0];

  if ($debug > 0) {
    printf "cmd is ..$cmd.. \n";
  }
  $ret = `$cmd`;
  if ($debug > 0) {
    printf "ret is ..$ret.. \n";
  }

  return($ret);
}


##############################################################################
# PROCEDURE: strip_box
#
# DESCRIPTION: We are given the filename of a file which contains
#              the HTML for a game's boxscore, plus a lot of other junk.
#
#              Ignore all the other junk, and strip off HTML,
#              leaving a plain text boxscore for batting only.
#
#              Arguments:
#                   input_file         HTML with lots of stuff
#                   output_file        clean boxscore in plain text
#
# RETURNS:
#              0       if all goes well
#              1       if an error occurs
#
#
sub strip_box {

  my($infile, $outfile, $tempfile);
  my($line, $num_lines, $current_box);
  my($i, $j);
  my($current_box, $save_lines, @num_lines, @line_array);
  my($cmd);

  $tempfile = "./get_stats.strip_box.tmp";

  $infile = $_[0];
  $outfile = $_[1];

  if ($debug > 0) {
    printf "infile is ..$infile..  outfile is ..$outfile.. \n";
  }

  $cmd = "lynx --dump -width=500 $infile > $tempfile ";
  exec_cmd($cmd);

  # now, read through the output file
  $current_box = -1;
  $save_lines = 0;
  $num_lines[0] = 0;
  $num_lines[1] = 0;
  open(BOXTEMPFILE, "$tempfile") || die("can't open file $tempfile");
  while (<BOXTEMPFILE>) {
    $line = $_;
    chomp($line);
    # we save a single header line with the date
    if ($line =~ /Box Score and Play by Play/) {
      $header_line = $line;
    }

    # now look for lines indicating the top of a batting box score
    if ($line =~ /AB   R   H RBI   BB  SO/) {
      $current_box++;
      $save_lines = 1;
      if ($debug > 0) {
        printf " found start of box for team %d \n", $current_box;
      }
    }
 
    if ($save_lines == 1) {
      $line_array[$current_box][$num_lines[$current_box]] = $line;
      $num_lines[$current_box]++;
    }

    # should we stop saving lines?
    if (($save_lines == 1) && ($line =~ /Totals/)) {
      $save_lines = 0;
    }

  }
  close(BOXTEMPFILE); 
  if ($debug > 0) {
    printf "  strip_box: have %5d lines for team 0   %5d lines for team 1 \n",
              $num_lines[0], $num_lines[1];
  }

  # now we write just these lines into the output file
  open(OUTFILE, ">$outfile") || die("can't open file $outfile");
  printf OUTFILE "# %s \n", $header_line;
  for ($i = 0; $i < 2; $i++) {
    for ($j = 0; $j < $num_lines[$i]; $j++) {
      printf OUTFILE "%s\n", $line_array[$i][$j];
    }
  }
  close(OUTFILE);

  # and we use 'sed' to get rid of the HTML that lynx didn't strip
  $cmd = "sed 's/<[^>]*>//g' < $outfile > $tempfile";
  $ret = exec_cmd($cmd);
  $cmd = "mv $tempfile $outfile";
  $ret = exec_cmd($cmd);

  # delete temp file
  # unlink($tempfile);

  return(0);
}




