#!/usr/bin/perl
#
# Grab data for a single team over many years from Project Retrosheet.
#    Use the "TEAMxxxx" file within the dataset to create a list of
#    all teams which played during each season.
#
# MWR 6/22/2009
#

$debug = 1;


$data_dir = "./retrosheet";
$team = "BOS";

$start_year = 1998;
$end_year = 2008;


for ($year = $start_year; $year <= $end_year; $year++) {



  $lower_team = lc($team); 
  $long_team = get_long_name($team);
  if ($debug > 0) {
    printf "working on year %04d team %s \n", $year, $lower_team;
  }

  
  # grab the zipped datafile for the team from Retrosheet site

    $out_dir = sprintf "%s/%04d/", $data_dir, $year;
    if (-e $out_dir) {
      # do nothing
    } 
    else {
      mkdir($out_dir);
    }
    $out_file = sprintf "%s/%04d%s.zip", $out_dir, $year, $lower_team;

    $base_url = "http://www.retrosheet.org/";
    $url = sprintf "%s/%04d/%04d%s.zip", $base_url, $year, $year, $lower_team;
    $cmd = "curl -o $out_file $url";
    $ret = exec_cmd($cmd);



    # unzip the datafile, pay attention only to the TEAMxxxx file
     $dir = sprintf "%s/%04d/", $data_dir, $year;

     # unzip the datafile for this team
     $zipfile = sprintf "%s/%04d%s.zip", $dir, $year, $lower_team;
     if (-e $zipfile) {
       # okay, we can unzip it
     } 
     else {
       # print error message, skip this team/year, keep going
       printf STDERR "error, can't find zipfile %s \n", $zipfile;
       next; 
     }

     # check to see if the "zip" file is actually just an HTML file
     #    which says that the desired zip file couldn't be found;
     #    do a simple grep for "DOCTYPE HTML PUBLIC", which will
     #    be in first line of the HTML file
     $cmd = "grep 'DOCTYPE HTML PUBLIC' $zipfile";
     $ret = exec_cmd($cmd);
     if ($ret ne "") {
       # whoops, we did find the 'DOCTYPE' string, so this is NOT a valid file
       printf STDERR "zipfile %s not valid, skip it \n", $zipfile;
       next;
     }



     $zipfile = sprintf "%04d%s.zip", $year, $lower_team;
     $cmd = "(cd $dir ; unzip -o $zipfile )";
     $ret = exec_cmd($cmd);


     # find the TEAMxxxx file
     $team_file = sprintf "%s/TEAM%04d", $dir, $year;
     if (!-e $team_file) {
       printf STDERR "can't find TEAM file %s \n", $team_file;
       exit(1);
     }


       # now delete temporary files, leaving only the ZIP file archive
       @delete_list = glob("$dir/*ROS"); 
       foreach $delete_file (@delete_list) {
         unlink($delete_file);
       }
#       @delete_list = glob("$dir/TEAM????"); 
#       foreach $delete_file (@delete_list) {
#         unlink($delete_file);
#       }
       @delete_list = glob("$dir/ALL*EVX"); 
       foreach $delete_file (@delete_list) {
         unlink($delete_file);
       }



   





# all done with this year
}




exit 0;


#############################################################################
# PROCEDURE: get_long_name 
#
# DESCRIPTION: Given a short team name, get the long team name.
#              Put underscores in the long team name, so it looks like
#
#                   Boston_Red_Sox
#
# RETURNS:     the long team name     if found, given the short name
#              or exits with error    if not found
#
sub get_long_name {

  my($name, $long_name);

  $name = $_[0];

  $long_name = "";

  if ($name eq "ANA") {
    $long_name = "Anaheim_Angels";
  }
  if ($name eq "BAL") {
    $long_name = "Baltimore_Orioles";
  }
  if ($name eq "BOS") {
    $long_name = "Boston_Red_Sox";
  }
  if ($name eq "CHA") {
    $long_name = "Chicago_White_Sox";
  }
  if ($name eq "CLE") {
    $long_name = "Cleveland_Indians";
  }
  if ($name eq "DET") {
    $long_name = "Detroit_Tigers";
  }
  if ($name eq "KCA") {
    $long_name = "Kansas_City_Royals";
  }
  if ($name eq "MIN") {
    $long_name = "Minnesota_Twins";
  }
  if ($name eq "NYA") {
    $long_name = "New_York_Yankees";
  }
  if ($name eq "OAK") {
    $long_name = "Oakland_Athletics";
  }
  if ($name eq "SEA") {
    $long_name = "Seattle_Mariners";
  }
  if ($name eq "TBA") {
    $long_name = "Tampa_Bay_Devil_Rays";
  }
  if ($name eq "TEX") {
    $long_name = "Texas_Rangers";
  }
  if ($name eq "TOR") {
    $long_name = "Toronto_Blue_Jays";
  }

  
  if ($name eq "ARI") {
    $long_name = "Arizona_Diamondbacks";
  }
  if ($name eq "ATL") {
    $long_name = "Atlanta_Braves";
  }
  if ($name eq "CHN") {
    $long_name = "Chicago_Cubs";
  }
  if ($name eq "CIN") {
    $long_name = "Cincinnati_Reds";
  }
  if ($name eq "COL") {
    $long_name = "Colorado_Rockies";
  }
  if ($name eq "FLO") {
    $long_name = "Florida_Marlins";
  }
  if ($name eq "HOU") {
    $long_name = "Houston_Astros";
  }
  if ($name eq "LAN") {
    $long_name = "Los_Angeles_Dodgers";
  }
  if ($name eq "MIL") {
    $long_name = "Milwaukee_Brewers";
  }
  if ($name eq "MON") {
    $long_name = "Montreal_Expos";
  }
  if ($name eq "NYN") {
    $long_name = "New_York_Mets";
  }
  if ($name eq "PHI") {
    $long_name = "Philadelphia_Phillies";
  }
  if ($name eq "PIT") {
    $long_name = "Pittsburgh_Pirates";
  }
  if ($name eq "SDN") {
    $long_name = "San_Diego_Padres";
  }
  if ($name eq "SFN") {
    $long_name = "San_Francisco_Giants";
  }
  if ($name eq "WAS") {
    $long_name = "Washington_Nationals";
  }


  # did we fail to find the name?
  if (length($long_name) < 1) {
    printf STDERR "get_long_name fails on ..%s.. \n", $name;
    exit(1);
  }

  return($long_name);
}




##############################################################################
# PROCEDURE: exec_cmd
#
# DESCRIPTION: Execute the given shell command line.  If the $debug flag
#              is set, we print to stdout the command line, and
#              also print out the result string.
#
# RETURNS:
#              the result of the command
#
#
sub exec_cmd {

  my($cmd, $ret);

  $cmd = $_[0];

  if ($debug > 0) {
    printf "cmd is ..$cmd.. \n";
  }
  $ret = `$cmd`;
  if ($debug > 0) {
    printf "ret is ..$ret.. \n";
  }

  return($ret);
}


##############################################################################
# PROCEDURE: read_evx
#
# DESCRIPTION: Read all the information from an EVX file, which has 
#              all play-by-play for one team for an entire year
#
# RETURNS:
#              0         if all goes well
#              1         if an error occurs
#
#
sub read_evx {

  my($cmd, $ret);
  my($filename, $line, @words);
  my($i, $j);
  my($good_team);
  my($check_str);


  $filename = $_[0];

  if ($debug > 0) {
    printf "read_evx will read ..$filename.. \n";
  }

  open(READ_FILE, "$filename") || die("can't open file $filename");
  $num_games = 0;
  while (<READ_FILE>) {
    $line = $_;
    chomp($line);
 
    # count games
    if ($line =~ /^id,/) {
      $num_games++;
      if ($debug > 0) {
        printf " game line %s   num_games now %3d \n", $line, $num_games;
      }
if ($specialcase == 1) {
printf " game line %s   num_games now %3d \n", $line, $num_games;
$castillo_abs = 0;
$castillo_hits = 0;
      next;
    }
}

    # look for at-bats by players on our team
    # "play"  entries look like
    #               play,1,0,pedrd001,01,CX,S68/G
    #
    # where    player_id = col 3
    #          play result = col 6       
    #    
    # Play result has important codes as follows
    #     starts with "S[0-9]"  or "S/" or "S" or "S."   single, a hit
    #     starts with "D[0-9]"  or "D/" or "D" or "D."   double, a hit
    #     starts with "DGR"     ground-rule double, a hit
    #     starts with "T[0-9]"  or "T/" or "T" or "T."   triple, a hit
    #     starts with "HR/"     or "H/" or "H" or "HR"   home run, a hit
    #     starts with "H[0-9]  or "HR[0-9]"  inside-park HR, a hit
    #     starts with "HP"                 hit by pitch, no at-bat
    #     starts with "NP"                 no play, no at-bat
    #     starts with "IW"      or "I/"    intentional walk, no at-bat
    #     starts with "W"                  walk, no at-bat
    #     starts with "C"                  catcher inter, no at-bat
    #
    #     anything else                    an at-bat, an out
    #      
    # If the string /SH appears in col 6, then the play was a sacrifice
    # hit, and does not count as an at-bat for hitting streaks.
    #
    if ($line =~ /^play,/) {
      @words = split(/,/, $line);
      $player_code = $words[3];
      $player_index = lookup_player($player_code);
      if ($player_index == -1) {
        # not a player on our team, so ignore it
        next;
      }

if ($specialcase == 1) {
if ($player_code eq "castl001") {
  printf " game %d ", $num_games;
  printf " line %s ", $line;
}
}

      # okay, this is one of our players.  Figure out if this 
      #     counts as
      #            an at-bat and a hit
      #            an at-bat and no hit
      #            no at-bat
      $result = $words[6];
      if (($result =~ /^S\//) || ($result =~ /^S[0-9]/) ||
          ($result =~ /^S\r/) || ($result =~ /^S\./) ) {
        $is_atbat = 1;
        $is_hit = 1;
      }
      elsif (($result =~ /^D\//) || ($result =~ /^D[0-9]/) || 
             ($result =~ /^D\r/) || ($result =~ /^D\./) ||
             ($result =~ /^DGR/)) {
        $is_atbat = 1;
        $is_hit = 1;
      }
      elsif (($result =~ /^T\//) || ($result =~ /^T[0-9]/) ||
             ($result =~ /^T\r/) || ($result =~ /^T\./) ) {
        $is_atbat = 1;
        $is_hit = 1;
      }
      elsif (($result =~ /^H\//) || ($result =~ /^HR/) ||
             ($result =~ /^H\r/) ||  ($result =~ /^H[0-9]/)) {
        $is_atbat = 1;
        $is_hit = 1;
      }
      elsif (($result =~ /^HP/) || ($result =~ /^NP/)) {
        $is_atbat = 0;
        $is_hit = 0;
      }
      elsif (($result =~ /^I\//) || ($result =~ /^IW/)) {
        $is_atbat = 0;
        $is_hit = 0;
      }
      elsif (($result =~ /^W/) || ($result =~ /^C/)) {
        $is_atbat = 0;
        $is_hit = 0;
      }
      elsif ($result =~ /\/SH\//) {
        $is_atbat = 0;
        $is_hit = 0;
      }
      else {
        $is_atbat = 1;
        $is_hit = 0;
      }
      if ($debug > 0) {
        printf " result ..%s..  yields atbat %d hit %d \n",
                    $result, $is_atbat, $is_hit;
      }

if ($specialcase == 1) {
if ($player_code eq "castl001") {
  if ($is_atbat == 1) {
    $castillo_abs++;
  }
  if ($is_hit == 1) {
    $castillo_hits++;
  }
  printf " result ..%s.. ", $result;
  if (($result =~ /^S/) || ($result =~ /^S[0-9]/)) {
    printf " is a single ";
  }
  else {
    printf " not  single ";
  }
  printf " atbat %d hit %d  cum %d %d \n", 
        $is_atbat, $is_hit, $castillo_abs, $castillo_hits;
}
}


      # record the result of this event in the arrays
      #        $ab_array[player_index][game_index]
      #  and
      #        $hit_array[player_index][game_index]
      $ab_array[$player_index][$num_games] += $is_atbat;
      $hit_array[$player_index][$num_games] += $is_hit;
   
    }
      
 
  }
  close(READ_FILE);

  if ($debug > 0) {
    printf " total of %3d games \n", $num_games;
  }


  return(0);
}


##############################################################################
# PROCEDURE: summarize_players
#
# DESCRIPTION: Go through the information for each player on a team
#              over the entire season.  Summarize by printing out a single
#              line per player with structure like this:
#
#                 col 1:     player ID
#                 col 2:     player name, lastname_firstname
#                 col 3:     number of official at-bats
#                 col 4:     number of hits
#                 col 5:     batting avg (hits / at-bats)
#                 col 6:     number of games with at least one official at-bat
#                 
#              then "N" pairs of numbers follow, where "N" = number of
#              games with at least one official at-bat.  The pairs are
#
#                     #atbats #hits   #atbats #hits  #atbats #hits ....
#              
#              so one pair per game with at least one official at-bat.
#
#              Send the output to RESULTS_FILE
#
# RETURNS:
#              0         if all goes well
#              1         if an error occurs
#
#
sub summarize_players {

  my($cmd, $ret);
  my($filename, $line, @words);
  my($i, $j);
  my($good_team);
  my($check_str);


  if ($debug > 0) {
    printf "entering summarize_players .... \n";
  }

  for ($i = 0; $i < $num_players; $i++) {

    # add up stuff:
    #         number of games with >= 1 at-bat
    #         number of at-bats
    #         number of hits
    $games_played = 0;
    $num_abs = 0;
    $num_hits = 0;
    for ($i_game = 0; $i_game <= $num_games; $i_game++) {
      if ($ab_array[$i][$i_game] > 0) {
        $games_played++;
        $num_abs += $ab_array[$i][$i_game];
        $num_hits += $hit_array[$i][$i_game];
      }
    }

    # compute batting average
    if ($num_abs > 0) {
      $avg = $num_hits / $num_abs;
    }
    else {
      $avg = 0.000;
    }

    # now print stuff out
    printf RESULTS_FILE "%-10s %25s", 
                $roster_id_array[$i], $roster_name_array[$i];
    printf RESULTS_FILE "%4d %3d %5.3f %3d ", 
                $num_abs, $num_hits, $avg, $games_played;
    for ($i_game = 0; $i_game <= $num_games; $i_game++) {
      if ($ab_array[$i][$i_game] > 0) {
        printf RESULTS_FILE "%2d %2d ", 
                $ab_array[$i][$i_game], $hit_array[$i][$i_game];
      }
    }
    printf RESULTS_FILE "\n";


  }


  return(0);
}


##############################################################################
# PROCEDURE: read_roster
#
# DESCRIPTION: Read a list of players from a roster file.
#              Return the number of players.  Place into
#              global arrays "roster_id_array[]" 
#              and "roster_name_array[]" 
#              their names and ID strings.
#
#              Format of lines in the roster file is
#
#                 aardd001,Aardsma,David,R,R,BOS,P
#
#
# RETURNS:
#              0         if all goes well
#              1         if an error occurs
#
#
sub read_roster {

  my($cmd, $ret);
  my($filename, $line, @words);
  my($i, $j);
  my($good_team);
  my($check_str);
  my($num_read);
  my($lastname, $firstname);


  $filename = $_[0];

  if ($debug > 0) {
    printf "read_roster will read ..$filename.. \n";
  }

  open(READ_FILE, "$filename") || die("can't open file $filename");
  $num_read = 0;
  while (<READ_FILE>) {
    $line = $_;
    chomp($line);

    @words = split(/,/, $line);
    $roster_id_array[$num_read] = $words[0];
    $lastname = $words[1];
    $lastname =~ s/ /_/g;
    $firstname = $words[2];
    $firstname =~ s/ /_/g;
    $roster_name_array[$num_read] = $lastname . "_" . $firstname;

    if ($debug > 0) {
      printf " player %5d is %s %s \n", 
            $num_read, $roster_id_array[$num_read],
            $roster_name_array[$num_read];
    }

    $num_read++;
    
  }
  close(READ_FILE);

  return($num_read);
}


##############################################################################
# PROCEDURE: lookup_player
#
# DESCRIPTION: Given a player ID string, check to see if it is one
#              listed in the array "roster_id_array[]". 
#
#              If yes, return its index in the array.
#              If no, return -1.
#
# RETURNS:
#              index     if player found in array
#              -1        if not
#
#
sub lookup_player {

  my($i);
  my($id_code);


  $id_code = $_[0];

  if ($debug > 0) {
    printf " lookup_player will look for ..$id_code.. \n";
  }

  for ($i = 0; $i < $num_players; $i++) {
    if ($roster_id_array[$i] eq $id_code) {
      if ($debug > 0) {
        printf " found him at index %d \n", $i;
      }
      return($i);
    }
  }

  if ($debug > 0) {
    printf "  not found \n";
  }
  return(-1);
}





##############################################################################
# PROCEDURE: save_player_data
#
# DESCRIPTION: Go through one box score's worth of data on players.
#
#                - Get player names and see if they appear in our 
#                        array of player names.  If not, add player
#                        to the array.
#
#
# RETURNS:
#              0         if all goes well
#              1         if an error occurs
#
#
sub save_player_data {

  my($cmd, $ret);
  my($line, @words);
  my($i);


  if ($debug > 0) {
    printf " about to enter save_player_data \n";
  }

  for ($i = 0; $i < $num_lines; $i++) {
    # get the player's name.  
    $name_str = substr($line_array[$i], 0, 27);
    if ($debug > 0) {
      printf "  line     %5d is ..%s.. \n", $i, $line_array[$i];
      printf "  name_str %5d is ..%s.. \n", $i, $name_str;
    }

    # crawl backwards through the name_str, which looks like this:
    #         N Garciaparra SS      
    #   the first non-blank string we encounter is the position.  
    #   We want to eliminate that, and then change all 
    #   remaining spaces to underscores, so the name becomes
    #         N_Garciaparra
    $fixed_name = 0;
    $len = length($name_str);
    for ($j = $len - 1; $j >= 0; $j--) {
      if (substr($name_str, $j, 1) ne " ") {
        # we have found the position string ... crawl to next space
        for ($pos = $j; $pos >= 0; $pos--) {
          if (substr($name_str, $pos, 1) eq " ") {
            last;
          }
        }
        if ($pos == 0) {
          printf STDERR " A: pos to zero for name ..%s.. ?! \n", $name_str;
          exit(1); 
        }
        # okay, now we keep crawling until we strike a non-space
        for ($pos = $j; $pos >= 0; $pos--) {
          if (substr($name_str, $pos, 1) ne " ") {
            last;
          }
        }
        if ($pos == 0) {
          printf STDERR " B: pos to zero for name ..%s.. ?! \n", $name_str;
          exit(1); 
        }
        # finally, the player's name starts at position 0 in the $name_str
        #    and continues to "pos"
        $name_str = substr($name_str, 0, $pos - 2);
        if ($debug > 0) {
          printf " name_str is now ..%s.. \n", $name_str;
        }

        # replace spaces by underscores
        $name_str =~ s/\s/_/g;
        if ($debug > 0) {
          printf " name_str is now ..%s.. \n", $name_str;
        }

        $fixed_name = 1;

        last;
      }
    }

    # check to make sure that we have changed the name
    if ($fixed_name == 0) {
      printf STDERR " could not fix name ..%s.. ?! \n", $name_str;
      exit(1); 
    }

    # now do something with the fixed name
    $found = 0;
    for ($j = 0; $j < $num_player; $j++) {
      if ($name_array[$j] eq $name_str) {
        $player_index = $j;
        $found = 1;
        if ($debug > 0) {
          printf "  found existing player ..%s.. index %3d \n", 
                   $name_str, $player_index;
        }
      }
    }
    if ($found == 0) {
      # must add the player to our array of player names
      $name_array[$num_player++] = $name_str;
      $player_index = $num_player - 1;
      if ($debug > 0) {
        printf "  added new      player ..%s.. index %3d \n", 
                  $name_str, $player_index;
      }
    }
      
 
    #  save other stats now ....
    #     these will go into 2-D arrays, with names like
    #
    #            $hit_array[player_index][game_index]
    #
    # at-bats
    $stat_str = substr($line_array[$i], 29, 23);
    if ($debug > 0) {
      printf "  stat_str is ..%s.. \n", $stat_str;
    }  
    @words = split(/\s+/, $stat_str);
    $abs = $words[1];
    $runs = $words[2];
    $hits = $words[3];
    $rbis = $words[4];
    $bbs = $words[5];
    $sos = $words[6];
    if ($debug > 0) {
      printf " ab %2d hits %2d sos %2d \n", $abs, $hits, $sos;
    }
 
    $ab_array[$player_index][$game_index] = $abs;
    $run_array[$player_index][$game_index] = $runs;
    $hit_array[$player_index][$game_index] = $hits;
    $rbi_array[$player_index][$game_index] = $rbis;
    $bb_array[$player_index][$game_index] = $bbs;
    $so_array[$player_index][$game_index] = $sos;

  }
     

  return(0);
}



##############################################################################
# PROCEDURE: season_streak
#
# DESCRIPTION: We have one season's worth of data for all players
#              on the team.  Now we can do an analysis, player-by-player,
#              of some sort.
#
#              Note that a hitting streak consists of games in which
#              a player collected at least one hit.  It's okay if he
#              did not play in a game -- only games he plays count.
#              Games in which he does not have an official at-bat
#              (due to walks, sacrifice bunt) do NOT break the streak.
#
#              So, I'll look for games which which 
#                    number of AB > 0
#              and use those to determine streaks
#
# RETURNS:
#              0         if all goes well
#              1         if an error occurs
#
#
sub season_streak {

  my($cmd, $ret);
  my($line, @words);
  my($i, $j, $k);
  my($player_index, $game_i);


  if ($debug > 0) {
    printf " about to enter season_streak \n";
  }

  for ($player_index = 0; $player_index < $num_player; $player_index++) {
    
    # make a histogram of hitting streak lengths
    ($max_length, @num_streaks) = calc_streaks($player_index);
    

  }


}


##############################################################################
# PROCEDURE: calc_streaks
#
# DESCRIPTION: Given a player's index, find all the hitting streaks 
#              he had during the season.
#              Make a histogram of the streaks, so that we end up with
#              
#                 - max length
#                 - num_streaks[i]     i = 1, 2, ..., max_length
#
# RETURNS:
#              0         if all goes well
#              1         if an error occurs
#
#
sub calc_streaks {

  my($cmd, $ret);
  my($line, @words);
  my($i, $j, $k);
  my($player_index, $game_i);

  $player_index = $_[0];

  if ($debug > 0) {
    printf " about to enter calc_streaks \n";
  }

  # clear out the variables we'll use
  $in_streak = 0;
  $cur_length = 0;
  $max_length = 0;
  for ($i = 0; $i < 163; $i++) {
    $num_streaks[$i] = 0;
  }

  # debugging info
  if ($debug > 0) {
    printf "  calc_streaks  player ..%s.. \n", $name_array[$player_index];
  }

  for ($game_i = 0; $game_i < $num_games; $game_i++) {
    if ($debug > 0) {
      printf " game %3d  ab %2d hit %2d \n",
             $game_i, $ab_array[$player_index][$game_i],
             $hit_array[$player_index][$game_i];
    }
    if ($ab_array[$player_index][$game_i] > 0) {
      if ($hit_array[$player_index][$game_i] == 0) {
        # any streak ends
        if ($in_streak == 1) {
          $in_streak = 0;
          if ($cur_length > $max_length) {
            $max_length = $cur_length;
          }
          $num_streaks[$cur_length]++;
          $cur_length = 0;
          if ($debug > 0) {
            printf "  streak ends at %4d \n", $cur_length; 
          }
        }          

      }
      else {
        # any streak continues -- or start a new streak
        if ($in_streak == 0) {
          $in_streak = 1;
          if ($debug > 0) {
            printf "  new streak starts at %3d \n", 1;
          }
        }
        else {
        $cur_length++;
          if ($debug > 0) {
            printf "      streak cont   at %3d \n", $cur_length;
          }
        }
      }
    }
    else {
      # player did not have official AB, so ignore the game
      if ($debug > 0) {
        printf "    ignore game \n";
      }
    }

  }
  # end any final streak which was on-going at end of season
  if ($in_streak == 1) {
    $in_streak = 0;
    if ($cur_length > $max_length) {
      $max_length = $cur_length;
    }
    $num_streaks[$cur_length]++;
    $cur_length = 0;
  }          

  if ($debug > 0) {
    for ($i = 1; $i <= $max_length; $i++) {
      printf " there were %5d streaks of length %5d \n",
             $num_streaks[$i], $i;
    }
  }

if ($player_index > 3) {
exit 0;
}



}
