$cache_dir_name = dirname(__FILE__) . DIRECTORY_SEPARATOR . '..'

    . DIRECTORY_SEPARATOR . 'cache' . DIRECTORY_SEPARATOR;

  $cache_auto_dir_name = $cache_dir_name . 'auto' . DIRECTORY_SEPARATOR;

  $cache_auto_school_dir_name = $cache_auto_dir_name . $school['id'] . DIRECTORY_SEPARATOR;

  if (!is_dir($cache_auto_school_dir_name))

    {

      if (!mkdir($cache_auto_school_dir_name, 0755, TRUE))

	error_log('Unable to create needed directory: `' . $cache_auto_dir_name . '\'');

    }

  $school_crawl_semesters_list_func = $school['id'] . '_crawl_semester_list';

  if (!function_exists($school_crawl_semesters_list_func))

    school_crawl_logf($school_crawl_log, 2, "Calling %s()...", $school_crawl_semesters_list_func);

  $ret = $school_crawl_semesters_list_func($school, $semesters, $school_crawl_log);

			$school['id'], $school_crawl_semesters_list_func);

  $successful_semesters = array();

  $school_crawl_semester_func = $school['id'] . '_crawl_semester';

  if (!function_exists($school_crawl_semester_func))

    {

      school_crawl_logf($school_crawl_log, 3, "%s() is defined but %s() isn't.",

			$school_crawl_semesters_list_func, $school_crawl_semester_func);

      return;

    }

  foreach ($semesters as $semester)

    {

      school_crawl_logf($school_crawl_log, 2, "Calling %s(%s)...", $school_crawl_semester_func, $semester);

      $ret = $school_crawl_semester_func($school, $semester, $school_crawl_log);

      if ($ret)

	{

	  school_crawl_logf($school_crawl_log, 1, "Failed to crawl semester %s. Skipping semester.", $semester);

	  continue;

	}

      /*

       * Write out this semester's cache now that we're here.

       */

      $cache_auto_school_semester_dir_name = $cache_auto_school_dir_name . $semester->id() . DIRECTORY_SEPARATOR;

	      if (!is_dir($cache_auto_school_semester_dir_name))

		{

		  if (!mkdir($cache_auto_school_semester_dir_name, 0755, TRUE))

		    error_log('Unable to create needed directory: `' . $cache_auto_school_semester_dir_name . '\'');

		}

	      $departments = $semester->departments_get();

	      sort($departments);

	      $dept_file = fopen($cache_auto_school_semester_dir_name . '-depts', 'wb');

	      fwrite($dept_file, serialize($departments));

	      fclose($dept_file);

	      /* now per-department autocomplete */

	      foreach ($departments as $department)

		{

		  $classes = $semester->department_classes_get($department);

		  $classes_file = fopen($cache_auto_school_semester_dir_name . $department . '.sects', 'wb');

		  fwrite($classes_file, serialize($classes));

		  fclose($classes_file);

		  /* now individual section informations, pre-JSON-ized */

		  foreach ($classes as $class)

		    {

		      if (!is_dir($cache_auto_school_semester_dir_name . $department))

			mkdir($cache_auto_school_semester_dir_name . $department);

		      $class_file = fopen($cache_auto_school_semester_dir_name . $department . DIRECTORY_SEPARATOR . $class, 'wb');

		      fwrite($class_file, json_encode($semester->class_get($department, $class)->to_json_array()));

		      fclose($class_file);

		    }

		}

      /* Purge the data written to disk from memory */

      $semester->purge();

      school_crawl_logf($school_crawl_log, 6, "");

      $successful_semesters[] = $semester;

    }

  $school['crawled_semesters'] = $successful_semesters;

	  fprintf(STDERR, "error: Invalid school_id specified for crawling: %s\n",

  /**

   * \brief

   *   Clean the semester of all sections, keeping metadata intact.

   */

  public function purge()

  {

    $this->departments = array();

  }

 *   Equivalent of gmmktime() except that it accepts strptime()'s

 *   output format as an input.

 * \param $timezone_offset

 *   Optional offset of the school's timezone in seconds from

 *   UTC. This offset gets _added_ to the resulting timestamp. So, for

 *   example, Eastern Daylight Time would use a value of 60*60 * -4

 *   since it is -0400 during Daylight time.

function school_crawl_gmmktime(array $tm, $timezone_offset = 0)

  return gmmktime($tm['tm_hour'], $tm['tm_min'], $tm['tm_sec'],

		$tm['tm_mon'] + 1, $tm['tm_mday'], $tm['tm_year'] + 1900)

    + $timezone_offset;

 *   Retrieve a list of crawlable semesters from Calvin College.

 * \param $school

 *   The calvin school handle.

 *   The array to populate with empty Semester objects.

 *   A school_crawl_log handle for informing the user/developer of

 *   progress.

function calvin_crawl_semester_list(array $school, array &$semesters, &$school_crawl_log)

  $season_map = array(

		      'FA' => Semester::SEASON_FALL,

		      'IN' => 'interim',

		      'SP' => Semester::SEASON_SPRING,

		      'MA' => 'may',

		      /* I don't know if SU is a valid Calvin Semester ID or not */

		      'SU' => Semester::SEASON_SUMMER);

  $uri = 'https://kvdata.calvin.edu/walive/WebAdvisor?CONSTITUENCY=WBST&type=P&pid=ST-WESTS12A&LASTTOKEN=NULL';

  $semesters_html = calvin_crawl_geturi($uri, $cookies, $school_crawl_log);

  $semesters_dom = new DOMDocument();

  $semesters_dom->loadHTML($semesters_html);

  $semesters_var1 = $semesters_dom->getElementById('VAR1');

  if (empty($semesters_var1))

    {

      school_crawl_logf($school_crawl_log, 0, "Error: Unable to load list of semesters.");

      return 1;

    }

  $semesters_select_nodes = $semesters_var1->childNodes;

      $semester_str = $semester_node->getAttribute('value');

			    $semester_str);

      $year = $year_timespec['tm_year'] + 1900;

      $semesters[$semester_str] = $semester;

    }

  $semester = array_reverse($semesters, TRUE);

  return 0;

}

/**

 * \brief

 *   Crawl the courses for a semester from Calvin College.

 *

 * \param $school

 *   The calvin school handle.

 * \param $semester

 *   The Semester object to populate with courses.

 * \param $school_crawl_log

 *   The logger handle.

 */

function calvin_crawl_semester(array $school, Semester $semester, &$school_crawl_log)

{

  $cookies = array();

  $uri = 'https://kvdata.calvin.edu/walive/WebAdvisor?CONSTITUENCY=WBST&type=P&pid=ST-WESTS12A&LASTTOKEN=NULL';

  $html = calvin_crawl_geturi($uri, $cookies, $school_crawl_log);

  $seed_dom = new DOMDocument();

  $seed_dom->loadHTML($html);

  $return_url = dom_input_value($seed_dom, 'RETURN.URL');

  $semester_str = sprintf("%02d/%s", $semester->year_get() % 100, strtoupper(substr($semester->season, 0, 2)));

  school_crawl_logf($school_crawl_log, 6, 'Using %s for a semester string.',

		    $semester_str);

		'LIST.VAR1_1' => '',

		'LIST.VAR2_1' => '',

  $semester_start_min = 0;

  $semester_end_max = 0;

  $skipped_sections = array('incomplete meeting info' => 0, 'invalid meeting info format' => 0);

  $pages = array(1 => 0, 2 => 1);

	   * The same info below should be retrievable with 

	      $date_start_time = school_crawl_gmmktime($date_start_time, -5 * 60*60);

		{

		  school_crawl_logf($school_crawl_log, 1, "Using section %s for the minimum start time.", $section_id['department'] . '-' . $section_id['course'] . '-' . $section_id['section']);

		  $semester_start_min = $date_start_time;

		}

	      $date_end_time = school_crawl_gmmktime($date_end_time, -5 * 60*60);

  $semester->time_end_set($semester_end_max);

  $semester->time_start_set($semester_start_min);

  /*

   * Calculate lab-based course dependencies.

   */

  school_crawl_logf($school_crawl_log, 7, 'Adding implicit lab dependencies.');

  foreach ($semester->departments_get() as $department)

    foreach ($semester->department_classes_get($department) as $course)

    {

      $the_course = $semester->class_get($department, $course);

      $lab_course = $semester->class_get($department, $course . 'L');

      if (!empty($lab_course))

	{

	  $the_course->dependency_add($lab_course);

	  school_crawl_logf($school_crawl_log, 8, "Adding dependency of %s-%s for %s-%s.",

			    $department, $course . 'L', $department, $course);

define('CCBCMD_CRAWL_URI', 'http://ccbcmd.edu/schedule/sched.html');

/**

 * \brief

 *   Obtain list of crawlable semesters offered by CCBCMD.

 *

 * \parram $school

 *   The CCBCMD school handle.

 * \param $semesters

 *   Array to populate with available semesters.

 * \return

 *   0 on success.

 */

function ccbcmd_crawl_semester_list(array $school, array &$semesters, &$school_crawl_log)

{

  $cookies = array();

  /*

   * It seems that http://ccbcmd.edu/schedule/sched.html is what we're

   * meant to start from. That's just a redirect to some other page

   * from which we get a listing of available semesters and choose

   * one.

   */

  $uri = CCBCMD_CRAWL_URI;

  $semesters_dom = new DOMDocument();

  $semesters_dom->loadHTML(school_crawl_geturi($uri, $cookies, $school_crawl_log, NULL, TRUE, 'ccbcmd_crawl_curlhook'));

  $semesters_select_node = $semesters_dom->getElementById('term_input_id');

  if ($semesters_select_node === NULL)

    {

      school_crawl_logf($school_crawl_log, 0, "Could not get list of available semesters to choose from.");

      return 1;

    }

  foreach ($semesters_select_node->childNodes as $semesters_option_node)

    {

      $semester_text = $semesters_option_node->textContent;

      $semester_value = $semesters_option_node->getAttribute('value');

      if (empty($semester_value))

	/* skip the empty ``None'' semester */

	continue;

      if (stripos($semester_text, 'continuing') !== FALSE)

	/* skip the year-long semesters dedicated to continuing education */

	continue;

      list($semester_season, $semester_year) = explode(' ', $semester_text);

      /* the college has two separate summer sessions, so distinguish between them */

      if (preg_match(';session ([0-9]+);i', $semester_text, $matches))

	$semester_season .= '_' . $matches[1];

      $semesters[] = new Semester($semester_year, strtolower($semester_season));

    }

  return 0;

}

/**

 * \brief

 *   Crawl a CCBCMD semester.

 *

 * \param $school

 *   The CCBCMD school handle.

 * \param $semester

 *   The semester to fill with courses.

 */

function ccbcmd_crawl_semester($school, $semester, &$school_crawl_log)

{

  $cookies = array();

  $uri = CCBCMD_CRAWL_URI;

  $semesters_dom = new DOMDocument();

  $semesters_dom->loadHTML(school_crawl_geturi($uri, $cookies, $school_crawl_log, NULL, TRUE, 'ccbcmd_crawl_curlhook'));

  $semesters_select_node = $semesters_dom->getElementById('term_input_id');

  if (empty($semesters_select_node))

    {

      school_crawl_logf($school_crawl_log, 0, "Could not locate the list of semesters from which to choose.");

      return 1;

    }

  $semesters_form = school_crawl_element_ancestor($semesters_select_node, 'form');

  if ($semesters_form === NULL)

    {

      school_crawl_logf($school_crawl_log, 0, "Unable to find <form /> associated with semester.");

      return 1;

    }

  $semesters_post = school_crawl_form($semesters_form);

  $semester_found = FALSE;

  foreach ($semesters_select_node->childNodes as $semesters_option_node)

    {

      $semester_text = $semesters_option_node->textContent;

      $semester_value = $semesters_option_node->getAttribute('value');

      if (empty($semester_value))

	continue;

      list($semester_season, $semester_year) = explode(' ', $semester_text);

      if (preg_match(';session ([0-9]+);i', $semester_text, $matches))

	$semester_season .= '_' . $matches[1];

      $semester_season = strtolower($semester_season);

      if ($semester_year == $semester->year_get()

	  && $semester_season == $semester->season_get())

	{

	  $semester_found = TRUE;

	  break;

	}

    }

  if (!$semester_found)

    {

      school_crawl_logf($school_crawl_log, 1, "Unable to find the entry for semester %s.", $semester);

      return 1;

    }

  $semesters_post[$semesters_select_node->getAttribute('name')] = $semester_value;

  $subjects_dom = new DOMDocument();

  $uri = school_crawl_url($uri, $semesters_form->getAttribute('action'));

  $subjects_dom->loadHTML(school_crawl_geturi($uri, $cookies, $school_crawl_log, $semesters_post, TRUE, 'ccbcmd_crawl_curlhook'));

  $subjects_form_nodelist = $subjects_dom->getElementsByTagName('form');

  if (!$subjects_form_nodelist->length)

    {

      school_crawl_logf($school_crawl_log, 0, "Unable to find <form /> to submit for the subjects-choosing page.");

      return 1;

    }

  $subjects_form_node = $subjects_form_nodelist->item(0);

  $subjects_post = school_crawl_form($subjects_form_node);

  $subjects_select_node = $subjects_dom->getElementById('subj_id');

  foreach ($subjects_select_node->childNodes as $subjects_option_node)

    if (!strcasecmp('all', trim($subjects_option_node->textContent)))

      $subjects_post[$subjects_select_node->getAttribute('name')][] = $subjects_option_node->getAttribute('value');

  $courses_dom = new DOMDocument();

  $uri = school_crawl_url($uri, $subjects_form_node->getAttribute('action'));

  $courses_dom->loadHTML(school_crawl_geturi($uri, $cookies, $school_crawl_log, $subjects_post, TRUE, 'ccbcmd_crawl_curlhook'));

  $courses_xpath = new DOMXPath($courses_dom);

  /* The second row of the table has all of the headers in it */

  $tr_header_nodelist = $courses_xpath->query('//table[@class="datadisplaytable" and position()=1]//tr[position()=2]');

  if (!$tr_header_nodelist->length)

    {

      school_crawl_logf($school_crawl_log, 0, "Unable to find the row of the course/section data table which gives us the mappings of column names onto columns.");

      return 1;

    }

  $tr_header_node = $tr_header_nodelist->item(0);

  $section_offsets = array(

			   'registration_number' => school_crawl_table_resolve_column($tr_header_node, 'CRN'),

			   'section_id' => school_crawl_table_resolve_column($tr_header_node, 'subj/crse/sec'),

			   /* there's a boolean column which says whether or not the course has any prerequisites/corequisites.... */

			   'credits' => school_crawl_table_resolve_column($tr_header_node, 'credhrs'),

			   /* there's a column for the number of contact hours, vs. credit hours */

			   'dates' => school_crawl_table_resolve_column($tr_header_node, 'sessiondates'),

			   );

  foreach (array('title', 'days', 'times', 'instructor', 'location') as $column_key)

    $section_offsets[$column_key] = school_crawl_table_resolve_column($tr_header_node, $column_key);

  /* there's also a column for ``session dates'' */

  /* error check and calculate the number of children that a node must have to be  */

  $max_offset = 0;

  foreach ($section_offsets as $name => $value)

    {

      if ($value === FALSE)

	{

	  school_crawl_logf($school_crawl_log, 0, "Unable to find column offset for `%s'.",

		  $name);

	  return 1;

	}

      else

	school_crawl_logf($school_crawl_log, 9, "%s -> %s", $name, $value);

      $max_offset = max($max_offset, $value);

    }

  foreach ($courses_xpath->query('//table[@class="datadisplaytable" and position()=1]//tr') as $tr_node)

    {

      $children = school_crawl_table_rownodes($tr_node);

      if ($children->length < $max_offset)

	/*

	 * Skip this row because it doesn't have all of the columns we

	 * want and thus it can't be a row containing information

	 * about a section.

	 */

	continue;

      if (!strcmp($children->item($section_offsets['section_id'])->tagName, 'th'))

	/*

	 * We've hit one of the <tr/>s filled with <th/>s. Skip this one.

	 */

	continue;

      /*

       * There are some rows with the time set to TBA and with empty

       * section_id columns. Respond to this by skipping empty

       * section_id columns since there's no useful data in these

       * rows. We use strlen() < 3 because trim() doesn't take care of

       *   :-/

       */

      $section_id = trim($children->item($section_offsets['section_id'])->textContent);

      if (strlen($section_id) < 3)

	continue;

      $section_id_parts = Section::parse($section_id);

      $registration_number = $children->item($section_offsets['registration_number'])->textContent;

      $instructor = $children->item($section_offsets['instructor'])->textContent;

      $section_meetings = array();

      {

	$time_range_text = $children->item($section_offsets['times'])->textContent;

	if (strpos($time_range_text, 'TBA') !== FALSE)

	  {

	    /*

	     * Add the section to the autocomplete list, just without

	     * any meeting info (i.e., $section_meetings is still

	     * empty now).

	     */

	    $semester->section_add($section_id_parts['department'], $section_id_parts['course'],

				   new Section($section_id_parts['section'], $section_meetings, $registration_number));

	    continue;

	  }

	if (($dash_pos = strpos($time_range_text, '-')) === FALSE)

	  {

	    school_crawl_logf($school_crawl_log, 0, "Unable to understand course's time range format, cannot find dash: ``%s''.",

		    $time_range_text);

	    return 1;

	  }

	$time_start_text = substr($time_range_text, 0, $dash_pos);

	$time_start = strptime($time_start_text, '%I:%M %p');

	$time_end_text = substr($time_range_text, $dash_pos + 1);

	/*

	 * Make sure that _only_ one date range is specified to ensure

	 * data integrity. I.e., make sure that the college doesn't

	 * suddenly support multiple meeting times without our

	 * anticipating that and then cause us to have invalid

	 * data. ;-). --binki

	 */

	if (strpos($time_end_text, '-') !== FALSE)

	  {

	    school_crawl_logf($school_crawl_log, 0, "College seems to support multiple meeting times per semester which we don't know how to parse (even though slate_permutate itself can handle this situation): ``%s'' time_end_text: ``%s''.",

		    $time_range_text, $time_end_text);

	    return 1;

	  }

	$time_end = strptime($time_end_text, '%I:%M %p');

	if ($time_end === FALSE || $time_start === FALSE)

	  {

	    school_crawl_logf($school_crawl_log, 0, "Error parsing start or end time: start: ``%s'' end: ``%s''.",

		    $time_start_text, $time_end_text);

	    return 1;

	  }

	$days = school_crawl_days_str_format($school_crawl_log, $children->item($section_offsets['days'])->textContent);

	$section_meetings[] = new SectionMeeting($days, school_crawl_time_format($time_start), school_crawl_time_format($time_end),

						 $children->item($section_offsets['location'])->textContent,

						 $instructor);

	/* check if a semester's date range should be increased */

	$section_dates = $children->item($section_offsets['dates'])->textContent;

	if (preg_match(';^([0-9]+)/([0-9]+)-([0-9]+)/([0-9]+)$;', $section_dates, $section_dates_matches))

	  {

	    $semester->time_start_set_test(gmmktime(0, 0, 0, $section_dates_matches[1], $section_dates_matches[2], $semester->year_get()));

	    $semester->time_end_set_test(gmmktime(0, 0, 0, $section_dates_matches[3], $section_dates_matches[4], $semester->year_get()));

	  }

      }

      $semester->section_add($section_id_parts['department'], $section_id_parts['course'],

			     new Section($section_id_parts['section'], $section_meetings, $registration_number));

    }

  return 0;

}

define('CEDARVILLE_BASE_URI', 'http://cedarville.edu/courses/schedule/');

define('CEDARVILLE_TIMEZONE_OFFSET', 60*60 * -4);

/**

 * \brief

 *   Obtain the list of crawlable semesters offered by Cedarville.

 *

 * \param $school

 *   The school's info array/handle.

 * \param $semesters

 *   An array to insert the semesters into.

 * \return

 *   0 on success.

 */

function cedarville_crawl_semester_list(array $school, array &$semesters, &$school_crawl_log)

{

  $uri = CEDARVILLE_BASE_URI;

  $cookies = array();

  $html = school_crawl_geturi($uri, $cookies, $school_crawl_log);

  if (empty($html))

      school_crawl_logf($school_crawl_log, 1, "Unable to fetch %s.", CEDARVILLE_BASE_URI);

  $semesters_dom = new DOMDocument();

  $semesters_dom->loadHTML($html);

  $have_semesters = FALSE;

  foreach ($departments_xpath->query('//*[@id="contenttext"]//li/a') as $department_a_dom)