diff --git a/school.d/cedarville.crawl.inc b/school.d/cedarville.crawl.inc new file mode 100644 --- /dev/null +++ b/school.d/cedarville.crawl.inc @@ -0,0 +1,262 @@ +. + */ + +/** + * \file + * \brief + * Crawler implementation for Cedarville University. + */ + +/** + * \brief + * Parse given html into an array, first row is row headers. + * + * \param $html + * HTML that PHP's DOM would willingly would eat. + */ +function table_parse($html) +{ + libxml_use_internal_errors(true); // Suppress warnings + $arr = array(); + $dom = new DOMDocument; + if(!$html) + return NULL; + + $dom->loadHTML($html); + $dom->preserveWhiteSpace = FALSE; + $tables = $dom->getElementsByTagName('table'); + $rows = $tables->item(0)->getElementsByTagName('tr'); // Get first table on page + foreach ($rows as $rownum => $row) { + $cols = $row->getElementsByTagName('td'); + foreach($cols as $colnum => $col){ + $arr[$rownum][$colnum] = $col->nodeValue; + } + } + return $arr; +} + +/** Crawls Cedarville course listings. $season is "fa" or "sp", year is 4-digit year */ +function cedarville_crawl($semester, $verbosity = 1) +{ + + $season = strtolower(substr($semester->season_get(), 0, 2)); + $year = $semester->year_get(); + $season_string = $year . $season; + + $basepath = 'http://cedarville.edu/courses/schedule/'; + + if ($verbosity) + echo "cedarville_crawl(): Beginning crawl of Cedarville:\n"; + + if ($verbosity > 1) + echo "cedarville_crawl(): Determining list of departments.\n"; + /* + * We need two passes because the first department's code name is + * not accessible available in the first pageload. + */ + $departments = array(); + if (cedarville_crawl_departments_get($basepath . $year . $season . '_index.htm', $departments, $season_string)) + return 1; + if (!count($departments)) + { + echo "cedarville_crawl(): Unable to get a listing of departments.\n"; + return 1; + } + /* find the first department whose name we don't yet know */ + if (cedarville_crawl_departments_get($basepath . $year . $season . '_' . current(array_keys($departments)) . '_all.htm', $departments, $season_string)) + return 1; + + $tables = array(); + foreach ($departments as $department => $dept_name) + { + echo 'cedarville_crawl(): Crawling department ' . $department . ' (' . $dept_name . ")...\n"; + $html = file_get_contents($basepath . $year . $season . '_' . $department . '_' . 'all.htm'); + if (!$html) + continue; + $tables[$department] = table_parse(cedarville_html_fix($html)); + } + + $meeting_type_maps = array('LAB' => 'lab', 'LECT' => 'lecture'); + + foreach ($tables as $dept_table) + { + /* + * Discard the first row, which has the contents of the + * elements. + */ + unset($dept_table[0]); + + foreach($dept_table as $course_table) + { + /* + * format: + * 0: course synonym, an unsigned integer. + * 1: section spec, parsable by Section::parse(). + * 2: friendly course title. + * 3: Instructor name. + * 4: Number of credit hours in decimal notation. + * 5: Fee. + * 6: Meeting time, explained below. + * 7: Cap. + * 8-10: Textbook link. Most rows only have column 8, not + * all the way through 10. This information seems + * quite useless. + * + * Section meeting time/place format: + * + * Confusing example: ' ILB WI219 TR 08:30A-09:45A' + * Complete example plus lab: ' LEC TYL203 MWF 08:00A-08:50A LAB ENS118 TR 03:00P-04:30P' + * + * Appears to have format: + * : - + * + * It appears tht may be: + * LEC: normal lecture meeting. + * ONL: online course. + * ILB: ethan says a partially online course...? + * HYB: hybrid of...? + * FLD: field...? + * FE2: ? + * CLN: ? + * LAB: Lab + * LES: something for some PFMU/PLMU class? + */ + + $synonym = $course_table[0]; + $section_parts = Section::parse($course_table[1]); + if (count($section_parts) < 3) + { + error_log('Error parsing section_id. Given `' . $course_table[1] . '\', interpreted as `' + . implode('-', $section_parts) . '\'. Skipping.'); + continue; + } + + $instructor = $course_table[3]; + + /* + * Each course may have multiple meeting times associated + * with it at Cedarville. We are not sure how to handle this + * quite, because different class sections may be tied with + * different lab meetings and stuff... + */ + $meetings_str = $course_table[6]; + if (strpos($meetings_str, 'TBA') !== FALSE) + { + if ($verbosity > 1) + error_log('Skipping ' . implode('-', $section_parts) . ' because its meeting time info has `TBA\' in it.'); + continue; + } + $meetings = array(); + $meeting_multiple_types = array(); + while (strlen($meetings_str) > 5) + { + if (!preg_match(';^ ([A-Z]+) +([A-Z]+[A-Z0-9]*) +([MTWRF]{1,5}) +([0-9:AP]+)-([0-9:AP]+);', + $meetings_str, $meeting_matches)) + { + if (preg_match(';^Dates:[^0-9]+([/0-9]{8})-([/0-9]{8});', + $meetings_str, $meeting_matches)) + { + if ($verbosity > 4) + error_log('Skipping some meeting data for ' + . implode('-', $section_parts) . ' because it is a date range: `' + . $meeting_matches[0] . '\''); + $meetings_str = substr($meetings_str, strlen($meeting_matches[0])); + continue; + } + + if ($verbosity > 0) + error_log('Error parsing meeting time. Given `' . $meetings_str . '\'. Skipping ' + . implode('-', $section_parts)); + break; + } + /* prepare for parsing the next meeting time */ + $meetings_str = substr($meetings_str, strlen($meeting_matches[0])); + + $days = school_crawl_days_str_format($meeting_matches[3]); + $time_start = school_crawl_time_format(strptime($meeting_matches[4] . 'M', '%I:%M%p')); + $time_end = school_crawl_time_format(strptime($meeting_matches[5] . 'M', '%I:%M%p')); + $room = $meeting_matches[2]; + + $type = $meeting_matches[1]; + while (isset($meeting_type_maps[$type])) + $type = $meeting_type_maps[$type]; + $type = strtolower($type); + + $meetings[] = new SectionMeeting($days, $time_start, $time_end, + $room, $type); + } + + $semester->section_add($section_parts['department'], $section_parts['course'], + new Section($section_parts['section'], $meetings, + $synonym, $instructor)); + } + } + + return 0; +} + +/** + * \brief + * Scan cedarville's course listing pages for departments. + * + * \return + * An associative array mapping department codes onto department + * friendly names. + */ +function cedarville_crawl_departments_get($dept_url, array &$departments, $season_string) +{ + $html = file_get_contents($dept_url); + $dept_dom = new DOMDocument(); + if (!$dept_dom->loadHTML(cedarville_html_fix($html))) + { + echo "cedarville_crawl(): Error determining list of available departments: Unable to parse HTML.\n"; + return 1; + } + $xpath = new DOMXPath($dept_dom); + + $dept_node_list = $xpath->query('/descendant::div[@id="contenttext"]/child::span[position()=1 or position()=2]/child::a'); + foreach ($dept_node_list as $dept_node) + { + $href = $dept_node->getAttribute('href'); + if (!preg_match('/^' . preg_quote($season_string, '/') . '_([a-z]+)_[a-z]+\.htm$/', $href, $matches)) + { + echo 'cedarvillege_crawl(): Error determining list of available departments: Unable to parse the department string out of href="' . $href . "\".\n"; + return 1; + } + + $dept = $matches[1]; + $departments[$dept] = $dept_node->textContent; + } + + return 0; +} + +/** + * \brief + * Fix some incorrect usage of the HTML entity delimiter, the ampersand. + */ +function cedarville_html_fix($html) +{ + $html = preg_replace('/&&/', '&&', $html); + $html = preg_replace('/&([^;]{5})/', '&$1', $html); + $html = preg_replace('/ID="(LINKS|HERE)"/', '', $html); + + return $html; +}