| Index: trunk/extensions/ExternalData/ED_ParserFunctions.php |
| — | — | @@ -10,7 +10,8 @@ |
| 11 | 11 | * @author Yaron Koren |
| 12 | 12 | */ |
| 13 | 13 | class EDParserFunctions { |
| 14 | | - |
| | 14 | + //how many times to try an http request: |
| | 15 | + private $http_number_of_tries=3; |
| 15 | 16 | // XML-handling functions based on code found at |
| 16 | 17 | // http://us.php.net/xml_set_element_handler |
| 17 | 18 | static function startElement( $parser, $name, $attrs ) { |
| — | — | @@ -92,7 +93,9 @@ |
| 93 | 94 | $params = func_get_args(); |
| 94 | 95 | array_shift( $params ); // we already know the $parser ... |
| 95 | 96 | $url = array_shift( $params ); |
| 96 | | - $url_contents = file_get_contents( $url ); |
| | 97 | + |
| | 98 | + $url_contents = EDParserFunctions::doRequest( $url ); |
| | 99 | + |
| 97 | 100 | $format = array_shift( $params ); |
| 98 | 101 | $external_values = array(); |
| 99 | 102 | if ($format == 'xml') { |
| — | — | @@ -126,4 +129,38 @@ |
| 127 | 130 | else |
| 128 | 131 | return ''; |
| 129 | 132 | } |
| | 133 | + |
| | 134 | + static function doRequest( $url, $post_vars = array(), $get_fresh=false, $try_count=1 ) { |
| | 135 | + $dbr = wfGetDB( DB_SLAVE ); |
| | 136 | + //do any special variable replace (right now just sunlight api key) |
| | 137 | + global $mvSunlightAPIKey; |
| | 138 | + $url = str_replace('$$SLAPIKEY', $mvSunlightAPIKey, $url ); |
| | 139 | + |
| | 140 | + // check the cache (only the first 254 chars of the url) |
| | 141 | + $res = $dbr->select( 'mv_url_cache', '*', array( 'url' => substr($url,0,254) ), 'EDParserFunctions::doRequest' ); |
| | 142 | + // @@todo check date |
| | 143 | + if ( $res->numRows() == 0 || $get_fresh) { |
| | 144 | + //echo "do web request: " . $url . "\n"; |
| | 145 | + $page = @file_get_contents( $url ); |
| | 146 | + if ( $page === false ) { |
| | 147 | + //echo( "error getting url retrying (".$try_count." of $this->http_number_of_tries)" ); |
| | 148 | + sleep( 1 ); |
| | 149 | + if( $try_count >= $this->http_number_of_tries ){ |
| | 150 | + echo "could not get url after $this->http_number_of_tries \n\n"; |
| | 151 | + return ''; |
| | 152 | + } |
| | 153 | + $try_count++; |
| | 154 | + return $this->doRequest( $url, $post_vars, $get_fresh, $try_count ); |
| | 155 | + } |
| | 156 | + if ( $page != '' ) { |
| | 157 | + $dbw = wfGetDB( DB_MASTER ); |
| | 158 | + // insert back into the db: |
| | 159 | + $dbw->insert( 'mv_url_cache', array( 'url' => substr($url,0,254), 'result' => $page, 'req_time' => time() ) ); |
| | 160 | + return $page; |
| | 161 | + } |
| | 162 | + } else { |
| | 163 | + $row = $dbr->fetchObject( $res ); |
| | 164 | + return $row->result; |
| | 165 | + } |
| | 166 | + } |
| 130 | 167 | } |
| Index: trunk/extensions/ExternalData/ExternalData.sql |
| — | — | @@ -0,0 +1,9 @@ |
| | 2 | +CREATE TABLE IF NOT EXISTS `mv_url_cache` ( |
| | 3 | + `id` int(10) unsigned NOT NULL auto_increment, |
| | 4 | + `url` varchar(255) NOT NULL, |
| | 5 | + `post_vars` text, |
| | 6 | + `req_time` int(11) NOT NULL, |
| | 7 | + `result` longtext character set utf8 collate utf8_unicode_ci, |
| | 8 | + UNIQUE KEY `id` (`id`), |
| | 9 | + KEY `url` (`url`) |
| | 10 | +) ENGINE=MyISAM DEFAULT CHARSET=latin1; |
| \ No newline at end of file |
| Index: trunk/extensions/ExternalData/README |
| — | — | @@ -2,6 +2,7 @@ |
| 3 | 3 | |
| 4 | 4 | Version 0.2 |
| 5 | 5 | Yaron Koren |
| | 6 | + Michael Dale (quick add of url cache) |
| 6 | 7 | |
| 7 | 8 | This is free software licensed under the GNU General Public License. Please |
| 8 | 9 | see http://www.gnu.org/copyleft/gpl.html for further details, including the |