r61078 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r61077‎ | r61078 | r61079 >
Date:05:56, 15 January 2010
Author:mah
Status:reverted (Comments)
Tags:
Comment:
follow-up r60811 clean up code, write some tests for the existing uses of HttpFunctions. Still need to test background download, proxy tests, etc.
Modified paths:
  • /trunk/phase3/includes/HttpFunctions.php (modified) (history)
  • /trunk/phase3/tests/HttpTest.php (added) (history)

Diff [purge]

Index: trunk/phase3/tests/HttpTest.php
@@ -0,0 +1,185 @@
 2+<?php
 3+
 4+class HttpTest extends PHPUnit_Framework_TestCase {
 5+ static $content;
 6+ static $headers;
 7+ var $test_geturl = array( "http://www.example.com/",
 8+ "http://pecl.php.net/feeds/pkg_apc.rss",
 9+ "http://toolserver.org/~jan/poll/dev/main.php?page=wiki_output&id=3",
 10+ "http://meta.wikimedia.org/w/index.php?title=Interwiki_map&action=raw",
 11+ "http://www.mediawiki.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:MediaWiki_hooks&cmlimit=500&format=php",
 12+ );
 13+ var $test_requesturl = array( "http://en.wikipedia.org/wiki/Special:Export/User:MarkAHershberger/Weekly_reports/2010-W01" );
 14+
 15+ var $test_posturl = array( "http://www.comp.leeds.ac.uk/cgi-bin/Perl/environment-example" => "review=test" );
 16+
 17+ function setup() {
 18+ if ( is_array( self::$content ) ) {
 19+ return;
 20+ }
 21+ $content = tempnam( sys_get_temp_dir(), "" );
 22+ $headers = tempnam( sys_get_temp_dir(), "" );
 23+ if ( !$content && !$headers ) {
 24+ die( "Couldn't create temp file!" );
 25+ }
 26+
 27+ /* Maybe use wget instead of curl here ... just to use a different codebase? */
 28+ foreach ( $this->test_geturl as $u ) {
 29+ system( "curl -0 -s -D $headers '$u' -o $content" );
 30+ self::$content["GET $u"] = file_get_contents( $content );
 31+ self::$headers["GET $u"] = file_get_contents( $headers );
 32+ }
 33+ foreach ( $this->test_requesturl as $u ) {
 34+ system( "curl -0 -s -X POST -H 'Content-Length: 0' -D $headers '$u' -o $content" );
 35+ self::$content["POST $u"] = file_get_contents( $content );
 36+ self::$headers["POST $u"] = file_get_contents( $headers );
 37+ }
 38+ foreach ( $this->test_posturl as $u => $postdata ) {
 39+ system( "curl -0 -s -X POST -d '$postdata' -D $headers '$u' -o $content" );
 40+ self::$content["POST $u => $postdata"] = file_get_contents( $content );
 41+ self::$headers["POST $u => $postdata"] = file_get_contents( $headers );
 42+ }
 43+ unlink( $content );
 44+ unlink( $headers );
 45+ }
 46+
 47+ /* ./phase3/includes/Import.php:1108: $data = Http::request( $method, $url ); */
 48+ /* ./includes/Import.php:1124: $link = Title::newFromText( "$interwiki:Special:Export/$page" ); */
 49+ /* ./includes/Import.php:1134: return ImportStreamSource::newFromURL( $url, "POST" ); */
 50+ function runHTTPRequests() {
 51+ global $wgForceHTTPEngine;
 52+
 53+ /* no postdata here because the only request I could find in code so far didn't have any */
 54+ foreach ( $this->test_requesturl as $u ) {
 55+ $r = Http::request( "POST", $u );
 56+ $this->assertEquals( self::$content["POST $u"], $r, "POST $u with $wgForceHTTPEngine" );
 57+ }
 58+ }
 59+
 60+ function testRequestPHP() {
 61+ global $wgForceHTTPEngine;
 62+
 63+ $wgForceHTTPEngine = "php";
 64+ self::runHTTPRequests();
 65+ }
 66+
 67+ function testRequestCurl() {
 68+ global $wgForceHTTPEngine;
 69+
 70+ $wgForceHTTPEngine = "curl";
 71+ self::runHTTPRequests();
 72+ }
 73+
 74+ /* ./extensions/SpamBlacklist/SpamBlacklist_body.php:164: $httpText = Http::get( $fileName ); */
 75+ /* ./extensions/ApiSVGProxy/ApiSVGProxy.body.php:44: $contents = Http::get( $file->getFullUrl() ); */
 76+ /* ./extensions/BookInformation/drivers/IsbnDb.php:24: if( ( $xml = Http::get( $uri ) ) !== false ) { */
 77+ /* ./extensions/BookInformation/drivers/Amazon.php:23: if( ( $xml = Http::get( $uri ) ) !== false ) { */
 78+ /* ./extensions/TitleBlacklist/TitleBlacklist.list.php:217: $result = Http::get( $url ); */
 79+ /* ./extensions/TSPoll/TSPoll.php:68: $get_server = Http::get( 'http://toolserver.org/~jan/poll/dev/main.php?page=wiki_output&id='.$id ); */
 80+ /* ./extensions/TSPoll/TSPoll.php:70: $get_server = Http::get( 'http://toolserver.org/~jan/poll/main.php?page=wiki_output&id='.$id ); */
 81+ /* ./extensions/DoubleWiki/DoubleWiki.php:56: $translation = Http::get( $url.$sep.'action=render' ); */
 82+ /* ./extensions/ExternalPages/ExternalPages_body.php:177: $serializedText = Http::get( $this->mPageURL ); */
 83+ /* ./extensions/Translate/utils/TranslationHelpers.php:143: $suggestions = Http::get( $url, $timeout ); */
 84+ /* ./extensions/Translate/SpecialImportTranslations.php:169: $filedata = Http::get( $url ); ; */
 85+ /* ./extensions/Translate/TranslateEditAddons.php:338: $suggestions = Http::get( $url, $timeout ); */
 86+ /* ./extensions/SecurePoll/includes/user/Auth.php:283: $value = Http::get( $url, 20, $curlParams ); */
 87+ /* ./extensions/DumpHTML/dumpHTML.inc:778: $contents = Http::get( $url ); */
 88+ /* ./extensions/DumpHTML/dumpHTML.inc:1298: $contents = Http::get( $sourceUrl ); */
 89+ /* ./extensions/DumpHTML/dumpHTML.inc:1373: $contents = Http::get( $sourceUrl ); */
 90+ /* ./phase3/maintenance/rebuildInterwiki.inc:101: $intermap = Http::get( 'http://meta.wikimedia.org/w/index.php?title=Interwiki_map&action=raw', 30 ); */
 91+ /* ./phase3/maintenance/findhooks.php:98: $allhookdata = Http::get( 'http://www.mediawiki.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:MediaWiki_hooks&cmlimit=500&format=php' ); */
 92+ /* ./phase3/maintenance/findhooks.php:109: $oldhookdata = Http::get( 'http://www.mediawiki.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Removed_hooks&cmlimit=500&format=php' ); */
 93+ /* ./phase3/maintenance/dumpInterwiki.inc:95: $intermap = Http::get( 'http://meta.wikimedia.org/w/index.php?title=Interwiki_map&action=raw', 30 ); */
 94+ /* ./phase3/includes/parser/Parser.php:3204: $text = Http::get($url); */
 95+ /* ./phase3/includes/filerepo/ForeignAPIRepo.php:131: $data = Http::get( $url ); */
 96+ /* ./phase3/includes/filerepo/ForeignAPIRepo.php:205: $thumb = Http::get( $foreignUrl ); */
 97+ /* ./phase3/includes/filerepo/File.php:1105: $res = Http::get( $renderUrl ); */
 98+ /* ./phase3/includes/GlobalFunctions.php:2760: * @deprecated Use Http::get() instead */
 99+ /* ./phase3/includes/GlobalFunctions.php:2764: return Http::get( $url ); */
 100+ /* ./phase3/includes/ExternalStoreHttp.php:18: $ret = Http::get( $url ); */
 101+ /* ./phase3/includes/Import.php:357: $data = Http::get( $src ); */
 102+ /* ./extensions/ExternalData/ED_Utils.php:291: return Http::get( $url, 'default', array(CURLOPT_SSL_VERIFYPEER => false) ); */
 103+ /* ./extensions/ExternalData/ED_Utils.php:293: return Http::get( $url ); */
 104+ /* ./extensions/ExternalData/ED_Utils.php:306: $page = Http::get( $url, 'default', array(CURLOPT_SSL_VERIFYPEER => false) ); */
 105+ /* ./extensions/ExternalData/ED_Utils.php:308: $page = Http::get( $url ); */
 106+ /* ./extensions/CodeReview/backend/Subversion.php:320: $blob = Http::get( $target, $this->mTimeout ); */
 107+ /* ./extensions/AmazonPlus/AmazonPlus.php:214: $this->response = Http::get( $urlstr ); */
 108+ /* ./extensions/StaticWiki/StaticWiki.php:24: $text = Http::get( $url ) ; */
 109+ /* ./extensions/StaticWiki/StaticWiki.php:64: $history = Http::get ( $wgStaticWikiExternalSite . "index.php?title=" . urlencode ( $url_title ) . "&action=history" ) ; */
 110+ /* ./extensions/Configure/scripts/findSettings.php:126: $cont = Http::get( "http://www.mediawiki.org/w/index.php?title={$page}&action=raw" ); */
 111+ /* ./extensions/TorBlock/TorBlock.class.php:148: $data = Http::get( $url ); */
 112+ /* ./extensions/HoneypotIntegration/HoneypotIntegration.class.php:60: $data = Http::get( $wgHoneypotURLSource, 'default', */
 113+ /* ./extensions/SemanticForms/includes/SF_Utils.inc:378: $page_contents = Http::get($url); */
 114+ /* ./extensions/LocalisationUpdate/LocalisationUpdate.class.php:172: $basefilecontents = Http::get( $basefile ); */
 115+ /* ./extensions/APC/SpecialAPC.php:245: $rss = Http::get( 'http://pecl.php.net/feeds/pkg_apc.rss' ); */
 116+ /* ./extensions/Interlanguage/Interlanguage.php:56: $a = Http::get( $url ); */
 117+ /* ./extensions/MWSearch/MWSearch_body.php:492: $data = Http::get( $searchUrl, $wgLuceneSearchTimeout, $httpOpts); */
 118+ function runHTTPGets() {
 119+ global $wgForceHTTPEngine;
 120+
 121+ foreach ( $this->test_geturl as $u ) {
 122+ $r = Http::get( $u );
 123+ $this->assertEquals( self::$content["GET $u"], $r, "Get $u with $wgForceHTTPEngine" );
 124+ }
 125+ }
 126+
 127+ function testGetPHP() {
 128+ global $wgForceHTTPEngine;
 129+
 130+ $wgForceHTTPEngine = "php";
 131+ self::runHTTPGets();
 132+ }
 133+
 134+ function testGetCurl() {
 135+ global $wgForceHTTPEngine;
 136+
 137+ $wgForceHTTPEngine = "curl";
 138+ self::runHTTPGets();
 139+ }
 140+
 141+ /* ./phase3/maintenance/parserTests.inc:1618: return Http::post( $url, array( 'postdata' => wfArrayToCGI( $data ) ) ); */
 142+ function runHTTPPosts() {
 143+ global $wgForceHTTPEngine;
 144+
 145+ foreach ( $this->test_posturl as $u => $postdata ) {
 146+ $r = Http::post( $u, array( "postdata" => $postdata ) );
 147+ $this->assertEquals( self::$content["POST $u => $postdata"], $r, "POST $u (postdata=$postdata) with $wgForceHTTPEngine" );
 148+ }
 149+ }
 150+
 151+ function testPostPHP() {
 152+ global $wgForceHTTPEngine;
 153+
 154+ $wgForceHTTPEngine = "php";
 155+ self::runHTTPPosts();
 156+ }
 157+
 158+ function testPostCurl() {
 159+ global $wgForceHTTPEngine;
 160+
 161+ $wgForceHTTPEngine = "curl";
 162+ self::runHTTPPosts();
 163+ }
 164+
 165+ function testDoDownload() {
 166+ }
 167+
 168+ function testStartBackgroundDownload() {
 169+ }
 170+
 171+ function testGetUploadSessionKey() {
 172+ }
 173+
 174+ function testDoSessionIdDownload() {
 175+ }
 176+
 177+ function testIsLocalURL() {
 178+ }
 179+
 180+ /* ./extensions/DonationInterface/payflowpro_gateway/payflowpro_gateway.body.php:559: $user_agent = Http::userAgent(); */
 181+ function testUserAgent() {
 182+ }
 183+
 184+ function testIsValidURI() {
 185+ }
 186+}
Property changes on: trunk/phase3/tests/HttpTest.php
___________________________________________________________________
Name: svn:eol-syle
1187 + native
Index: trunk/phase3/includes/HttpFunctions.php
@@ -8,29 +8,23 @@
99 * @ingroup HTTP
1010 */
1111 class Http {
12 - // Syncronous download (in a single request)
13 - const SYNC_DOWNLOAD = 1;
14 -
15 - // Asynchronous download ( background process with multiple requests )
16 - const ASYNC_DOWNLOAD = 2;
17 -
1812 /**
19 - * Get the contents of a file by HTTP
 13+ * Perform an HTTP request
2014 * @param $method string HTTP method. Usually GET/POST
2115 * @param $url string Full URL to act on
22 - * @param $timeout int Seconds to timeout. 'default' falls to $wgHTTPTimeout
23 - * @param $curlOptions array Optional array of extra params to pass
24 - * to curl_setopt()
 16+ * @param $opts options to pass to HttpRequest object
 17+ * @returns mixed (bool)false on failure or a string on success
2518 */
2619 public static function request( $method, $url, $opts = array() ) {
27 - $opts['method'] = ( strtoupper( $method ) == 'GET' || strtoupper( $method ) == 'POST' )
28 - ? strtoupper( $method ) : null;
29 - $req = HttpRequest::newRequest( $url, $opts );
 20+ $opts['method'] = strtoupper( $method );
 21+ if ( !array_key_exists( 'timeout', $opts ) ) {
 22+ $opts['timeout'] = 'default';
 23+ }
 24+ $req = HttpRequest::factory( $url, $opts );
3025 $status = $req->doRequest();
31 - if( $status->isOK() ) {
32 - return $status->value;
 26+ if ( $status->isOK() ) {
 27+ return $req->getContent();
3328 } else {
34 - wfDebug( 'http error: ' . $status->getWikiText() );
3529 return false;
3630 }
3731 }
@@ -39,10 +33,8 @@
4034 * Simple wrapper for Http::request( 'GET' )
4135 * @see Http::request()
4236 */
43 - public static function get( $url, $timeout = false, $opts = array() ) {
44 - global $wgSyncHTTPTimeout;
45 - if( $timeout )
46 - $opts['timeout'] = $timeout;
 37+ public static function get( $url, $timeout = 'default', $opts = array() ) {
 38+ $opts['timeout'] = $timeout;
4739 return Http::request( 'GET', $url, $opts );
4840 }
4941
@@ -54,24 +46,90 @@
5547 return Http::request( 'POST', $url, $opts );
5648 }
5749
58 - public static function doDownload( $url, $target_file_path, $dl_mode = self::SYNC_DOWNLOAD,
59 - $redirectCount = 0 )
60 - {
 50+ /**
 51+ * Check if the URL can be served by localhost
 52+ * @param $url string Full url to check
 53+ * @return bool
 54+ */
 55+ public static function isLocalURL( $url ) {
 56+ global $wgCommandLineMode, $wgConf;
 57+ if ( $wgCommandLineMode ) {
 58+ return false;
 59+ }
 60+
 61+ // Extract host part
 62+ $matches = array();
 63+ if ( preg_match( '!^http://([\w.-]+)[/:].*$!', $url, $matches ) ) {
 64+ $host = $matches[1];
 65+ // Split up dotwise
 66+ $domainParts = explode( '.', $host );
 67+ // Check if this domain or any superdomain is listed in $wgConf as a local virtual host
 68+ $domainParts = array_reverse( $domainParts );
 69+ for ( $i = 0; $i < count( $domainParts ); $i++ ) {
 70+ $domainPart = $domainParts[$i];
 71+ if ( $i == 0 ) {
 72+ $domain = $domainPart;
 73+ } else {
 74+ $domain = $domainPart . '.' . $domain;
 75+ }
 76+ if ( $wgConf->isLocalVHost( $domain ) ) {
 77+ return true;
 78+ }
 79+ }
 80+ }
 81+ return false;
 82+ }
 83+
 84+ /**
 85+ * A standard user-agent we can use for external requests.
 86+ * @returns string
 87+ */
 88+ public static function userAgent() {
 89+ global $wgVersion;
 90+ return "MediaWiki/$wgVersion";
 91+ }
 92+
 93+ /**
 94+ * Checks that the given URI is a valid one
 95+ * @param $uri Mixed: URI to check for validity
 96+ * @returns bool
 97+ */
 98+ public static function isValidURI( $uri ) {
 99+ return preg_match(
 100+ '/(ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/',
 101+ $uri,
 102+ $matches
 103+ );
 104+ }
 105+ /**
 106+ * Fetch a URL, write the result to a file.
 107+ * @params $url string url to fetch
 108+ * @params $targetFilePath string full path (including filename) to write the file to
 109+ * @params $async bool whether the download should be asynchronous (defaults to false)
 110+ * @params $redirectCount int used internally to keep track of the number of redirects
 111+ *
 112+ * @returns Status -- for async requests this will contain the request key
 113+ */
 114+ public static function doDownload( $url, $targetFilePath, $async = false, $redirectCount = 0 ) {
61115 global $wgPhpCli, $wgMaxUploadSize, $wgMaxRedirects;
 116+
62117 // do a quick check to HEAD to insure the file size is not > $wgMaxUploadSize
63 - $headRequest = HttpRequest::newRequest( $url, array( 'headers_only' => true ) );
 118+ $headRequest = HttpRequest::factory( $url, array( 'headersOnly' => true ) );
64119 $headResponse = $headRequest->doRequest();
65 - if( !$headResponse->isOK() ) {
 120+ if ( !$headResponse->isOK() ) {
66121 return $headResponse;
67122 }
68123 $head = $headResponse->value;
69124
70125 // check for redirects:
71 - if( isset( $head['Location'] ) && strrpos( $head[0], '302' ) !== false ) {
72 - if( $redirectCount < $wgMaxRedirects ) {
73 - if( self::isValidURI( $head['Location'] ) ) {
74 - return self::doDownload( $head['Location'], $target_file_path,
75 - $dl_mode, $redirectCount++ );
 126+ if ( $redirectCount < 0 ) {
 127+ $redirectCount = 0;
 128+ }
 129+ if ( isset( $head['Location'] ) && strrpos( $head[0], '302' ) !== false ) {
 130+ if ( $redirectCount < $wgMaxRedirects ) {
 131+ if ( self::isValidURI( $head['Location'] ) ) {
 132+ return self::doDownload( $head['Location'], $targetFilePath,
 133+ $async, $redirectCount++ );
76134 } else {
77135 return Status::newFatal( 'upload-proto-error' );
78136 }
@@ -80,332 +138,355 @@
81139 }
82140 }
83141 // we did not get a 200 ok response:
84 - if( strrpos( $head[0], '200 OK' ) === false ) {
 142+ if ( strrpos( $head[0], '200 OK' ) === false ) {
85143 return Status::newFatal( 'upload-http-error', htmlspecialchars( $head[0] ) );
86144 }
87145
88 - $content_length = ( isset( $head['Content-Length'] ) ) ? $head['Content-Length'] : null;
89 - if( $content_length ) {
90 - if( $content_length > $wgMaxUploadSize ) {
91 - return Status::newFatal( 'requested file length ' . $content_length .
92 - ' is greater than $wgMaxUploadSize: ' . $wgMaxUploadSize );
 146+ $contentLength = $head['Content-Length'];
 147+ if ( $contentLength ) {
 148+ if ( $contentLength > $wgMaxUploadSize ) {
 149+ return Status::newFatal( 'requested file length ' . $contentLength .
 150+ ' is greater than $wgMaxUploadSize: ' . $wgMaxUploadSize );
93151 }
94152 }
95153
96154 // check if we can find phpCliPath (for doing a background shell request to
97155 // php to do the download:
98 - if( $wgPhpCli && wfShellExecEnabled() && $dl_mode == self::ASYNC_DOWNLOAD ) {
 156+ if ( $async && $wgPhpCli && wfShellExecEnabled() ) {
99157 wfDebug( __METHOD__ . "\nASYNC_DOWNLOAD\n" );
100 - //setup session and shell call:
101 - return self::initBackgroundDownload( $url, $target_file_path, $content_length );
 158+ // setup session and shell call:
 159+ return self::startBackgroundRequest( $url, $targetFilePath, $contentLength );
102160 } else {
103161 wfDebug( __METHOD__ . "\nSYNC_DOWNLOAD\n" );
104162 // SYNC_DOWNLOAD download as much as we can in the time we have to execute
105163 $opts['method'] = 'GET';
106 - $opts['target_file_path'] = $target_file_path;
107 - $req = HttpRequest::newRequest( $url, $opts );
 164+ $opts['targetFilePath'] = $mTargetFilePath;
 165+ $req = HttpRequest::factory( $url, $opts );
108166 return $req->doRequest();
109167 }
110168 }
111169
112170 /**
113 - * a non blocking request (generally an exit point in the application)
114 - * should write to a file location and give updates
 171+ * Start backgrounded (i.e. non blocking) request. The
 172+ * backgrounded request will provide updates to the user's session
 173+ * data.
 174+ * @param $url string the URL to download
 175+ * @param $targetFilePath string the destination for the downloaded file
 176+ * @param $contentLength int (optional) the length of the download from the HTTP header
115177 *
 178+ * @returns Status
116179 */
117 - private static function initBackgroundDownload( $url, $target_file_path,
118 - $content_length = null )
119 - {
 180+ private static function startBackgroundRequest( $url, $targetFilePath, $contentLength = null ) {
120181 global $IP, $wgPhpCli, $wgServer;
121182 $status = Status::newGood();
122183
123 - // generate a session id with all the details for the download (pid, target_file_path )
124 - $upload_session_key = self::getUploadSessionKey();
125 - $session_id = session_id();
 184+ // generate a session id with all the details for the download (pid, targetFilePath )
 185+ $requestKey = self::createRequestKey();
 186+ $sessionID = session_id();
126187
127188 // store the url and target path:
128 - $_SESSION['wsDownload'][$upload_session_key]['url'] = $url;
129 - $_SESSION['wsDownload'][$upload_session_key]['target_file_path'] = $target_file_path;
 189+ $_SESSION['wsBgRequest'][$requestKey]['url'] = $url;
 190+ $_SESSION['wsBgRequest'][$requestKey]['targetFilePath'] = $targetFilePath;
130191 // since we request from the cmd line we lose the original host name pass in the session:
131 - $_SESSION['wsDownload'][$upload_session_key]['orgServer'] = $wgServer;
 192+ $_SESSION['wsBgRequest'][$requestKey]['orgServer'] = $wgServer;
132193
133 - if( $content_length )
134 - $_SESSION['wsDownload'][$upload_session_key]['content_length'] = $content_length;
 194+ if ( $contentLength ) {
 195+ $_SESSION['wsBgRequest'][$requestKey]['contentLength'] = $contentLength;
 196+ }
135197
136198 // set initial loaded bytes:
137 - $_SESSION['wsDownload'][$upload_session_key]['loaded'] = 0;
 199+ $_SESSION['wsBgRequest'][$requestKey]['loaded'] = 0;
138200
139201 // run the background download request:
140 - $cmd = $wgPhpCli . ' ' . $IP . "/maintenance/http_session_download.php " .
141 - "--sid {$session_id} --usk {$upload_session_key} --wiki " . wfWikiId();
 202+ $cmd = $wgPhpCli . ' ' . $IP . "/maintenance/httpSessionDownload.php " .
 203+ "--sid {$sessionID} --usk {$requestKey} --wiki " . wfWikiId();
142204 $pid = wfShellBackgroundExec( $cmd );
143205 // the pid is not of much use since we won't be visiting this same apache any-time soon.
144 - if( !$pid )
145 - return Status::newFatal( 'could not run background shell exec' );
 206+ if ( !$pid )
 207+ return Status::newFatal( 'http-could-not-background' );
146208
147 - // update the status value with the $upload_session_key (for the user to
148 - // check on the status of the upload)
149 - $status->value = $upload_session_key;
 209+ // update the status value with the $requestKey (for the user to
 210+ // check on the status of the download)
 211+ $status->value = $requestKey;
150212
151213 // return good status
152214 return $status;
153215 }
154216
155 - static function getUploadSessionKey() {
156 - $key = mt_rand( 0, 0x7fffffff );
157 - $_SESSION['wsUploadData'][$key] = array();
 217+ /**
 218+ * Returns a unique, random string that can be used as an request key and
 219+ * preloads it into the session data.
 220+ *
 221+ * @returns string
 222+ */
 223+ static function createRequestKey() {
 224+ if ( !array_key_exists( 'wsBgRequest', $_SESSION ) ) {
 225+ $_SESSION['wsBgRequest'] = array();
 226+ }
 227+
 228+ $key = uniqid( 'bgrequest', true );
 229+
 230+ // This is probably over-defensive.
 231+ while ( array_key_exists( $key, $_SESSION['wsBgRequest'] ) ) {
 232+ $key = uniqid( 'bgrequest', true );
 233+ }
 234+ $_SESSION['wsBgRequest'][$key] = array();
 235+
158236 return $key;
159237 }
160238
161239 /**
162 - * used to run a session based download. Is initiated via the shell.
 240+ * Recover the necessary session and request information
 241+ * @param $sessionID string
 242+ * @param $requestKey string the HTTP request key
163243 *
164 - * @param $session_id String: the session id to grab download details from
165 - * @param $upload_session_key String: the key of the given upload session
166 - * (a given client could have started a few http uploads at once)
 244+ * @returns array request information
167245 */
168 - public static function doSessionIdDownload( $session_id, $upload_session_key ) {
169 - global $wgUser, $wgEnableWriteAPI, $wgAsyncHTTPTimeout, $wgServer,
170 - $wgSessionsInMemcached, $wgSessionHandler, $wgSessionStarted;
171 - wfDebug( __METHOD__ . "\n\n doSessionIdDownload :\n\n" );
 246+ private static function recoverSession( $sessionID, $requestKey ) {
 247+ global $wgUser, $wgServer, $wgSessionsInMemcached;
 248+
172249 // set session to the provided key:
173 - session_id( $session_id );
174 - //fire up mediaWiki session system:
 250+ session_id( $sessionID );
 251+ // fire up mediaWiki session system:
175252 wfSetupSession();
176253
177254 // start the session
178 - if( session_start() === false ) {
 255+ if ( session_start() === false ) {
179256 wfDebug( __METHOD__ . ' could not start session' );
180257 }
181258 // get all the vars we need from session_id
182 - if( !isset( $_SESSION[ 'wsDownload' ][$upload_session_key] ) ) {
183 - wfDebug( __METHOD__ . ' Error:could not find upload session');
 259+ if ( !isset( $_SESSION[ 'wsBgRequest' ][ $requestKey ] ) ) {
 260+ wfDebug( __METHOD__ . ' Error:could not find upload session' );
184261 exit();
185262 }
186263 // setup the global user from the session key we just inherited
187264 $wgUser = User::newFromSession();
188265
189266 // grab the session data to setup the request:
190 - $sd =& $_SESSION['wsDownload'][$upload_session_key];
 267+ $sd =& $_SESSION['wsBgRequest'][$requestKey];
191268
192269 // update the wgServer var ( since cmd line thinks we are localhost
193270 // when we are really orgServer)
194 - if( isset( $sd['orgServer'] ) && $sd['orgServer'] ) {
 271+ if ( isset( $sd['orgServer'] ) && $sd['orgServer'] ) {
195272 $wgServer = $sd['orgServer'];
196273 }
197274 // close down the session so we can other http queries can get session
198275 // updates: (if not $wgSessionsInMemcached)
199 - if( !$wgSessionsInMemcached )
 276+ if ( !$wgSessionsInMemcached ) {
200277 session_write_close();
 278+ }
201279
202 - $req = HttpRequest::newRequest( $sd['url'], array(
203 - 'target_file_path' => $sd['target_file_path'],
204 - 'upload_session_key'=> $upload_session_key,
205 - 'timeout' => $wgAsyncHTTPTimeout,
206 - 'do_close_session_update' => true
207 - ) );
208 - // run the actual request .. (this can take some time)
209 - wfDebug( __METHOD__ . 'do Session Download :: ' . $sd['url'] . ' tf: ' .
210 - $sd['target_file_path'] . "\n\n");
211 - $status = $req->doRequest();
212 - //wfDebug("done with req status is: ". $status->isOK(). ' '.$status->getWikiText(). "\n");
 280+ return $sd;
 281+ }
213282
214 - // start up the session again:
215 - if( session_start() === false ) {
216 - wfDebug( __METHOD__ . ' ERROR:: Could not start session');
 283+ /**
 284+ * Update the session with the finished information.
 285+ * @param $sessionID string
 286+ * @param $requestKey string the HTTP request key
 287+ */
 288+ private static function updateSession( $sessionID, $requestKey, $status ) {
 289+
 290+ if ( session_start() === false ) {
 291+ wfDebug( __METHOD__ . ' ERROR:: Could not start session' );
217292 }
218 - // grab the updated session data pointer
219 - $sd =& $_SESSION['wsDownload'][$upload_session_key];
220 - // if error update status:
221 - if( !$status->isOK() ) {
 293+
 294+ $sd =& $_SESSION['wsBgRequest'][$requestKey];
 295+ if ( !$status->isOK() ) {
222296 $sd['apiUploadResult'] = FormatJson::encode(
223297 array( 'error' => $status->getWikiText() )
224298 );
 299+ } else {
 300+ $sd['apiUploadResult'] = FormatJson::encode( $status->value );
225301 }
226 - // if status okay process upload using fauxReq to api:
227 - if( $status->isOK() ){
228 - // setup the FauxRequest
 302+
 303+ session_write_close();
 304+ }
 305+
 306+ /**
 307+ * Take care of the downloaded file
 308+ * @param $sd array
 309+ * @param $status Status
 310+ *
 311+ * @returns Status
 312+ */
 313+ private static function doFauxRequest( $sd, $status ) {
 314+ global $wgEnableWriteAPI;
 315+
 316+ if ( $status->isOK() ) {
229317 $fauxReqData = $sd['mParams'];
230318
231319 // Fix boolean parameters
232 - foreach( $fauxReqData as $k => $v ) {
233 - if( $v === false )
 320+ foreach ( $fauxReqData as $k => $v ) {
 321+ if ( $v === false )
234322 unset( $fauxReqData[$k] );
235323 }
236324
237325 $fauxReqData['action'] = 'upload';
238326 $fauxReqData['format'] = 'json';
239 - $fauxReqData['internalhttpsession'] = $upload_session_key;
 327+ $fauxReqData['internalhttpsession'] = $requestKey;
 328+
240329 // evil but no other clean way about it:
241 - $faxReq = new FauxRequest( $fauxReqData, true );
242 - $processor = new ApiMain( $faxReq, $wgEnableWriteAPI );
 330+ $fauxReq = new FauxRequest( $fauxReqData, true );
 331+ $processor = new ApiMain( $fauxReq, $wgEnableWriteAPI );
243332
244 - //init the mUpload var for the $processor
 333+ // init the mUpload var for the $processor
245334 $processor->execute();
246335 $processor->getResult()->cleanUpUTF8();
247336 $printer = $processor->createPrinterByName( 'json' );
248337 $printer->initPrinter( false );
249338 ob_start();
250339 $printer->execute();
251 - $apiUploadResult = ob_get_clean();
252340
253341 // the status updates runner will grab the result form the session:
254 - $sd['apiUploadResult'] = $apiUploadResult;
 342+ $status->value = ob_get_clean();
255343 }
256 - // close the session:
257 - session_write_close();
 344+ return $status;
258345 }
259346
260347 /**
261 - * Check if the URL can be served by localhost
262 - * @param $url string Full url to check
263 - * @return bool
 348+ * Run a session based download.
 349+ *
 350+ * @param $sessionID string: the session id with the download details
 351+ * @param $requestKey string: the key of the given upload session
 352+ * (a given client could have started a few http uploads at once)
264353 */
265 - public static function isLocalURL( $url ) {
266 - global $wgCommandLineMode, $wgConf;
267 - if ( $wgCommandLineMode ) {
268 - return false;
269 - }
 354+ public static function doSessionIdDownload( $sessionID, $requestKey ) {
 355+ global $wgAsyncHTTPTimeout;
270356
271 - // Extract host part
272 - $matches = array();
273 - if ( preg_match( '!^http://([\w.-]+)[/:].*$!', $url, $matches ) ) {
274 - $host = $matches[1];
275 - // Split up dotwise
276 - $domainParts = explode( '.', $host );
277 - // Check if this domain or any superdomain is listed in $wgConf as a local virtual host
278 - $domainParts = array_reverse( $domainParts );
279 - for ( $i = 0; $i < count( $domainParts ); $i++ ) {
280 - $domainPart = $domainParts[$i];
281 - if ( $i == 0 ) {
282 - $domain = $domainPart;
283 - } else {
284 - $domain = $domainPart . '.' . $domain;
285 - }
286 - if ( $wgConf->isLocalVHost( $domain ) ) {
287 - return true;
288 - }
289 - }
290 - }
291 - return false;
292 - }
 357+ wfDebug( __METHOD__ . "\n\n doSessionIdDownload :\n\n" );
 358+ $sd = self::recoverSession( $sessionID );
 359+ $req = HttpRequest::factory( $sd['url'],
 360+ array(
 361+ 'targetFilePath' => $sd['targetFilePath'],
 362+ 'requestKey' => $requestKey,
 363+ 'timeout' => $wgAsyncHTTPTimeout,
 364+ ) );
293365
294 - /**
295 - * Return a standard user-agent we can use for external requests.
296 - */
297 - public static function userAgent() {
298 - global $wgVersion;
299 - return "MediaWiki/$wgVersion";
300 - }
 366+ // run the actual request .. (this can take some time)
 367+ wfDebug( __METHOD__ . 'do Session Download :: ' . $sd['url'] . ' tf: ' .
 368+ $sd['targetFilePath'] . "\n\n" );
 369+ $status = $req->doRequest();
301370
302 - /**
303 - * Checks that the given URI is a valid one
304 - * @param $uri Mixed: URI to check for validity
305 - */
306 - public static function isValidURI( $uri ){
307 - return preg_match(
308 - '/(ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/',
309 - $uri,
310 - $matches
311 - );
 371+ self::updateSession( $sessionID, $requestKey,
 372+ self::handleFauxResponse( $sd, $status ) );
312373 }
313374 }
314375
 376+/**
 377+ * This wrapper class will call out to curl (if available) or fallback
 378+ * to regular PHP if necessary for handling internal HTTP requests.
 379+ */
315380 class HttpRequest {
316 - var $target_file_path;
317 - var $upload_session_key;
318 - function __construct( $url, $opt ){
 381+ private $targetFilePath;
 382+ private $requestKey;
 383+ protected $content;
 384+ protected $timeout = 'default';
 385+ protected $headersOnly = null;
 386+ protected $postdata = null;
 387+ protected $proxy = null;
 388+ protected $no_proxy = false;
 389+ protected $sslVerifyHost = true;
 390+ protected $caInfo = null;
 391+ protected $method = "GET";
 392+ protected $url;
 393+ public $status;
319394
320 - global $wgSyncHTTPTimeout;
 395+ /**
 396+ * @param $url string url to use
 397+ * @param $options array (optional) extra params to pass
 398+ * Possible keys for the array:
 399+ * method
 400+ * timeout
 401+ * targetFilePath
 402+ * requestKey
 403+ * headersOnly
 404+ * postdata
 405+ * proxy
 406+ * no_proxy
 407+ * sslVerifyHost
 408+ * caInfo
 409+ */
 410+ function __construct( $url = null, $opt ) {
 411+ global $wgHTTPTimeout;
 412+
321413 $this->url = $url;
322 - // set the timeout to default sync timeout (unless the timeout option is provided)
323 - $this->timeout = ( isset( $opt['timeout'] ) ) ? $opt['timeout'] : $wgSyncHTTPTimeout;
324 - //check special key default
325 - if($this->timeout == 'default'){
326 - $opts['timeout'] = $wgSyncHTTPTimeout;
 414+
 415+ if ( !ini_get( 'allow_url_fopen' ) ) {
 416+ $this->status = Status::newFatal( 'allow_url_fopen needs to be enabled for http copy to work' );
 417+ } elseif ( !Http::isValidURI( $this->url ) ) {
 418+ $this->status = Status::newFatal( 'bad-url' );
 419+ } else {
 420+ $this->status = Status::newGood( 100 ); // continue
327421 }
328422
329 - $this->method = ( isset( $opt['method'] ) ) ? $opt['method'] : 'GET';
330 - $this->target_file_path = ( isset( $opt['target_file_path'] ) )
331 - ? $opt['target_file_path'] : false;
332 - $this->upload_session_key = ( isset( $opt['upload_session_key'] ) )
333 - ? $opt['upload_session_key'] : false;
334 - $this->headers_only = ( isset( $opt['headers_only'] ) ) ? $opt['headers_only'] : false;
335 - $this->do_close_session_update = isset( $opt['do_close_session_update'] );
336 - $this->postData = isset( $opt['postdata'] ) ? $opt['postdata'] : '';
 423+ if ( array_key_exists( 'timeout', $opt ) && $opt['timeout'] != 'default' ) {
 424+ $this->timeout = $opt['timeout'];
 425+ } else {
 426+ $this->timeout = $wgHTTPTimeout;
 427+ }
337428
338 - $this->proxy = isset( $opt['proxy'] )? $opt['proxy'] : '';
 429+ $members = array( "targetFilePath", "requestKey", "headersOnly", "postdata",
 430+ "proxy", "no_proxy", "sslVerifyHost", "caInfo", "method" );
 431+ foreach ( $members as $o ) {
 432+ if ( array_key_exists( $o, $opt ) ) {
 433+ $this->$o = $opt[$o];
 434+ }
 435+ }
339436
340 - $this->ssl_verifyhost = (isset( $opt['ssl_verifyhost'] ))? $opt['ssl_verifyhost']: false;
 437+ if ( is_array( $this->postdata ) ) {
 438+ $this->postdata = wfArrayToCGI( $this->postdata );
 439+ }
 440+ }
341441
342 - $this->cainfo = (isset( $opt['cainfo'] ))? $op['cainfo']: false;
343 -
 442+ /**
 443+ * For backwards compatibility, we provide a __toString method so
 444+ * that any code that expects a string result from Http::Get()
 445+ * will see the content of the request.
 446+ */
 447+ function __toString() {
 448+ return $this->content;
344449 }
345450
346 - public static function newRequest($url, $opt){
347 - # select the handler (use curl if available)
348 - if ( function_exists( 'curl_init' ) ) {
349 - return new curlHttpRequest($url, $opt);
 451+ /**
 452+ * Generate a new request object
 453+ * @see HttpRequest::__construct
 454+ */
 455+ public static function factory( $url, $opt ) {
 456+ global $wgForceHTTPEngine;
 457+
 458+ if ( function_exists( 'curl_init' ) && $wgForceHTTPEngine == "curl" ) {
 459+ return new CurlHttpRequest( $url, $opt );
350460 } else {
351 - return new phpHttpRequest($url, $opt);
 461+ return new PhpHttpRequest( $url, $opt );
352462 }
353463 }
354464
355 - /**
356 - * Get the contents of a file by HTTP
357 - * @param $url string Full URL to act on
358 - * @param $Opt associative array Optional array of options:
359 - * 'method' => 'GET', 'POST' etc.
360 - * 'target_file_path' => if curl should output to a target file
361 - * 'adapter' => 'curl', 'soket'
362 - */
363 - public function doRequest() {
364 - # Make sure we have a valid url
365 - if( !Http::isValidURI( $this->url ) )
366 - return Status::newFatal('bad-url');
367 - //do the actual request:
368 - return $this->doReq();
 465+ public function getContent() {
 466+ return $this->content;
369467 }
370 -}
371 -class curlHttpRequest extends HttpRequest {
372 - public function doReq(){
373 - global $wgHTTPProxy, $wgTitle;
374468
375 - $status = Status::newGood();
376 - $c = curl_init( $this->url );
377 -
378 - // only do proxy setup if ( not suppressed $this->proxy === false )
379 - if( $this->proxy !== false ){
380 - if( $this->proxy ){
381 - curl_setopt( $c, CURLOPT_PROXY, $this->proxy );
382 - } else if ( Http::isLocalURL( $this->url ) ) {
383 - curl_setopt( $c, CURLOPT_PROXY, 'localhost:80' );
384 - } else if ( $wgHTTPProxy ) {
385 - curl_setopt( $c, CURLOPT_PROXY, $wgHTTPProxy );
 469+ public function handleOutput() {
 470+ // if we wrote to a target file close up or return error
 471+ if ( $this->targetFilePath ) {
 472+ $this->writer->close();
 473+ if ( !$this->writer->status->isOK() ) {
 474+ $this->status = $this->writer->status;
 475+ return $this->status;
386476 }
387477 }
 478+ }
388479
389 - curl_setopt( $c, CURLOPT_TIMEOUT, $this->timeout );
390 - curl_setopt( $c, CURLOPT_USERAGENT, Http::userAgent() );
 480+ public function doRequest() {
 481+ global $wgTitle;
391482
392 - if( $this->ssl_verifyhost )
393 - curl_setopt( $c, CURLOPT_SSL_VERIFYHOST, $this->ssl_verifyhost);
 483+ if ( !$this->status->isOK() ) {
 484+ return $this->status;
 485+ }
394486
395 - if( $this->cainfo )
396 - curl_setopt( $c, CURLOPT_CAINFO, $this->cainfo);
 487+ $this->initRequest();
397488
398 - if ( $this->headers_only ) {
399 - curl_setopt( $c, CURLOPT_NOBODY, true );
400 - curl_setopt( $c, CURLOPT_HEADER, true );
401 - } elseif ( $this->method == 'POST' ) {
402 - curl_setopt( $c, CURLOPT_POST, true );
403 - curl_setopt( $c, CURLOPT_POSTFIELDS, $this->postData );
404 - // Suppress 'Expect: 100-continue' header, as some servers
405 - // will reject it with a 417 and Curl won't auto retry
406 - // with HTTP 1.0 fallback
407 - curl_setopt( $c, CURLOPT_HTTPHEADER, array( 'Expect:' ) );
408 - } else {
409 - curl_setopt( $c, CURLOPT_CUSTOMREQUEST, $this->method );
 489+ if ( !$this->no_proxy ) {
 490+ $this->proxySetup();
410491 }
411492
412493 # Set the referer to $wgTitle, even in command-line mode
@@ -414,268 +495,332 @@
415496 # $_SERVER['REQUEST_URI'] gives a less reliable indication of the
416497 # referring page.
417498 if ( is_object( $wgTitle ) ) {
418 - curl_setopt( $c, CURLOPT_REFERER, $wgTitle->getFullURL() );
 499+ $this->set_referer( $wgTitle->getFullURL() );
419500 }
420501
421 - // set the write back function (if we are writing to a file)
422 - if( $this->target_file_path ) {
423 - $cwrite = new simpleFileWriter( $this->target_file_path,
424 - $this->upload_session_key,
425 - $this->do_close_session_update
426 - );
427 - if( !$cwrite->status->isOK() ) {
428 - wfDebug( __METHOD__ . "ERROR in setting up simpleFileWriter\n" );
429 - $status = $cwrite->status;
430 - return $status;
431 - }
432 - curl_setopt( $c, CURLOPT_WRITEFUNCTION, array( $cwrite, 'callbackWriteBody' ) );
 502+ $this->setupOutputHandler();
 503+
 504+ if ( $this->status->isOK() ) {
 505+ $this->spinTheWheel();
433506 }
434507
435 - // start output grabber:
436 - if( !$this->target_file_path )
437 - ob_start();
 508+ if ( !$this->status->isOK() ) {
 509+ return $this->status;
 510+ }
438511
439 - //run the actual curl_exec:
440 - try {
441 - if ( false === curl_exec( $c ) ) {
442 - $error_txt ='Error sending request: #' . curl_errno( $c ) .' '. curl_error( $c );
443 - wfDebug( __METHOD__ . $error_txt . "\n" );
444 - $status = Status::newFatal( $error_txt );
 512+ $this->handleOutput();
 513+
 514+ $this->finish();
 515+ return $this->status;
 516+ }
 517+
 518+ public function setupOutputHandler() {
 519+ if ( $this->targetFilePath ) {
 520+ $this->writer = new SimpleFileWriter( $this->targetFilePath,
 521+ $this->requestKey );
 522+ if ( !$this->writer->status->isOK() ) {
 523+ wfDebug( __METHOD__ . "ERROR in setting up SimpleFileWriter\n" );
 524+ $this->status = $this->writer->status;
 525+ return $this->status;
445526 }
446 - } catch ( Exception $e ) {
447 - // do something with curl exec error?
 527+ $this->setCallback( array( $this, 'readAndSave' ) );
 528+ } else {
 529+ $this->setCallback( array( $this, 'readOnly' ) );
448530 }
449 - // if direct request output the results to the stats value:
450 - if( !$this->target_file_path && $status->isOK() ) {
451 - $status->value = ob_get_contents();
452 - ob_end_clean();
 531+ }
 532+}
 533+
 534+/**
 535+ * HttpRequest implemented using internal curl compiled into PHP
 536+ */
 537+class CurlHttpRequest extends HttpRequest {
 538+ private $c;
 539+
 540+ public function initRequest() {
 541+ $this->c = curl_init( $this->url );
 542+ }
 543+
 544+ public function proxySetup() {
 545+ global $wgHTTPProxy;
 546+
 547+ if ( is_string( $this->proxy ) ) {
 548+ curl_setopt( $this->c, CURLOPT_PROXY, $this->proxy );
 549+ } else if ( Http::isLocalURL( $this->url ) ) { /* Not sure this makes any sense. */
 550+ curl_setopt( $this->c, CURLOPT_PROXY, 'localhost:80' );
 551+ } else if ( $wgHTTPProxy ) {
 552+ curl_setopt( $this->c, CURLOPT_PROXY, $wgHTTPProxy );
453553 }
454 - // if we wrote to a target file close up or return error
455 - if( $this->target_file_path ) {
456 - $cwrite->close();
457 - if( !$cwrite->status->isOK() ) {
458 - return $cwrite->status;
459 - }
 554+ }
 555+
 556+ public function setCallback( $cb ) {
 557+ curl_setopt( $this->c, CURLOPT_WRITEFUNCTION, $cb );
 558+ }
 559+
 560+ public function spinTheWheel() {
 561+ curl_setopt( $this->c, CURLOPT_TIMEOUT, $this->timeout );
 562+ curl_setopt( $this->c, CURLOPT_USERAGENT, Http::userAgent() );
 563+ curl_setopt( $this->c, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0 );
 564+
 565+ if ( $this->sslVerifyHost ) {
 566+ curl_setopt( $this->c, CURLOPT_SSL_VERIFYHOST, $this->sslVerifyHost );
460567 }
461568
462 - if ( $this->headers_only ) {
463 - $headers = explode( "\n", $status->value );
464 - $headerArray = array();
465 - foreach ( $headers as $header ) {
466 - if ( !strlen( trim( $header ) ) )
467 - continue;
468 - $headerParts = explode( ':', $header, 2 );
469 - if ( count( $headerParts ) == 1 ) {
470 - $headerArray[] = trim( $header );
471 - } else {
472 - list( $key, $val ) = $headerParts;
473 - $headerArray[trim( $key )] = trim( $val );
474 - }
475 - }
476 - $status->value = $headerArray;
 569+ if ( $this->caInfo ) {
 570+ curl_setopt( $this->c, CURLOPT_CAINFO, $this->caInfo );
 571+ }
 572+
 573+ if ( $this->headersOnly ) {
 574+ curl_setopt( $this->c, CURLOPT_NOBODY, true );
 575+ curl_setopt( $this->c, CURLOPT_HEADER, true );
 576+ } elseif ( $this->method == 'POST' ) {
 577+ curl_setopt( $this->c, CURLOPT_POST, true );
 578+ curl_setopt( $this->c, CURLOPT_POSTFIELDS, $this->postdata );
 579+ // Suppress 'Expect: 100-continue' header, as some servers
 580+ // will reject it with a 417 and Curl won't auto retry
 581+ // with HTTP 1.0 fallback
 582+ curl_setopt( $this->c, CURLOPT_HTTPHEADER, array( 'Expect:' ) );
477583 } else {
478 - # Don't return the text of error messages, return false on error
479 - $retcode = curl_getinfo( $c, CURLINFO_HTTP_CODE );
480 - if ( $retcode != 200 ) {
481 - wfDebug( __METHOD__ . ": HTTP return code $retcode\n" );
482 - $status = Status::newFatal( "HTTP return code $retcode\n" );
 584+ curl_setopt( $this->c, CURLOPT_CUSTOMREQUEST, $this->method );
 585+ }
 586+
 587+ try {
 588+ if ( false === curl_exec( $this->c ) ) {
 589+ $error_txt = 'Error sending request: #' . curl_errno( $this->c ) . ' ' . curl_error( $this->c );
 590+ wfDebug( __METHOD__ . $error_txt . "\n" );
 591+ $this->status->fatal( $error_txt ); /* i18n? */
483592 }
484 - # Don't return truncated output
485 - $errno = curl_errno( $c );
 593+ } catch ( Exception $e ) {
 594+ $errno = curl_errno( $this->c );
486595 if ( $errno != CURLE_OK ) {
487 - $errstr = curl_error( $c );
 596+ $errstr = curl_error( $this->c );
488597 wfDebug( __METHOD__ . ": CURL error code $errno: $errstr\n" );
489 - $status = Status::newFatal( " CURL error code $errno: $errstr\n" );
 598+ $this->status->fatal( "CURL error code $errno: $errstr\n" ); /* i18n? */
490599 }
491600 }
 601+ }
492602
493 - curl_close( $c );
494 - // return the result obj
495 - return $status;
 603+ public function readOnly( $curlH, $content ) {
 604+ $this->content .= $content;
 605+ return strlen( $content );
496606 }
497 -}
498 -class phpHttpRequest extends HttpRequest {
499 - public function doReq() {
500 - global $wgTitle, $wgHTTPProxy;
501 - # Check for php.ini allow_url_fopen
502 - if( !ini_get( 'allow_url_fopen' ) ) {
503 - return Status::newFatal( 'allow_url_fopen needs to be enabled for http copy to work' );
504 - }
505607
506 - // start with good status:
507 - $status = Status::newGood();
 608+ public function readAndSave( $curlH, $content ) {
 609+ return $this->writer->write( $content );
 610+ }
508611
509 - if ( $this->headers_only ) {
510 - $status->value = get_headers( $this->url, 1 );
511 - return $status;
 612+ public function getCode() {
 613+ # Don't return truncated output
 614+ $code = curl_getinfo( $this->c, CURLINFO_HTTP_CODE );
 615+ if ( $code < 400 ) {
 616+ $this->status->setResult( true, $code );
 617+ } else {
 618+ $this->status->setResult( false, $code );
512619 }
 620+ }
513621
514 - // setup the headers
515 - $headers = array( "User-Agent: " . Http::userAgent() );
516 - if ( is_object( $wgTitle ) ) {
517 - $headers[] = "Referer: ". $wgTitle->getFullURL();
518 - }
 622+ public function finish() {
 623+ curl_close( $this->c );
 624+ }
519625
520 - if( strcasecmp( $this->method, 'post' ) == 0 ) {
 626+}
 627+
 628+class PhpHttpRequest extends HttpRequest {
 629+ private $reqHeaders;
 630+ private $callback;
 631+ private $fh;
 632+
 633+ public function initRequest() {
 634+ $this->reqHeaders[] = "User-Agent: " . Http::userAgent();
 635+ $this->reqHeaders[] = "Accept: */*";
 636+ if ( $this->method == 'POST' ) {
521637 // Required for HTTP 1.0 POSTs
522 - $headers[] = "Content-Length: 0";
 638+ $this->reqHeaders[] = "Content-Length: " . strlen( $this->postdata );
 639+ $this->reqHeaders[] = "Content-type: application/x-www-form-urlencoded";
523640 }
 641+ }
524642
525 - $httpContextOptions = array(
526 - 'method' => $this->method,
527 - 'header' => implode( "\r\n", $headers ),
528 - 'timeout' => $this->timeout
529 - );
 643+ public function proxySetup() {
 644+ global $wgHTTPProxy;
530645
531 - // Proxy setup:
532 - if( $this->proxy ){
533 - $httpContextOptions['proxy'] = 'tcp://' . $this->proxy;
534 - }else if ( Http::isLocalURL( $this->url ) ) {
535 - $httpContextOptions['proxy'] = 'tcp://localhost:80';
 646+ if ( $this->proxy ) {
 647+ $this->proxy = 'tcp://' . $this->proxy;
 648+ } elseif ( Http::isLocalURL( $this->url ) ) {
 649+ $this->proxy = 'tcp://localhost:80';
536650 } elseif ( $wgHTTPProxy ) {
537 - $httpContextOptions['proxy'] = 'tcp://' . $wgHTTPProxy ;
 651+ $this->proxy = 'tcp://' . $wgHTTPProxy ;
538652 }
 653+ }
539654
540 - $fcontext = stream_context_create (
541 - array(
542 - 'http' => $httpContextOptions
543 - )
544 - );
 655+ public function setReferrer( $url ) {
 656+ $this->reqHeaders[] = "Referer: $url";
 657+ }
545658
546 - $fh = fopen( $this->url, "r", false, $fcontext);
 659+ public function setCallback( $cb ) {
 660+ $this->callback = $cb;
 661+ }
547662
548 - // set the write back function (if we are writing to a file)
549 - if( $this->target_file_path ) {
550 - $cwrite = new simpleFileWriter( $this->target_file_path,
551 - $this->upload_session_key, $this->do_close_session_update );
552 - if( !$cwrite->status->isOK() ) {
553 - wfDebug( __METHOD__ . "ERROR in setting up simpleFileWriter\n" );
554 - $status = $cwrite->status;
555 - return $status;
556 - }
 663+ public function readOnly( $contents ) {
 664+ if ( $this->headersOnly ) {
 665+ return false;
 666+ }
 667+ $this->content .= $contents;
557668
558 - // Read $fh into the simpleFileWriter (grab in 64K chunks since
559 - // it's likely a ~large~ media file)
560 - while ( !feof( $fh ) ) {
561 - $contents = fread( $fh, 65536 );
562 - $cwrite->callbackWriteBody( $fh, $contents );
563 - }
564 - $cwrite->close();
565 - // check for simpleFileWriter error:
566 - if( !$cwrite->status->isOK() ) {
567 - return $cwrite->status;
568 - }
 669+ return strlen( $contents );
 670+ }
 671+
 672+ public function readAndSave( $contents ) {
 673+ if ( $this->headersOnly ) {
 674+ return false;
 675+ }
 676+ return $this->writer->write( $content );
 677+ }
 678+
 679+ public function finish() {
 680+ fclose( $this->fh );
 681+ }
 682+
 683+ public function spinTheWheel() {
 684+ $opts = array();
 685+ if ( $this->proxy && !$this->no_proxy ) {
 686+ $opts['proxy'] = $this->proxy;
 687+ $opts['request_fulluri'] = true;
 688+ }
 689+
 690+ $opts['method'] = $this->method;
 691+ $opts['timeout'] = $this->timeout;
 692+ $opts['header'] = implode( "\r\n", $this->reqHeaders );
 693+ if ( version_compare( "5.3.0", phpversion(), ">" ) ) {
 694+ $opts['protocol_version'] = "1.0";
569695 } else {
570 - // read $fh into status->value
571 - $status->value = @stream_get_contents( $fh );
 696+ $opts['protocol_version'] = "1.1";
572697 }
573 - //close the url file wrapper
574 - fclose( $fh );
575698
576 - // check for "false"
577 - if( $status->value === false ) {
578 - $status->error( 'file_get_contents-failed' );
 699+ if ( $this->postdata ) {
 700+ $opts['content'] = $this->postdata;
579701 }
580 - return $status;
 702+
 703+ $context = stream_context_create( array( 'http' => $opts ) );
 704+ $this->fh = fopen( $this->url, "r", false, $context );
 705+ $result = stream_get_meta_data( $this->fh );
 706+
 707+ if ( $result['timed_out'] ) {
 708+ $this->status->error( __CLASS__ . '::timed-out-in-headers' );
 709+ }
 710+
 711+ $this->headers = $result['wrapper_data'];
 712+
 713+ $end = false;
 714+ $size = 8192;
 715+ while ( !$end ) {
 716+ $contents = fread( $this->fh, $size );
 717+ $size = call_user_func( $this->callback, $contents );
 718+ $end = ( $size == 0 ) || feof( $this->fh );
 719+ }
581720 }
582 -
583721 }
584722
585723 /**
586724 * SimpleFileWriter with session id updates
587725 */
588 -class simpleFileWriter {
589 - var $target_file_path;
590 - var $status = null;
591 - var $session_id = null;
592 - var $session_update_interval = 0; // how often to update the session while downloading
 726+class SimpleFileWriter {
 727+ private $targetFilePath = null;
 728+ private $status = null;
 729+ private $sessionId = null;
 730+ private $sessionUpdateInterval = 0; // how often to update the session while downloading
 731+ private $currentFileSize = 0;
 732+ private $requestKey = null;
 733+ private $prevTime = 0;
 734+ private $fp = null;
593735
594 - function simpleFileWriter( $target_file_path, $upload_session_key,
595 - $do_close_session_update = false )
596 - {
597 - $this->target_file_path = $target_file_path;
598 - $this->upload_session_key = $upload_session_key;
 736+ /**
 737+ * @param $targetFilePath string the path to write the file out to
 738+ * @param $requestKey string the request to update
 739+ */
 740+ function __construct__( $targetFilePath, $requestKey ) {
 741+ $this->targetFilePath = $targetFilePath;
 742+ $this->requestKey = $requestKey;
599743 $this->status = Status::newGood();
600 - $this->do_close_session_update = $do_close_session_update;
601744 // open the file:
602 - $this->fp = fopen( $this->target_file_path, 'w' );
603 - if( $this->fp === false ) {
 745+ $this->fp = fopen( $this->targetFilePath, 'w' );
 746+ if ( $this->fp === false ) {
604747 $this->status = Status::newFatal( 'HTTP::could-not-open-file-for-writing' );
605748 }
606749 // true start time
607750 $this->prevTime = time();
608751 }
609752
610 - public function callbackWriteBody( $ch, $data_packet ) {
 753+ public function write( $dataPacket ) {
611754 global $wgMaxUploadSize, $wgLang;
612755
 756+ if ( !$this->status->isOK() ) {
 757+ return false;
 758+ }
 759+
613760 // write out the content
614 - if( fwrite( $this->fp, $data_packet ) === false ) {
615 - wfDebug( __METHOD__ ." ::could-not-write-to-file\n" );
 761+ if ( fwrite( $this->fp, $dataPacket ) === false ) {
 762+ wfDebug( __METHOD__ . " ::could-not-write-to-file\n" );
616763 $this->status = Status::newFatal( 'HTTP::could-not-write-to-file' );
617 - return 0;
 764+ return false;
618765 }
619766
620767 // check file size:
621768 clearstatcache();
622 - $this->current_fsize = filesize( $this->target_file_path );
 769+ $this->currentFileSize = filesize( $this->targetFilePath );
623770
624 - if( $this->current_fsize > $wgMaxUploadSize ) {
625 - wfDebug( __METHOD__ . " ::http download too large\n" );
626 - $this->status = Status::newFatal( 'HTTP::file-has-grown-beyond-upload-limit-killing: ' .
627 - 'downloaded more than ' .
628 - $wgLang->formatSize( $wgMaxUploadSize ) . ' ' );
629 - return 0;
 771+ if ( $this->currentFileSize > $wgMaxUploadSize ) {
 772+ wfDebug( __METHOD__ . " ::http-download-too-large\n" );
 773+ $this->status = Status::newFatal( 'HTTP::file-has-grown-beyond-upload-limit-killing: ' . /* i18n? */
 774+ 'downloaded more than ' .
 775+ $wgLang->formatSize( $wgMaxUploadSize ) . ' ' );
 776+ return false;
630777 }
631 - // if more than session_update_interval second have passed update_session_progress
632 - if( $this->do_close_session_update && $this->upload_session_key &&
633 - ( ( time() - $this->prevTime ) > $this->session_update_interval ) ) {
634 - $this->prevTime = time();
635 - $session_status = $this->update_session_progress();
636 - if( !$session_status->isOK() ) {
637 - $this->status = $session_status;
638 - wfDebug( __METHOD__ . ' update session failed or was canceled');
639 - return 0;
640 - }
 778+ // if more than session_update_interval second have passed updateProgress
 779+ if ( $this->requestKey &&
 780+ ( ( time() - $this->prevTime ) > $this->sessionUpdateInterval ) ) {
 781+ $this->prevTime = time();
 782+ $session_status = $this->updateProgress();
 783+ if ( !$session_status->isOK() ) {
 784+ $this->status = $session_status;
 785+ wfDebug( __METHOD__ . ' update session failed or was canceled' );
 786+ return false;
 787+ }
641788 }
642 - return strlen( $data_packet );
 789+ return strlen( $dataPacket );
643790 }
644791
645 - public function update_session_progress() {
 792+ public function updateProgress() {
646793 global $wgSessionsInMemcached;
647 - $status = Status::newGood();
 794+
648795 // start the session (if necessary)
649 - if( !$wgSessionsInMemcached ) {
 796+ if ( !$wgSessionsInMemcached ) {
650797 wfSuppressWarnings();
651 - if( session_start() === false ) {
 798+ if ( session_start() === false ) {
652799 wfDebug( __METHOD__ . ' could not start session' );
653800 exit( 0 );
654801 }
655802 wfRestoreWarnings();
656803 }
657 - $sd =& $_SESSION['wsDownload'][ $this->upload_session_key ];
 804+ $sd =& $_SESSION['wsBgRequest'][ $this->requestKey ];
658805 // check if the user canceled the request:
659 - if( isset( $sd['user_cancel'] ) && $sd['user_cancel'] == true ) {
660 - //@@todo kill the download
 806+ if ( $sd['userCancel'] ) {
 807+ // @@todo kill the download
661808 return Status::newFatal( 'user-canceled-request' );
662809 }
663810 // update the progress bytes download so far:
664 - $sd['loaded'] = $this->current_fsize;
 811+ $sd['loaded'] = $this->currentFileSize;
665812
666813 // close down the session so we can other http queries can get session updates:
667 - if( !$wgSessionsInMemcached )
 814+ if ( !$wgSessionsInMemcached )
668815 session_write_close();
669816
670 - return $status;
 817+ return Status::newGood();
671818 }
672819
673820 public function close() {
674 - // do a final session update:
675 - if( $this->do_close_session_update ) {
676 - $this->update_session_progress();
677 - }
 821+ $this->updateProgress();
 822+
678823 // close up the file handle:
679 - if( false === fclose( $this->fp ) ) {
 824+ if ( false === fclose( $this->fp ) ) {
680825 $this->status = Status::newFatal( 'HTTP::could-not-close-file' );
681826 }
682827 }

Follow-up revisions

RevisionCommit summaryAuthorDate
r61096Revert r61078/r61079 for now. Throws a lot of PHP warnings at translatewiki. ...raymond18:57, 15 January 2010
r61352follow up r61078, fixes bug 22224...mah02:17, 22 January 2010
r78381Remove big commented list of URLs (added in r61078) that this class will neve...demon15:49, 14 December 2010

Past revisions this follows-up on

RevisionCommit summaryAuthorDate
r60811* fix for bug 20512 ( in both trunk and js2-work branch )...dale21:57, 7 January 2010

Comments

#Comment by Freakolowsky (talk | contribs)   09:52, 15 January 2010

Fatal error: Call to undefined method PhpHttpRequest::set_referer() in /usr/local/mediawiki-svn/phase3/includes/HttpFunctions.php on line 498

#Comment by Freakolowsky (talk | contribs)   10:12, 15 January 2010

quick-fix in r61079

#Comment by Tim Starling (talk | contribs)   11:58, 15 January 2010

Reviewed by email. Marking fixme mostly because I want a couple of things renamed, and I want the async download stuff deleted again.

#Comment by Raymond (talk | contribs)   19:04, 15 January 2010

Reverted with r61096. Throws a lot of PHP warnings on translatewiki:

PHP Warning: fopen(http://127.0.0.1:54321/tmserver/en/gl/unit/%3Cbr%20%2F%3E%20%3Cbr%20%2F%3E%20%3Cp%3EBelow%20is%20the%20list%20of%20email%20domains%20allowed%20for%20editors%20of%20this%20wiki.%20Each%20line%20designates%20an%20email%20suffix%20that%20is%20given%20access%20for%20editing.%20This%20should%20be%20formatted%20with%20one%20suffix%20per%20line.%20For%20example%3A%3C%2Fp%3E%20%3Cp
PHP Warning: stream_get_meta_data(): supplied argument is not a valid stream resource in /var/www/w/includes/HttpFunctions.php on line 707
PHP Warning: fopen(http://127.0.0.1:54321/tmserver/en/nn/unit/Allows%20for%20retrieving%20structured%20data%20from%20external%20URLs%2C%20databases%20and%20other%20sources) [<a href='function.fopen'>function.fopen</a>]: failed to open stream: HTTP request failed! in /var/www/w/includes/HttpFunctions.php on line 706
PHP Warning: Invalid argument supplied for foreach() in /var/www/w/extensions/Translate/TranslateEditAddons.php on line 342
PHP Warning: fopen(http://127.0.0.1:54321/tmserver/en/gl/unit/Query%20recent%20changes) [<a href='function.fopen'>function.fopen</a>]: failed to open stream: HTTP request failed! in /var/www/w/includes/HttpFunctions.php on line 706
PHP Warning: fopen(http://127.0.0.1:54321/tmserver/en/nn/unit/Unable%20to%20connect%20to%20%241) [<a href='function.fopen'>function.fopen</a>]: failed to open stream: HTTP request failed! in /var/www/w/includes/HttpFunctions.php on line 706
PHP Warning: fopen(http://127.0.0.1:54321/tmserver/en/nn/unit/Error%3A%20json_decode%28%29%20is%20not%20supported%20in%20this%20version%20of%20PHP) [<a href='function.fopen'>function.fopen</a>]: failed to open stream: HTTP request failed! in /var/www/w/includes/HttpFunctions.php on line 706
#Comment by Tim Starling (talk | contribs)   05:52, 21 January 2010

Mark, please re-add this code with fixes for the errors above. Test the error paths in the PHP backend to make sure they behave nicely and don't spew errors, it might just be that the server at translatewiki.net was down.

#Comment by Nikerabbit (talk | contribs)   07:26, 21 January 2010

It didn't throw any errors before even if the server was down, it just timed out.

Status & tagging log