r94600 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r94599‎ | r94600 | r94601 >
Date:03:29, 16 August 2011
Author:nelson
Status:deferred
Tags:
Comment:
First checkin of wmf/config.py and smtest.
move config vars out into config.py
fix a problem where metadata was not being reverted (missing return $ret)
change $var_iable into $varIable
remove the bulk of LocalSettings.php, leaving only the Swift-specific stuff.
Modified paths:
  • /trunk/extensions/SwiftMedia/LocalSettings.php (modified) (history)
  • /trunk/extensions/SwiftMedia/SwiftMedia.body.php (modified) (history)
  • /trunk/extensions/SwiftMedia/copyover (modified) (history)
  • /trunk/extensions/SwiftMedia/smtest.py (added) (history)
  • /trunk/extensions/SwiftMedia/wmf/config.py (added) (history)
  • /trunk/extensions/SwiftMedia/wmf/rewrite.py (modified) (history)

Diff [purge]

Index: trunk/extensions/SwiftMedia/LocalSettings.php
@@ -1,136 +1,3 @@
2 -<?php
3 -# This file was automatically generated by the MediaWiki 1.18alpha
4 -# installer. If you make manual changes, please keep track in case you
5 -# need to recreate them later.
6 -#
7 -# See includes/DefaultSettings.php for all configurable settings
8 -# and their default values, but don't forget to make changes in _this_
9 -# file, not there.
10 -#
11 -# Further documentation for configuration settings may be found at:
12 -# http://www.mediawiki.org/wiki/Manual:Configuration_settings
13 -
14 -# Protect against web entry
15 -if ( !defined( 'MEDIAWIKI' ) ) {
16 - exit;
17 -}
18 -
19 -## Uncomment this to disable output compression
20 -# $wgDisableOutputCompression = true;
21 -
22 -$wgSitename = "mediastore";
23 -$wgMetaNamespace = "Mediastore";
24 -
25 -## The URL base path to the directory containing the wiki;
26 -## defaults for all runtime URL paths are based off of this.
27 -## For more information on customizing the URLs please see:
28 -## http://www.mediawiki.org/wiki/Manual:Short_URL
29 -$wgScriptPath = "";
30 -$wgScriptExtension = ".php";
31 -
32 -## The relative URL path to the skins directory
33 -$wgStylePath = "$wgScriptPath/skins";
34 -
35 -## The relative URL path to the logo. Make sure you change this from the default,
36 -## or else you'll overwrite your logo when you upgrade!
37 -$wgLogo = "$wgStylePath/common/images/wiki.png";
38 -
39 -## UPO means: this is also a user preference option
40 -
41 -$wgEnableEmail = true;
42 -$wgEnableUserEmail = true; # UPO
43 -
44 -$wgEmergencyContact = "apache@ersch.wikimedia.org";
45 -$wgPasswordSender = "apache@ersch.wikimedia.org";
46 -
47 -$wgEnotifUserTalk = false; # UPO
48 -$wgEnotifWatchlist = false; # UPO
49 -$wgEmailAuthentication = true;
50 -
51 -## Database settings
52 -$wgDBtype = "mysql";
53 -$wgDBserver = "localhost";
54 -$wgDBname = "my_wiki";
55 -$wgDBuser = "root";
56 -$wgDBpassword = "leakywiks";
57 -
58 -# MySQL specific settings
59 -$wgDBprefix = "mw2";
60 -
61 -# MySQL table options to use during installation or update
62 -$wgDBTableOptions = "ENGINE=InnoDB, DEFAULT CHARSET=binary";
63 -
64 -# Experimental charset support for MySQL 4.1/5.0.
65 -$wgDBmysql5 = false;
66 -
67 -## Shared memory settings
68 -$wgMainCacheType = CACHE_NONE;
69 -$wgMemCachedServers = array();
70 -
71 -## To enable image uploads, make sure the 'images' directory
72 -## is writable, then set this to true:
73 -$wgEnableUploads = true;
74 -#$wgUseImageMagick = true;
75 -#$wgImageMagickConvertCommand = "/usr/bin/convert";
76 -
77 -# InstantCommons allows wiki to use images from http://commons.wikimedia.org
78 -$wgUseInstantCommons = false;
79 -
80 -## If you use ImageMagick (or any other shell command) on a
81 -## Linux server, this will need to be set to the name of an
82 -## available UTF-8 locale
83 -$wgShellLocale = "en_US.utf8";
84 -
85 -## If you want to use image uploads under safe mode,
86 -## create the directories images/archive, images/thumb and
87 -## images/temp, and make them all writable. Then uncomment
88 -## this, if it's not already uncommented:
89 -$wgHashedUploadDirectory = true;
90 -
91 -## If you have the appropriate support software installed
92 -## you can enable inline LaTeX equations:
93 -$wgUseTeX = false;
94 -
95 -## Set $wgCacheDirectory to a writable directory on the web server
96 -## to make your wiki go slightly faster. The directory should not
97 -## be publically accessible from the web.
98 -#$wgCacheDirectory = "$IP/cache";
99 -
100 -# Site language code, should be one of ./languages/Language(.*).php
101 -$wgLanguageCode = "en";
102 -
103 -$wgSecretKey = "cea5a22adfa63e596b38ff892af5ab63fecaca26a43a4c9df1a35c0a85e85259";
104 -
105 -# Site upgrade key. Must be set to a string (default provided) to turn on the
106 -# web installer while LocalSettings.php is in place
107 -$wgUpgradeKey = "5fd6cb729fc715bf";
108 -
109 -## Default skin: you can change the default skin. Use the internal symbolic
110 -## names, ie 'standard', 'nostalgia', 'cologneblue', 'monobook', 'vector':
111 -$wgDefaultSkin = "vector";
112 -
113 -## For attaching licensing metadata to pages, and displaying an
114 -## appropriate copyright notice / icon. GNU Free Documentation
115 -## License and Creative Commons licenses are supported so far.
116 -#$wgEnableCreativeCommonsRdf = true;
117 -$wgRightsPage = ""; # Set to the title of a wiki page that describes your license/copyright
118 -$wgRightsUrl = "";
119 -$wgRightsText = "";
120 -$wgRightsIcon = "";
121 -# $wgRightsCode = ""; # Not yet used
122 -
123 -# Path to the GNU diff3 utility. Used for conflict resolution.
124 -$wgDiff3 = "/usr/bin/diff3";
125 -
126 -# Enabled Extensions. Most extensions are enabled by including the base extension file here
127 -# but check specific extension documentation for more details
128 -# The following extensions were automatically enabled:
129 -require( "extensions/SwiftMedia/SwiftMedia.php" );
130 -
131 -
132 -# End of automatically generated settings.
133 -# Add more configuration options below.
134 -
1352 $wgUploadDirectory = "$IP/images/swift";
1363 // we don't need this and will ignore it. $wgDeletedDirectory = "{$wgUploadDirectory}/deleted";
1374 $wgUploadPath = "http://alsted.wikimedia.org/images/swift";
@@ -159,3 +26,4 @@
16027 $wgDebugTimestamps = true;
16128 $wgShowExceptionDetails = true;
16229
 30+
Index: trunk/extensions/SwiftMedia/copyover
@@ -1,7 +1,8 @@
22 #!/bin/sh
33
4 -scp rnelson@ersch.wikimedia.org:/var/www/LocalSettings.php .
 4+scp rnelson@ersch.wikimedia.org:/var/www/LocalSettings.php LocalSettings.real
 5+sed '1,/SWIFT/d' <LocalSettings.real >LocalSettings.php
56 scp rnelson@alsted.wikimedia.org:/etc/swift/proxy-server.conf .
67 scp rnelson@ersch.wikimedia.org:/var/www/extensions/SwiftMedia/{SwiftMedia.body.php,SwiftMedia.i18n.php,SwiftMedia.php,TODO} .
7 -scp rnelson@alsted.wikimedia.org:/usr/local/lib/python2.6/dist-packages/wmf/{client.py,__init__.py,rewrite.py} wmf/
 8+scp rnelson@alsted.wikimedia.org:/usr/local/lib/python2.6/dist-packages/wmf/{client.py,__init__.py,rewrite.py,config.py} wmf/
89 perl -pi -e "s/'key' => '.*'/'key' => 'secret'/" LocalSettings.php
Index: trunk/extensions/SwiftMedia/smtest.py
@@ -0,0 +1,169 @@
 2+#!/usr/bin/python
 3+# http://www.deheus.net/petrik/blog/2005/11/20/creating-a-wikipedia-watchlist-rss-feed-with-python-and-twill/
 4+
 5+import sys, string, datetime, time, os, re, stat
 6+import twill
 7+import twill.commands as t
 8+import gd
 9+
 10+temp_html = "/tmp/wikipedia.html"
 11+rss_title = "Wikipedia watchlist"
 12+rss_link = "http://en.wikipedia.org"
 13+host = "http://ersch.wikimedia.org/"
 14+host = "http://www.ra-tes.org/phase3/"
 15+
 16+def login(username, password):
 17+ t.add_extra_header("User-Agent", "python-twill-russnelson@gmail.com")
 18+
 19+ t.go(host+"index.php/Special:UserLogin")
 20+ t.fv("1", "wpName", username)
 21+ t.fv("1", "wpPassword", password)
 22+ t.submit("wpLoginAttempt")
 23+
 24+
 25+def upload_list(browser, pagename, uploads):
 26+
 27+ # get the file sizes for later comparison.
 28+ filesizes = []
 29+ for fn in uploads:
 30+ filesizes.append(os.stat(fn)[stat.ST_SIZE])
 31+ filesizes.reverse() # because they get listed newest first.
 32+
 33+ # Upload copy #1.
 34+ t.go(host+"index.php/Special:Upload")
 35+ t.formfile("1", "wpUploadFile", uploads[0])
 36+ t.fv("1", "wpDestFile", pagename)
 37+ t.fv("1", "wpUploadDescription", "Uploading %s" % pagename)
 38+ t.submit("wpUpload")
 39+
 40+ # Verify that we succeeded.
 41+ t.find("File:%s" % pagename)
 42+
 43+ for fn in uploads[1:]:
 44+ # propose that we upload a replacement
 45+ t.go(host+"index.php?title=Special:Upload&wpDestFile=%s&wpForReUpload=1" % pagename)
 46+ t.formfile("1", "wpUploadFile", fn)
 47+ t.fv("1", "wpUploadDescription", "Uploading %s as %s" % (fn, pagename))
 48+ t.submit("wpUpload")
 49+
 50+ # get the URLs for the thumbnails
 51+ urls = []
 52+ for url in re.finditer(r'<td><a href="([^"]*?)"><img alt="Thumbnail for version .*?" src="(.*?)"', browser.get_html()):
 53+ urls.append(url.group(1))
 54+ urls.append(url.group(2))
 55+
 56+ print filesizes
 57+ for i, url in enumerate(urls):
 58+ t.go(url)
 59+ if i % 2 == 0 and len(browser.get_html()) != filesizes[i / 2]:
 60+ print i,len(browser.get_html()), filesizes[i / 2]
 61+ t.find("Files differ in size")
 62+ t.code("200")
 63+ t.back()
 64+
 65+ # delete all versions
 66+ t.go(host+"index.php?title=File:%s&action=delete" % pagename)
 67+ # after we get the confirmation page, commit to the action.
 68+ t.fv("1", "wpReason", "Test Deleting...")
 69+ t.submit("mw-filedelete-submit")
 70+
 71+ # make sure that we can't visit their URLs.
 72+ for i, url in enumerate(urls):
 73+ t.go(url)
 74+ if 0 and i % 2 == 1 and i > 0 and browser.get_code() == 200:
 75+ # bug 30192: the archived file's thumbnail doesn't get deleted.
 76+ print "special-casing the last URL"
 77+ continue
 78+ t.code("404")
 79+
 80+ # restore the current and archived version.
 81+ t.go(host+"index.php/Special:Undelete/File:%s" % pagename)
 82+ t.fv("1", "wpComment", "Test Restore")
 83+ t.submit("restore")
 84+
 85+ # visit the page to make sure that the thumbs get re-rendered properly.
 86+ # when we get the 404 handler working correctly, this won't be needed.
 87+ t.go(host+"index.php?title=File:%s" % pagename)
 88+
 89+ # make sure that they got restored correctly.
 90+ for i, url in enumerate(urls):
 91+ t.go(url)
 92+ if i % 2 == 0 and len(browser.get_html()) != filesizes[i / 2]:
 93+ t.find("Files differ in size")
 94+ t.code("200")
 95+ t.back()
 96+
 97+ if len(uploads) != 2:
 98+ return
 99+
 100+ match = re.search(r'"([^"]+?)" title="[^"]+?">revert', browser.get_html())
 101+ if not match:
 102+ t.find('revert')
 103+ t.go(match.group(1).replace('&amp;', '&'))
 104+
 105+def make_files(pagename):
 106+ redfilename = "/tmp/Red-%s" % pagename
 107+ greenfilename = "/tmp/Green-%s" % pagename
 108+ bluefilename = "/tmp/Blue-%s" % pagename
 109+
 110+ # create a small test image.
 111+ gd.gdMaxColors = 256
 112+ i = gd.image((200,100))
 113+ black = i.colorAllocate((0,0,0))
 114+ white = i.colorAllocate((255,255,255))
 115+ red = i.colorAllocate((255,55,55))
 116+ green = i.colorAllocate((55,255,55))
 117+ blue = i.colorAllocate((55,55,255))
 118+
 119+ # now write a red version
 120+ i.rectangle((0,0),(199,99),red, red)
 121+ i.line((0,0),(199,99),black)
 122+ i.string(gd.gdFontLarge, (5,50), pagename, white)
 123+ i.writePng(redfilename)
 124+
 125+ # now write a green version
 126+ i.rectangle((0,0),(199,99),green, green)
 127+ i.line((0,0),(99,99),black)
 128+ i.string(gd.gdFontLarge, (5,50), pagename, white)
 129+ i.writePng(greenfilename)
 130+
 131+ # write a blue version
 132+ i.rectangle((0,0),(199,99),blue,blue)
 133+ i.line((0,0),(99,199),black)
 134+ i.string(gd.gdFontLarge, (5,50), pagename, white)
 135+ i.writePng(bluefilename)
 136+
 137+ # propose that we delete it (in case it exists)
 138+ t.go(host+"index.php?title=File:%s&action=delete" % pagename)
 139+ # make sure that we've NOT gotten the wrong page and HAVE gotten the right one.
 140+ t.notfind('You are about to delete the file')
 141+ t.find("could not be deleted")
 142+
 143+ return (redfilename, greenfilename, bluefilename )
 144+
 145+def main():
 146+ try:
 147+ username = sys.argv[1]
 148+ password = sys.argv[2]
 149+ except IndexError:
 150+ print "Please supply username password"
 151+ sys.exit(1)
 152+ browser = twill.get_browser()
 153+ login(username, password)
 154+
 155+ serial = time.time()
 156+ pagename = "Test-%s.png" % serial
 157+ filenames = make_files(pagename)
 158+ upload_list(browser, pagename, filenames[0:2])
 159+
 160+ # try it again with two replacement files.
 161+# pagename = "Test-%sA.png" % serial
 162+# filenames = make_files(pagename)
 163+# upload_list(browser, pagename, filenames)
 164+
 165+ t.showforms()
 166+ t.save_html("/tmp/testabcd")
 167+
 168+if __name__ == "__main__":
 169+ main()
 170+
Property changes on: trunk/extensions/SwiftMedia/smtest.py
___________________________________________________________________
Added: svn:eol-style
1171 + native
Added: svn:executable
2172 + *
Index: trunk/extensions/SwiftMedia/wmf/rewrite.py
@@ -9,6 +9,7 @@
1010 import re
1111 from eventlet.green import urllib2
1212 import wmf.client
 13+import wmf.config
1314 import time
1415
1516 # the auth system turns our login and key into an account / token pair.
@@ -20,7 +21,6 @@
2122 Given an open file and a Swift object, we hand back an iterator which reads from the file,
2223 writes a copy into a Swift object, and returns what it read.
2324 """
24 - token = 'AUTH_tk95804b3cb6a44cfd994c28742cd3333f'
2525 token = None
2626
2727 def __init__(self, conn, app, url, container, obj, authurl, login, key, content_type=None, modified=None):
@@ -73,7 +73,7 @@
7474
7575 def __init__(self, app, conf):
7676 self.app = app
77 - self.account = account
 77+ self.account = wmf.config.account
7878 self.authurl = conf['url'].strip()
7979 self.login = conf['login'].strip()
8080 self.key = conf['key'].strip()
@@ -127,11 +127,21 @@
128128 app_iter=app_iter)(env, start_response) #01a
129129 elif status == 404: #4
130130 # go to the thumb media store for unknown files
131 - reqorig.host = 'ms5.pmtpa.wmnet'
 131+ reqorig.host = wmf.config.thumbhost
132132 # upload doesn't like our User-agent, otherwise we could call it using urllib2.url()
133133 opener = urllib2.build_opener()
134 - opener.addheaders = [('User-agent', 'Mozilla/5.0')]
135 - upcopy = opener.open(reqorig.url)
 134+ opener.addheaders = [('User-agent', wmf.config.user_agent)]
 135+ # At least in theory, we shouldn't be handing out links to files that we don't have
 136+ # (or in the case of thumbs, can't generate). However, someone may have a formerly
 137+ # valid link to a file, so we should do them the favor of giving them a 404.
 138+ try:
 139+ upcopy = opener.open(reqorig.url)
 140+ except urllib2.HTTPError,status:
 141+ if status == 404:
 142+ return webob.exc.HTTPNotFound('Expected original file not found')(env, start_response)
 143+ else:
 144+ return webob.exc.HTTPNotFound('Unexpected error %s' % `status`)(env, start_response)
 145+
136146 # get the Content-Type.
137147 uinfo = upcopy.info()
138148 c_t = uinfo.gettype()
Index: trunk/extensions/SwiftMedia/wmf/config.py
@@ -0,0 +1,10 @@
 2+# the auth system turns our login and key into an account / token pair.
 3+# the account remains valid forever, but the token times out.
 4+account = 'AUTH_dea4a45c-a80b-43b5-8e8b-e452f0dc778f'
 5+
 6+# the name of the scaler cluster.
 7+thumbhost = 'ms5.pmtpa.wmnet'
 8+
 9+# upload doesn't like our User-agent (Python-urllib/2.6), otherwise we could call it using urllib2.urlopen()
 10+user_agent = 'Mozilla/5.0'
 11+
Property changes on: trunk/extensions/SwiftMedia/wmf/config.py
___________________________________________________________________
Added: svn:eol-style
112 + native
Index: trunk/extensions/SwiftMedia/SwiftMedia.body.php
@@ -87,7 +87,7 @@
8888
8989 parent::__construct( $title, $repo );
9090
91 - $this->temp_path = false; // Points to our local copy.
 91+ $this->tempPath = false; // Points to our local copy.
9292 }
9393
9494 /** splitMime inherited */
@@ -98,14 +98,14 @@
9999 /** isVisible inherited */
100100
101101 function getPath() {
102 - $this->temp_path = $this->repo->getLocalCopy($this->repo->container, $this->getRel());
103 - return $this->temp_path;
 102+ $this->tempPath = $this->repo->getLocalCopy($this->repo->container, $this->getRel());
 103+ return $this->tempPath;
104104 }
105105
106106 /** Get the path of the archive directory, or a particular file if $suffix is specified */
107107 function getArchivePath( $suffix = false ) {
108 - $this->temp_path = $this->repo->getLocalCopy($this->repo->getZoneContainer('public'), $this->getArchiveRel( $suffix ));
109 - return $this->temp_path;
 108+ $this->tempPath = $this->repo->getLocalCopy($this->repo->getZoneContainer('public'), $this->getArchiveRel( $suffix ));
 109+ return $this->tempPath;
110110 }
111111
112112 /** Get the path of the thumbnail directory, or a particular file if $suffix is specified */
@@ -114,15 +114,15 @@
115115 if ( $suffix !== false ) {
116116 $path .= '/' . $suffix;
117117 }
118 - $this->temp_path = $this->repo->getLocalCopy($this->repo->getZoneContainer('thumb'), $path);
119 - return $this->temp_path;
 118+ $this->tempPath = $this->repo->getLocalCopy($this->repo->getZoneContainer('thumb'), $path);
 119+ return $this->tempPath;
120120 }
121121
122122 function __destruct() {
123 - if ($this->temp_path) {
 123+ if ($this->tempPath) {
124124 // Clean up temporary data.
125 - unlink( $this->temp_path );
126 - $this->temp_path = null;
 125+ unlink( $this->tempPath );
 126+ $this->tempPath = null;
127127 }
128128 }
129129
@@ -869,13 +869,13 @@
870870 /**
871871 * Given a container and relative path, return an absolute path pointing at a
872872 * copy of the file MUST delete the produced file, or else store it in
873 - * SwiftFile->temp_path so it will be deleted when the object goes out of
 873+ * SwiftFile->tempPath so it will be deleted when the object goes out of
874874 * scope.
875875 */
876876 function getLocalCopy($container, $rel) {
877877
878878 // get a temporary place to put the original.
879 - $temp_path = tempnam( wfTempDir(), 'swift_in_' );
 879+ $tempPath = tempnam( wfTempDir(), 'swift_in_' );
880880
881881 /* Fetch the image out of Swift */
882882 $conn = $this->connect();
@@ -887,16 +887,16 @@
888888 throw new MWException( "Unable to open original file at $container/$rel");
889889 }
890890
891 - wfDebug( __METHOD__ . " writing to " . $temp_path . "\n");
 891+ wfDebug( __METHOD__ . " writing to " . $tempPath . "\n");
892892 try {
893 - $obj->save_to_filename( $temp_path);
 893+ $obj->save_to_filename( $tempPath);
894894 } catch (IOException $e) {
895895 throw new MWException( __METHOD__ . ": error opening '$e'" );
896896 } catch (InvalidResponseException $e) {
897897 throw new MWException( __METHOD__ . "unexpected response '$e'" );
898898 }
899899
900 - return $temp_path;
 900+ return $tempPath;
901901 }
902902
903903
@@ -908,6 +908,7 @@
909909 $path = $this->resolveVirtualUrl( $virtualUrl );
910910 $ret = File::getPropsFromPath( $path );
911911 unlink( $path );
 912+ return $ret;
912913 }
913914
914915

Status & tagging log