| Index: trunk/extensions/XMLRC/generator/rc2udp.py |
| — | — | @@ -0,0 +1,271 @@ |
| | 2 | +#!/usr/bin/python |
| | 3 | + |
| | 4 | +############################################################################## |
| | 5 | +# UDP generator for testing the UDP-to-XMPP bridge and XMPP client |
| | 6 | +# |
| | 7 | +# |
| | 8 | +# Copyright (c) 2010, Wikimedia Deutschland; Author: Daniel Kinzler |
| | 9 | +# All rights reserved. |
| | 10 | +# |
| | 11 | +# This program is free software: you can redistribute it and/or modify |
| | 12 | +# it under the terms of the GNU General Public License as published by |
| | 13 | +# the Free Software Foundation, either version 3 of the License, or |
| | 14 | +# (at your option) any later version. |
| | 15 | +# |
| | 16 | +# This program is distributed in the hope that it will be useful, |
| | 17 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| | 18 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| | 19 | +# GNU General Public License for more details. |
| | 20 | +# |
| | 21 | +# You should have received a copy of the GNU General Public License |
| | 22 | +# along with this program. If not, see <http://www.gnu.org/licenses/>. |
| | 23 | +############################################################################## |
| | 24 | + |
| | 25 | +import sys, os, os.path, traceback, time |
| | 26 | +import ConfigParser, optparse |
| | 27 | +import socket, urllib |
| | 28 | +import xmpp.simplexml # could use a different XML lib, but since udp2xmpp and rcclient use this, just stick with it |
| | 29 | + |
| | 30 | +from xml.parsers.expat import ExpatError |
| | 31 | + |
| | 32 | +LOG_MUTE = 0 |
| | 33 | +LOG_QUIET = 10 |
| | 34 | +LOG_VERBOSE = 20 |
| | 35 | +LOG_DEBUG = 30 |
| | 36 | + |
| | 37 | +MAX_RC_XML_SIZE = 4 * 1024 # largest size a <rc> tag may have. |
| | 38 | + |
| | 39 | +################################################################################## |
| | 40 | + |
| | 41 | +class RCPoller(object): |
| | 42 | + """ RCPoller polls recent changes from a MediaWiki web API, |
| | 43 | + and passes them to a handler. """ |
| | 44 | + |
| | 45 | + def __init__( self, api_url, interval = 10, console_encoding = 'utf-8', |
| | 46 | + rclimit = 500, rcprops = 'user|comment|flags|timestamp|title|ids|sizes|redirect|loginfo|tags' ): |
| | 47 | + |
| | 48 | + self.console_encoding = console_encoding |
| | 49 | + self.api_url = api_url |
| | 50 | + |
| | 51 | + self.handlers = [] |
| | 52 | + self.loglevel = LOG_VERBOSE |
| | 53 | + |
| | 54 | + self.console_encoding = console_encoding |
| | 55 | + self.interval = interval |
| | 56 | + |
| | 57 | + self.rcprops = rcprops |
| | 58 | + self.rclimit = rclimit |
| | 59 | + |
| | 60 | + def warn(self, message): |
| | 61 | + if self.loglevel >= LOG_QUIET: |
| | 62 | + sys.stderr.write( "WARNING: %s\n" % ( message.encode( self.console_encoding ) ) ) |
| | 63 | + |
| | 64 | + def info(self, message): |
| | 65 | + if self.loglevel >= LOG_VERBOSE: |
| | 66 | + sys.stderr.write( "INFO: %s\n" % ( message.encode( self.console_encoding ) ) ) |
| | 67 | + |
| | 68 | + def debug(self, message): |
| | 69 | + if self.loglevel >= LOG_DEBUG: |
| | 70 | + sys.stderr.write( "DEBUG: %s\n" % ( message.encode( self.console_encoding ) ) ) |
| | 71 | + |
| | 72 | + def poll_loop( self ): |
| | 73 | + self.online = 1 |
| | 74 | + |
| | 75 | + limit = 1 #NOTE: on the first call, only fetch one RC |
| | 76 | + last_id = 0 |
| | 77 | + last_timestamp = False |
| | 78 | + |
| | 79 | + try: |
| | 80 | + while self.online: |
| | 81 | + self.debug( "fetching RCs from ID %s" % last_id ) |
| | 82 | + |
| | 83 | + try: |
| | 84 | + rcs = self.fetch_recent_changes( last_id, last_timestamp, limit ) |
| | 85 | + |
| | 86 | + if len(rcs) > 0: |
| | 87 | + limit = self.rclimit |
| | 88 | + |
| | 89 | + except Exception, e: |
| | 90 | + error_type, error_value, trbk = sys.exc_info() |
| | 91 | + self.warn( "Failed to fetch changes: %s" % " ".join( traceback.format_exception( error_type, error_value, trbk ) ) ) |
| | 92 | + continue |
| | 93 | + |
| | 94 | + for rc in rcs: |
| | 95 | + self.debug( "dispatching RC #%s (%s)" % (rc['rcid'], rc['timestamp']) ) |
| | 96 | + |
| | 97 | + #FIXME: inject wikiid. fetch once from API. |
| | 98 | + |
| | 99 | + self.dispatch_rc( rc ) |
| | 100 | + |
| | 101 | + last_id = int( rc['rcid'] ) |
| | 102 | + last_timestamp = rc['timestamp'] |
| | 103 | + |
| | 104 | + time.sleep( self.interval ) |
| | 105 | + |
| | 106 | + |
| | 107 | + except KeyboardInterrupt: |
| | 108 | + pass |
| | 109 | + |
| | 110 | + self.info("service loop terminated, disconnecting") |
| | 111 | + |
| | 112 | + # XXX: close handlers? |
| | 113 | + |
| | 114 | + self.info("done.") |
| | 115 | + |
| | 116 | + def fetch_recent_changes(self, last_id, last_timestamp, limit = None ): |
| | 117 | + if limit is None: |
| | 118 | + limit = self.rclimit |
| | 119 | + |
| | 120 | + u = self.api_url + "?format=xml&action=query&list=recentchanges&rcdir=older" |
| | 121 | + u += "&rcprop=%s&rclimit=%d" % ( self.rcprops, limit ) |
| | 122 | + |
| | 123 | + if last_timestamp: |
| | 124 | + u += "rcstart=%s" % ( last_timestamp, ) |
| | 125 | + |
| | 126 | + print u |
| | 127 | + x = urllib.urlopen( u ) |
| | 128 | + xml = x.read( self.rclimit * MAX_RC_XML_SIZE ) |
| | 129 | + |
| | 130 | + try: |
| | 131 | + dom = xmpp.simplexml.XML2Node( xml ) |
| | 132 | + if dom is None: |
| | 133 | + raise IOError( "Failed to parse changes. data: %s" % xml[:128] ) |
| | 134 | + |
| | 135 | + rcs = dom.T.query.T.recentchanges.getTags( 'rc' ) |
| | 136 | + |
| | 137 | + rcs.reverse() # oldest first! |
| | 138 | + |
| | 139 | + #skip redundant RC entries |
| | 140 | + i = 0 |
| | 141 | + while i < len(rcs) and int( rcs[i]['rcid'] ) <= last_id: |
| | 142 | + i += 1 |
| | 143 | + |
| | 144 | + rcs = rcs[i:] |
| | 145 | + |
| | 146 | + return rcs |
| | 147 | + |
| | 148 | + except ExpatError, e: |
| | 149 | + error_type, error_value, trbk = sys.exc_info() |
| | 150 | + raise IOError( "Failed to parse changes: <%s> %s; data: %s" % ( error_type, error_value, xml[:128] ) ) |
| | 151 | + |
| | 152 | + def dispatch_rc(self, rc): |
| | 153 | + for h in self.handlers: |
| | 154 | + try: |
| | 155 | + h( rc ) |
| | 156 | + except Exception, e: |
| | 157 | + error_type, error_value, trbk = sys.exc_info() |
| | 158 | + self.warn( "Error in handler: %s" % " ".join( traceback.format_exception( error_type, error_value, trbk ) ) ) |
| | 159 | + |
| | 160 | + def add_handler(self, handler): |
| | 161 | + self.handlers.append( handler ) |
| | 162 | + |
| | 163 | + def remove_handler(self, handler): |
| | 164 | + self.handlers.remove( handler ) |
| | 165 | + |
| | 166 | +class UDPSender(object): |
| | 167 | + def __init__(self, host, port): |
| | 168 | + self.address = ( host, port ) |
| | 169 | + self.socket = None |
| | 170 | + |
| | 171 | + def connect( self ): |
| | 172 | + self.sock = socket.socket( socket.AF_INET, socket.SOCK_DGRAM ) |
| | 173 | + self.sock.setsockopt( socket.SOL_SOCKET, socket.SO_REUSEADDR, 1 ) |
| | 174 | + self.sock.setblocking( 1 ) |
| | 175 | + |
| | 176 | + return True |
| | 177 | + |
| | 178 | + def send( self, data ): |
| | 179 | + if type(data) == unicode: |
| | 180 | + data = data.encode('utf-8') |
| | 181 | + |
| | 182 | + if type(data) != str: |
| | 183 | + data = data.__str__() |
| | 184 | + data = data.encode('utf-8') |
| | 185 | + |
| | 186 | + self.sock.sendto( data, 0, self.address ) |
| | 187 | + |
| | 188 | + def close( self ): |
| | 189 | + self.socket.close() |
| | 190 | + |
| | 191 | + def __call__(self, data ): |
| | 192 | + self.send( data ) |
| | 193 | + |
| | 194 | +################################################################################## |
| | 195 | + |
| | 196 | +if __name__ == '__main__': |
| | 197 | + |
| | 198 | + # -- CONFIG & COMMAND LINE ---------------------------------------------------------------------- |
| | 199 | + |
| | 200 | + # find the location of this script |
| | 201 | + bindir= os.path.dirname( os.path.realpath( sys.argv[0] ) ) |
| | 202 | + extdir= os.path.dirname( bindir ) |
| | 203 | + |
| | 204 | + # set up command line options........ |
| | 205 | + option_parser = optparse.OptionParser() |
| | 206 | + option_parser.set_usage( "usage: %prog [options] <api-url> <target-host> [port]" ) |
| | 207 | + |
| | 208 | + option_parser.add_option("--config", dest="config_file", |
| | 209 | + help="read config from FILE", metavar="FILE") |
| | 210 | + |
| | 211 | + option_parser.add_option("--quiet", action="store_const", dest="loglevel", const=LOG_QUIET, default=LOG_VERBOSE, |
| | 212 | + help="suppress informational messages, only print warnings and errors") |
| | 213 | + |
| | 214 | + option_parser.add_option("--debug", action="store_const", dest="loglevel", const=LOG_DEBUG, |
| | 215 | + help="print debug messages") |
| | 216 | + |
| | 217 | + (options, args) = option_parser.parse_args() |
| | 218 | + |
| | 219 | + # find config file........ |
| | 220 | + if options.config_file: |
| | 221 | + cfg = options.config_file #take it from --config |
| | 222 | + else: |
| | 223 | + cfg = extdir + "/../../rc2udp.ini" #installation root |
| | 224 | + |
| | 225 | + if not os.path.exists( cfg ): |
| | 226 | + cfg = extdir + "/../../phase3/rc2udp.ini" #installation root in dev environment |
| | 227 | + |
| | 228 | + if not os.path.exists( cfg ): |
| | 229 | + cfg = bindir + "/rc2udp.ini" #extension dir |
| | 230 | + |
| | 231 | + # define config defaults........ |
| | 232 | + config = ConfigParser.SafeConfigParser() |
| | 233 | + |
| | 234 | + config.add_section( 'rc2udp' ) |
| | 235 | + config.set( 'rc2udp', 'port', '4455' ) |
| | 236 | + config.set( 'rc2udp', 'poll-interval', '10' ) |
| | 237 | + config.set( 'rc2udp', 'rclimit', '500' ) |
| | 238 | + config.set( 'rc2udp', 'rcprops', 'user|comment|flags|timestamp|title|ids|sizes|redirect|loginfo|tags' ) |
| | 239 | + |
| | 240 | + # read config file........ |
| | 241 | + if not config.read( cfg ): |
| | 242 | + sys.stderr.write( "failed to read config from %s\n" % cfg ) |
| | 243 | + sys.exit(2) |
| | 244 | + |
| | 245 | + api = args[0] |
| | 246 | + host = args[1] |
| | 247 | + |
| | 248 | + if len(args) >= 3: |
| | 249 | + port = int( args[2] ) |
| | 250 | + else: |
| | 251 | + port = config.getint( 'rc2udp', 'port' ) |
| | 252 | + |
| | 253 | + # -- DO STUFF ----------------------------------------------------------------------------------- |
| | 254 | + |
| | 255 | + # create poller |
| | 256 | + poller = RCPoller( api, interval = config.getint( 'rc2udp', 'poll-interval' ), |
| | 257 | + rclimit = config.getint( 'rc2udp', 'rclimit' ), |
| | 258 | + rcprops = config.get( 'rc2udp', 'rcprops' ) ) |
| | 259 | + |
| | 260 | + poller.loglevel = options.loglevel |
| | 261 | + |
| | 262 | + # add an echo handler that prints the RC info to the shell |
| | 263 | + sender = UDPSender( host, port ) |
| | 264 | + sender.connect() |
| | 265 | + |
| | 266 | + poller.add_handler( sender ) |
| | 267 | + |
| | 268 | + # run listener loop................ |
| | 269 | + poller.poll_loop( ) |
| | 270 | + |
| | 271 | + print "done." |
| | 272 | + |
| \ No newline at end of file |
| Property changes on: trunk/extensions/XMLRC/generator/rc2udp.py |
| ___________________________________________________________________ |
| Added: svn:mergeinfo |
| Index: trunk/extensions/XMLRC/generator/.htaccess |
| — | — | @@ -0,0 +1 @@ |
| | 2 | +Deny From All |
| \ No newline at end of file |
| Index: trunk/extensions/XMLRC/generator/README |
| — | — | @@ -0,0 +1,21 @@ |
| | 2 | +This directory contains rc2udp.py, a program that polls the MediaWiki |
| | 3 | +web API for recent-changes records, and sends them on as UDP packets. |
| | 4 | + |
| | 5 | +This emuates the behavior of the XMLRC extension using UDP transport. |
| | 6 | +It exissts purely for testing, for production use, the XMLRC extension |
| | 7 | +should be configured to emit UDP packets from the wiki iteself. |
| | 8 | + |
| | 9 | +NOTE: You can place this directory anywhere on your system. In fact, for |
| | 10 | +security reasons, it is recommended to move it to a location that is not |
| | 11 | +accessible from the web or by the web server. |
| | 12 | + |
| | 13 | +To use rc2udp.py, you need python 2.5 and the xmpppy library from |
| | 14 | +<http://xmpppy.sourceforge.net/>. You also need an account on an XMPP |
| | 15 | +(Jabber) server for use by udp2xmpp.py. |
| | 16 | + |
| | 17 | +You can start rc2udp.py simply by typing: |
| | 18 | + |
| | 19 | + python rc2udp.py http://example.com/w/api.php 127.0.0.1 |
| | 20 | + |
| | 21 | +The second argument is the IP address to send to UDP packets to. you can |
| | 22 | +specify the port as a third parameter, the default port is 4455. |
| \ No newline at end of file |
| Property changes on: trunk/extensions/XMLRC/generator/README |
| ___________________________________________________________________ |
| Added: svn:mergeinfo |
| Index: trunk/extensions/XMLRC/generator/rc2udp.ini |
| — | — | @@ -0,0 +1,7 @@ |
| | 2 | +# Configuration file for rc2udp |
| | 3 | + |
| | 4 | +[rc2udp] |
| | 5 | +port: 4455 |
| | 6 | +poll-interval: 10 |
| | 7 | +rclimit: 500 |
| | 8 | +rcprops: user|comment|flags|timestamp|title|ids|sizes|redirect|loginfo|tags |
| Index: trunk/extensions/XMLRC/generator/rc2udp.ini.sample |
| — | — | @@ -0,0 +1,7 @@ |
| | 2 | +# Configuration file for rc2udp |
| | 3 | + |
| | 4 | +[rc2udp] |
| | 5 | +port: 4455 |
| | 6 | +poll-interval: 10 |
| | 7 | +rclimit: 500 |
| | 8 | +rcprops: user|comment|flags|timestamp|title|ids|sizes|redirect|loginfo|tags |
| Property changes on: trunk/extensions/XMLRC/generator/rc2udp.ini.sample |
| ___________________________________________________________________ |
| Added: svn:mergeinfo |
| Property changes on: trunk/extensions/XMLRC/generator |
| ___________________________________________________________________ |
| Added: svn:mergeinfo |