| Index: trunk/udpprofile/debian/udpprofile.init |
| — | — | @@ -16,6 +16,8 @@ |
| 17 | 17 | PATH=/sbin:/usr/sbin:/bin:/usr/bin |
| 18 | 18 | DESC="UDP Profile Collector" |
| 19 | 19 | DAEMON="/usr/udpprofile/sbin/collector" |
| | 20 | +DAEMON2="/usr/udpprofile/sbin/profiler-to-carbon" |
| | 21 | + |
| 20 | 22 | USER="nobody" |
| 21 | 23 | SCRIPTNAME=/etc/init.d/udpprofile |
| 22 | 24 | |
| — | — | @@ -45,6 +47,14 @@ |
| 46 | 48 | || return 1 |
| 47 | 49 | start-stop-daemon --start --quiet -m --pidfile /var/lock/collector -d /tmp -c $USER --exec $DAEMON \ |
| 48 | 50 | || return 2 |
| | 51 | + |
| | 52 | + if [ -x "$DAEMON2" ] ; then |
| | 53 | + start-stop-daemon --start --quiet -m --pidfile /var/lock/profiler-to-carbon -d /tmp -b -c $USER \ |
| | 54 | + --exec $DAEMON2 --test > /dev/null || return 1 |
| | 55 | + start-stop-daemon --start --quiet -m --pidfile /var/lock/profiler-to-carbon -d /tmp -b -c $USER \ |
| | 56 | + --exec $DAEMON2 > /dev/null || return 2 |
| | 57 | + fi |
| | 58 | + |
| 49 | 59 | # Add code here, if necessary, that waits for the process to be ready |
| 50 | 60 | # to handle requests from services started subsequently which depend |
| 51 | 61 | # on this one. As a last resort, sleep for some time. |
| — | — | @@ -62,6 +72,7 @@ |
| 63 | 73 | # other if a failure occurred |
| 64 | 74 | start-stop-daemon --stop --quiet -c $USER --exec $DAEMON |
| 65 | 75 | RETVAL="$?" |
| | 76 | + start-stop-daemon --stop --quiet -c $USER --exec $DAEMON2 |
| 66 | 77 | [ "$RETVAL" = 2 ] && return 2 |
| 67 | 78 | return "$RETVAL" |
| 68 | 79 | } |
| Index: trunk/udpprofile/debian/install |
| — | — | @@ -1,5 +1,6 @@ |
| 2 | 2 | collector /usr/udpprofile/sbin |
| 3 | 3 | exporter /usr/udpprofile/sbin |
| | 4 | +profiler-to-carbon /usr/udpprofile/sbin |
| 4 | 5 | web /usr/udpprofile/web |
| 5 | 6 | web/admin.py /usr/udpprofile/web |
| 6 | 7 | web/config.py /usr/udpprofile/web |
| Index: trunk/udpprofile/debian/changelog |
| — | — | @@ -1,3 +1,10 @@ |
| | 2 | +udpprofile (1.1) lucid-wikimedia; urgency=low |
| | 3 | + |
| | 4 | + * profiler-to-carbon - realtime graphing of rate and timing in ms of |
| | 5 | + profiling data in the all and stats/all dbs |
| | 6 | + |
| | 7 | + -- Asher Feldman <afeldman@wikimedia.org> Wed, 11 Oct 2011 21:55:00 +0000 |
| | 8 | + |
| 2 | 9 | udpprofile (1.0-2) lucid-wikimedia; urgency=low |
| 3 | 10 | |
| 4 | 11 | * Fix init script agin |
| Index: trunk/udpprofile/profiler-to-carbon |
| — | — | @@ -0,0 +1,109 @@ |
| | 2 | +#!/usr/bin/python |
| | 3 | + |
| | 4 | +# udpprofile to graphite - uses collector as an aggregator |
| | 5 | +# for MediaWiki profiling stats, feeding all data in the |
| | 6 | +# all and stats/all db's into carbon, in one minute moving |
| | 7 | +# averages. |
| | 8 | + |
| | 9 | +# this should be wherever extractprofile.py is |
| | 10 | +sys.path.append( '/usr/udpprofile/web' ) |
| | 11 | + |
| | 12 | +from extractprofile import ExtractProfile |
| | 13 | +import time |
| | 14 | +import socket |
| | 15 | +import re |
| | 16 | +import sys |
| | 17 | +import logging |
| | 18 | + |
| | 19 | +profile_host = "spence.wikimedia.org" |
| | 20 | +profile_port = 3811 |
| | 21 | +carbon_host = '127.0.0.1' |
| | 22 | +carbon_port = 2003 |
| | 23 | +dbs = ('all', 'stats/all') |
| | 24 | +delay = 60 |
| | 25 | + |
| | 26 | +invalid = re.compile ( '[^\w\-]+' ) |
| | 27 | +prior={} |
| | 28 | + |
| | 29 | +class SocketSource (socket.socket): |
| | 30 | + def read(self,what): |
| | 31 | + enc = self.recv(what,0) |
| | 32 | + return enc.decode('latin-1').encode('utf-8') |
| | 33 | + |
| | 34 | +def BuildStats(db, fullprofile): |
| | 35 | + stats = {} |
| | 36 | + events=fullprofile[db]["-"].items() |
| | 37 | + |
| | 38 | + for event in events: |
| | 39 | + if "close" in event[0]: continue |
| | 40 | + if "Profiling error" in event[0]: continue |
| | 41 | + # . is the graphite path separator |
| | 42 | + # stats are sent as stats.$stat, regular functions with |
| | 43 | + # up to two path levels (i.e. API::Foo::Bar = API.Foo.Bar) |
| | 44 | + if (db.startswith('stats')): |
| | 45 | + name = 'stats.'+invalid.sub('_', str(event[0])).rstrip('_') |
| | 46 | + else: |
| | 47 | + name = invalid.sub('_', str(event[0])).rstrip('_').replace('_', '.', 2) |
| | 48 | + stats[name] = {} |
| | 49 | + stats[name]['count'] = event[1]['count'] |
| | 50 | + # real = time in ms |
| | 51 | + stats[name]['real'] = event[1]['real']*1000 |
| | 52 | + |
| | 53 | + return stats |
| | 54 | + |
| | 55 | +def SendStats(db, current, graph): |
| | 56 | + now = int(time.time()) |
| | 57 | + message = "" |
| | 58 | + for key in current.keys(): |
| | 59 | + if key not in prior[db]: continue |
| | 60 | + count = current[key]['count'] - prior[db][key]['count'] |
| | 61 | + |
| | 62 | + if (count <= 0): continue |
| | 63 | + |
| | 64 | + message = "%s.count %d %d\n" % ( key, count, now ) |
| | 65 | + logging.debug("sending: %s", message) |
| | 66 | + graph.send(message) |
| | 67 | + |
| | 68 | + if (db.startswith('stats')): continue |
| | 69 | + |
| | 70 | + real = current[key]['real'] - prior[db][key]['real'] |
| | 71 | + message = "%s.time %.3g %d\n" % ( key, real/count, now ) |
| | 72 | + logging.debug("sending: %s", message) |
| | 73 | + graph.send(message) |
| | 74 | + |
| | 75 | +while True: |
| | 76 | + profsock=SocketSource() |
| | 77 | + try: |
| | 78 | + profsock.connect((profile_host,profile_port)) |
| | 79 | + except: |
| | 80 | + logging.debug ("Couldn't connect to %s on port %d, is collector running?", \ |
| | 81 | + profile_host, profile_port) |
| | 82 | + time.sleep(delay) |
| | 83 | + continue |
| | 84 | + |
| | 85 | + graph = socket.socket() |
| | 86 | + try: |
| | 87 | + graph.connect((carbon_host,carbon_port)) |
| | 88 | + except: |
| | 89 | + logging.debug ("Couldn't connect to %s on port %d, is carbon-agent.py running?", \ |
| | 90 | + carbon_host, profile_port) |
| | 91 | + time.sleep(delay) |
| | 92 | + continue |
| | 93 | + |
| | 94 | + fullprofile=ExtractProfile().extract(profsock) |
| | 95 | + profsock.shutdown(socket.SHUT_RDWR) |
| | 96 | + profsock.close() |
| | 97 | + |
| | 98 | + current={} |
| | 99 | + |
| | 100 | + for db in dbs: |
| | 101 | + current[db] = BuildStats(db, fullprofile) |
| | 102 | + if (len(current[db]) > 1): |
| | 103 | + try: |
| | 104 | + SendStats(db, current[db], graph) |
| | 105 | + except: |
| | 106 | + logging.debug("error sending stats") |
| | 107 | + continue |
| | 108 | + |
| | 109 | + prior = current |
| | 110 | + time.sleep(delay) |
| Property changes on: trunk/udpprofile/profiler-to-carbon |
| ___________________________________________________________________ |
| Added: svn:executable |
| 1 | 111 | + * |