| Index: trunk/squid-log/page-views |
| — | — | @@ -0,0 +1,53 @@ |
| | 2 | +#!/usr/bin/python |
| | 3 | +import sys, re, urlparse, time, ipaddr |
| | 4 | + |
| | 5 | +localNetworkSet = ipaddr.ip_set_from_list([ |
| | 6 | + ipaddr.network('66.230.200.0', 24), |
| | 7 | + ipaddr.network('145.97.39.128', 26), |
| | 8 | + ipaddr.network('211.115.107.128', 26), |
| | 9 | + ipaddr.network('10.0.0.0', 16) ]) |
| | 10 | + |
| | 11 | +reportTime = time.time() |
| | 12 | +numPageViews = 0 |
| | 13 | +numRequests = 0 |
| | 14 | +interval = 5 |
| | 15 | +line = sys.stdin.readline() |
| | 16 | +while line != '': |
| | 17 | + line = line.rstrip("\n") |
| | 18 | + fields = line.split() |
| | 19 | + if len(fields) != 11: |
| | 20 | + sys.stderr.write("Invalid line, got %d fields, expecting 11\n" % len(fields)) |
| | 21 | + sys.stderr.write(line + "\n") |
| | 22 | + sys.stderr.flush() |
| | 23 | + line = sys.stdin.readline() |
| | 24 | + continue |
| | 25 | + try: |
| | 26 | + delta = int(fields[0]) |
| | 27 | + except: |
| | 28 | + delta = -1 |
| | 29 | + if delta < 0 or delta > 1000000: |
| | 30 | + sys.stderr.write("Invalid delta\n") |
| | 31 | + sys.stderr.flush() |
| | 32 | + |
| | 33 | + url = urlparse.urlsplit(fields[7]) |
| | 34 | + if fields[6] == "GET" and \ |
| | 35 | + url[2].startswith('/wiki/') and \ |
| | 36 | + url[4] == '' and \ |
| | 37 | + fields[4].endswith('/200') and \ |
| | 38 | + ipaddr.ipaddr(fields[3]) not in localNetworkSet: |
| | 39 | + numPageViews += delta |
| | 40 | + numRequests += delta |
| | 41 | + |
| | 42 | + currentTime = time.time() |
| | 43 | + if currentTime - reportTime > interval: |
| | 44 | + elapsed = currentTime - reportTime |
| | 45 | + print "req/s: %.2f, pages/s: %.2f" % ( |
| | 46 | + numRequests / elapsed, |
| | 47 | + numPageViews / elapsed) |
| | 48 | + reportTime = currentTime |
| | 49 | + numPageViews = 0 |
| | 50 | + numRequests = 0 |
| | 51 | + |
| | 52 | + line = sys.stdin.readline() |
| | 53 | + |
| | 54 | +# vim: set ts=4 sw=4: |
| Property changes on: trunk/squid-log/page-views |
| ___________________________________________________________________ |
| Added: svn:executable |
| 1 | 55 | + * |
| Index: trunk/squid-log/mime-types |
| — | — | @@ -0,0 +1,51 @@ |
| | 2 | +#!/usr/bin/python |
| | 3 | +import sys, re, urlparse, time, ipaddr |
| | 4 | + |
| | 5 | +localNetworkSet = ipaddr.ip_set_from_list([ |
| | 6 | + ipaddr.network('66.230.200.0', 24), |
| | 7 | + ipaddr.network('145.97.39.128', 26), |
| | 8 | + ipaddr.network('211.115.107.128', 26), |
| | 9 | + ipaddr.network('10.0.0.0', 16) ]) |
| | 10 | + |
| | 11 | +reportTime = time.time() |
| | 12 | +interval = 60 |
| | 13 | +line = sys.stdin.readline() |
| | 14 | +mimeTypeCounts = {} |
| | 15 | +total = 0 |
| | 16 | +while line != '': |
| | 17 | + line = line.rstrip("\n") |
| | 18 | + fields = line.split() |
| | 19 | + if len(fields) != 11: |
| | 20 | + sys.stderr.write("Invalid line, got %d fields, expecting 11\n" % len(fields)) |
| | 21 | + sys.stderr.flush() |
| | 22 | + line = sys.stdin.readline() |
| | 23 | + continue |
| | 24 | + try: |
| | 25 | + delta = int(fields[0]) |
| | 26 | + except: |
| | 27 | + delta = -1 |
| | 28 | + if delta < 0 or delta > 1000000: |
| | 29 | + sys.stderr.write("Invalid delta\n") |
| | 30 | + sys.stderr.flush() |
| | 31 | + |
| | 32 | + if fields[10] not in mimeTypeCounts: |
| | 33 | + mimeTypeCounts[fields[10]] = 0 |
| | 34 | + mimeTypeCounts[fields[10]] += delta |
| | 35 | + total += delta |
| | 36 | + |
| | 37 | + currentTime = time.time() |
| | 38 | + if currentTime - reportTime > interval: |
| | 39 | + elapsed = currentTime - reportTime |
| | 40 | + reportTime = currentTime |
| | 41 | + |
| | 42 | + print "\x0c", |
| | 43 | + s = list(mimeTypeCounts.items()) |
| | 44 | + s.sort(cmp=lambda x,y:cmp(y[1],x[1])) |
| | 45 | + for type, count in s: |
| | 46 | + print "%-30s %2.1f" % (type, float(count)/total*100) |
| | 47 | + mimeTypeCounts = {} |
| | 48 | + total = 0 |
| | 49 | + |
| | 50 | + line = sys.stdin.readline() |
| | 51 | + |
| | 52 | +# vim: set ts=4 sw=4: |
| Property changes on: trunk/squid-log/mime-types |
| ___________________________________________________________________ |
| Added: svn:executable |
| 1 | 53 | + * |
| Index: trunk/squid-log/udprecv.cpp |
| — | — | @@ -43,6 +43,7 @@ |
| 44 | 44 | if (buffer[bytesRead - 1] != '\n') { |
| 45 | 45 | cout << "\n"; |
| 46 | 46 | } |
| | 47 | + cout.flush(); |
| 47 | 48 | } |
| 48 | 49 | } |
| 49 | 50 | } |
| Index: trunk/squid-log/Makefile |
| — | — | @@ -1,7 +1,7 @@ |
| 2 | | -all: log2udp udprecv |
| | 2 | +all: log2udp udprecv delta |
| 3 | 3 | |
| 4 | 4 | clean: |
| 5 | | - rm -f *.o log2udp host |
| | 5 | + rm -f *.o log2udp host udprecv delta |
| 6 | 6 | |
| 7 | 7 | HOST_OBJS = host.o HostEntry.o IPAddress.o |
| 8 | 8 | LOG2UDP_OBJS = log2udp.o HostEntry.o IPAddress.o Socket.o SocketAddress.o |
| — | — | @@ -20,3 +20,5 @@ |
| 21 | 21 | udprecv: $(UDPRECV_OBJS) |
| 22 | 22 | g++ $(CFLAGS) -Wall $(UDPRECV_OBJS) -o udprecv |
| 23 | 23 | |
| | 24 | +delta: delta.cpp |
| | 25 | + g++ $(CFLAGS) -o delta delta.cpp |