r51324 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r51323‎ | r51324 | r51325 >
Date:01:10, 2 June 2009
Author:rainman
Status:deferred
Tags:
Comment:
Just send the udp stuff right away, don't do it in batches, also add timestamps.
Modified paths:
  • /branches/lucene-search-2.1/src/org/wikimedia/lsearch/search/SearchEngine.java (modified) (history)
  • /branches/lucene-search-2.1/src/org/wikimedia/lsearch/statistics/UDPLogThread.java (deleted) (history)
  • /branches/lucene-search-2.1/src/org/wikimedia/lsearch/statistics/UDPLogger.java (added) (history)

Diff [purge]

Index: branches/lucene-search-2.1/src/org/wikimedia/lsearch/statistics/UDPLogThread.java
——@@ -1,118 +0,0 @@
2 -package org.wikimedia.lsearch.statistics;
3 -
4 -import java.io.IOException;
5 -import java.io.UnsupportedEncodingException;
6 -import java.net.DatagramPacket;
7 -import java.net.DatagramSocket;
8 -import java.net.InetAddress;
9 -import java.net.SocketException;
10 -import java.net.UnknownHostException;
11 -import java.util.ArrayList;
12 -import java.util.Random;
13 -
14 -import org.apache.log4j.Logger;
15 -import org.wikimedia.lsearch.config.Configuration;
16 -
17 -public class UDPLogThread extends Thread {
18 - Logger log = Logger.getLogger(UDPLogThread.class);
19 - static private UDPLogThread instance = null;
20 - protected int port = 51234;
21 - protected String host = "localhost";
22 - protected ArrayList<String> queryQueue = new ArrayList<String>();
23 - protected boolean disabled = true;
24 -
25 - protected Object lock = new Object();
26 -
27 - public static synchronized UDPLogThread getInstance(){
28 - if(instance == null){
29 - instance = new UDPLogThread();
30 - if(! instance.disabled )
31 - instance.start();
32 - }
33 - return instance;
34 - }
35 -
36 - private UDPLogThread(){
37 - Configuration config = Configuration.open();
38 - host = config.getString("UDPLogger", "host", null);
39 - port = config.getInt("UDPLogger", "port", 0);
40 -
41 - // do a sanity check, see if udp logging is enabled at all
42 - if(host!=null && port>0)
43 - disabled = false;
44 -
45 - }
46 -
47 - @Override
48 - public void run() {
49 - log.info("UDP logger started");
50 -
51 - Random r = new Random();
52 - InetAddress dest = null;
53 - try {
54 - dest = InetAddress.getByName(host);
55 - } catch (UnknownHostException e1) {
56 - log.error("Cannot find destination host "+host,e1);
57 - return;
58 - }
59 -
60 - DatagramSocket socket;
61 - DatagramPacket packet;
62 -
63 - while(true){
64 - try {
65 - socket = new DatagramSocket();
66 -
67 - // send queries in separate packets
68 - ArrayList<String> queries = getQueries();
69 - for(String q : queries){
70 - byte[] raw = q.getBytes("UTF-8");
71 - packet = new DatagramPacket(raw, raw.length, dest, port);
72 - socket.send(packet);
73 - }
74 -
75 - } catch (SocketException e1) {
76 - log.warn("Socket problem in connecting to host="+host+", port="+port, e1);
77 - } catch (UnsupportedEncodingException e) {
78 - log.error("",e);
79 - } catch (IOException e) {
80 - log.warn("IO Exception in connecting to host="+host+", port="+port, e);
81 - }
82 -
83 - // sleep between one and two seconds
84 - try {
85 - Thread.sleep(1000 + r.nextInt(1000));
86 - } catch (InterruptedException e) {
87 - log.warn("UDPLogThread interrupted", e);
88 - }
89 -
90 - }
91 - }
92 -
93 - /** Queue a query for logging */
94 - public void log(String dbname, String query){
95 - // if disabled just do nothing
96 - if(disabled)
97 - return;
98 -
99 - synchronized(lock){
100 - // make sure queries are of reasonable size
101 - if(query.length() > 1024)
102 - query = query.substring(0, 1024);
103 -
104 - // replace any newlines and such
105 - query = query.replace("\n", " ").replace("\r", " ");
106 -
107 - queryQueue.add(dbname+" "+query.trim()+"\n");
108 - }
109 - }
110 -
111 - /** Fetch queries so far and empty the waiting queue */
112 - protected ArrayList<String> getQueries(){
113 - synchronized(lock){
114 - ArrayList<String> ret = queryQueue;
115 - queryQueue = new ArrayList<String>();
116 - return ret;
117 - }
118 - }
119 -}
Index: branches/lucene-search-2.1/src/org/wikimedia/lsearch/statistics/UDPLogger.java
——@@ -0,0 +1,106 @@
 2+package org.wikimedia.lsearch.statistics;
 3+
 4+import java.io.IOException;
 5+import java.io.UnsupportedEncodingException;
 6+import java.net.DatagramPacket;
 7+import java.net.DatagramSocket;
 8+import java.net.InetAddress;
 9+import java.net.SocketException;
 10+import java.net.UnknownHostException;
 11+import java.text.SimpleDateFormat;
 12+import java.util.ArrayList;
 13+import java.util.Calendar;
 14+import java.util.Random;
 15+import java.util.TimeZone;
 16+
 17+import org.apache.log4j.Logger;
 18+import org.wikimedia.lsearch.config.Configuration;
 19+
 20+public class UDPLogger {
 21+ Logger log = Logger.getLogger(UDPLogger.class);
 22+ static private UDPLogger instance = null;
 23+ protected int port = 51234;
 24+ protected String host = "localhost";
 25+ protected ArrayList<String> queryQueue = new ArrayList<String>();
 26+ protected boolean disabled = true;
 27+ protected InetAddress dest;
 28+
 29+ protected Object lock = new Object();
 30+
 31+ public static synchronized UDPLogger getInstance(){
 32+ if(instance == null){
 33+ instance = new UDPLogger();
 34+ }
 35+ return instance;
 36+ }
 37+
 38+ private UDPLogger(){
 39+ Configuration config = Configuration.open();
 40+ host = config.getString("UDPLogger", "host", null);
 41+ port = config.getInt("UDPLogger", "port", 0);
 42+
 43+ // do a sanity check, see if udp logging is enabled at all
 44+ if(host!=null && port>0)
 45+ disabled = false;
 46+
 47+ dest = null;
 48+ try {
 49+ dest = InetAddress.getByName(host);
 50+ } catch (UnknownHostException e1) {
 51+ log.error("Cannot find destination host "+host,e1);
 52+ return;
 53+ }
 54+
 55+ }
 56+
 57+ protected void sendLog(String query){
 58+ DatagramSocket socket;
 59+ DatagramPacket packet;
 60+
 61+ try{
 62+ socket = new DatagramSocket();
 63+
 64+ byte[] raw = query.getBytes("UTF-8");
 65+ packet = new DatagramPacket(raw, raw.length, dest, port);
 66+ socket.send(packet);
 67+ } catch (SocketException e1) {
 68+ log.warn("Socket problem in connecting to host="+host+", port="+port, e1);
 69+ } catch (UnsupportedEncodingException e) {
 70+ log.error("",e);
 71+ } catch (IOException e) {
 72+ log.warn("IO Exception in connecting to host="+host+", port="+port, e);
 73+ }
 74+ }
 75+
 76+ /** Queue a query for logging */
 77+ public void log(String dbname, String query){
 78+ // if disabled just do nothing
 79+ if(disabled)
 80+ return;
 81+
 82+ // output in iso8601
 83+ Calendar currentDate = Calendar.getInstance();
 84+ SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
 85+ String date = formatter.format(currentDate.getTime());
 86+
 87+ synchronized(lock){
 88+ // make sure queries are of reasonable size
 89+ if(query.length() > 1024)
 90+ query = query.substring(0, 1024);
 91+
 92+ // replace any newlines and such
 93+ query = query.replace("\n", " ").replace("\r", " ");
 94+
 95+ sendLog(dbname+" "+date+" "+query.trim()+"\n");
 96+ }
 97+ }
 98+
 99+ /** Fetch queries so far and empty the waiting queue */
 100+ protected ArrayList<String> getQueries(){
 101+ synchronized(lock){
 102+ ArrayList<String> ret = queryQueue;
 103+ queryQueue = new ArrayList<String>();
 104+ return ret;
 105+ }
 106+ }
 107+}
Index: branches/lucene-search-2.1/src/org/wikimedia/lsearch/search/SearchEngine.java
——@@ -56,7 +56,7 @@
5757 import org.wikimedia.lsearch.spell.SuggestQuery;
5858 import org.wikimedia.lsearch.spell.SuggestResult;
5959 import org.wikimedia.lsearch.spell.SuggestSimilar;
60 -import org.wikimedia.lsearch.statistics.UDPLogThread;
 60+import org.wikimedia.lsearch.statistics.UDPLogger;
6161 import org.wikimedia.lsearch.util.Localization;
6262
6363 /**
——@@ -75,7 +75,7 @@
7676 protected static GlobalConfiguration global = null;
7777 protected static Configuration config = null;
7878 protected static SearcherCache cache = null;
79 - protected static UDPLogThread udpLogger = null;
 79+ protected static UDPLogger udpLogger = null;
8080 /** dbname -> ns_string -> ns_index */
8181 protected static Hashtable<String,Hashtable<String,Integer>> dbNamespaces = new Hashtable<String,Hashtable<String,Integer>>();
8282 /** dbname -> ns_index -> ns_string */
——@@ -89,7 +89,7 @@
9090 if(cache == null)
9191 cache = SearcherCache.getInstance();
9292 if(udpLogger == null)
93 - udpLogger = UDPLogThread.getInstance();
 93+ udpLogger = UDPLogger.getInstance();
9494
9595 // timelimit = config.getInt("Search","timelimit",5000);
9696 }

Status & tagging log