r19540 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r19539‎ | r19540 | r19541 >
Date:01:04, 21 January 2007
Author:tstarling
Status:old
Tags:
Comment:
* Fixed EAGAIN issue in pipe logging
* Added UDP logging
Modified paths:
  • /trunk/debs/squid/debian/changelog (modified) (history)
  • /trunk/debs/squid/debian/log2udp_wm (deleted) (history)
  • /trunk/debs/squid/debian/patches/00list (modified) (history)
  • /trunk/debs/squid/debian/patches/22-pipelog.dpatch (deleted) (history)
  • /trunk/debs/squid/debian/patches/22-udplog.dpatch (added) (history)
  • /trunk/debs/squid/debian/rules (modified) (history)

Diff [purge]

Index: trunk/debs/squid/debian/log2udp_wm
——@@ -1,3 +0,0 @@
2 -#!/bin/sh
3 -# Change this file to change the squid logging host
4 -exec /usr/bin/log2udp -h henbane.yaseo.wikimedia.org
Index: trunk/debs/squid/debian/patches/22-pipelog.dpatch
——@@ -1,79 +0,0 @@
2 -#! /bin/sh /usr/share/dpatch/dpatch-run
3 -## pipelog.dpatch by <tstarling@wikimedia.org>
4 -##
5 -## All lines beginning with `## DP:' are a description of the patch.
6 -## DP: Access logging to pipe
7 -
8 -@DPATCH@
9 -
10 -diff -ru squid-2.6.5/src/logfile.c squid-2.6.5-pipelog/src/logfile.c
11 -+++ squid-2.6.5-pipelog/src/logfile.c 2007-01-16 17:34:33.000000000 +0000
12 -@@ -35,6 +35,7 @@
13 - #include "squid.h"
14 -
15 - static void logfileWriteWrapper(Logfile * lf, const void *buf, size_t len);
16 -+static void logfileOpenPipe(Logfile * lf);
17 -
18 - #if HAVE_SYSLOG
19 -
20 -@@ -137,7 +138,10 @@
21 - lf->syslog_priority |= LOG_INFO;
22 - } else
23 - #endif
24 -- {
25 -+ if (path[0] == '|') {
26 -+ lf->flags.isipc = 1;
27 -+ logfileOpenPipe(lf);
28 -+ } else {
29 - int fd = file_open(path, O_WRONLY | O_CREAT | O_TEXT);
30 - if (DISK_ERROR == fd) {
31 - if (ENOENT == errno && fatal_flag) {
32 -@@ -186,9 +190,17 @@
33 - int i;
34 - char from[MAXPATHLEN];
35 - char to[MAXPATHLEN];
36 -- assert(lf->path);
37 -+
38 - if (lf->flags.syslog)
39 - return;
40 -+
41 -+ if (lf->flags.isipc) {
42 -+ file_close(lf->fd);
43 -+ logfileOpenPipe(lf);
44 -+ return;
45 -+ }
46 -+
47 -+ assert(lf->path);
48 - #ifdef S_ISREG
49 - if (stat(lf->path, &sb) == 0)
50 - if (S_ISREG(sb.st_mode) == 0)
51 -@@ -302,3 +314,17 @@
52 - return;
53 - fatalf("logfileWrite: %s: %s\n", lf->path, xstrerror());
54 - }
55 -+
56 -+/*
57 -+ * Open a pipe for logging
58 -+ */
59 -+static void
60 -+logfileOpenPipe(Logfile * lf)
61 -+{
62 -+ const char *args[2];
63 -+ args[0] = "(logger)";
64 -+ args[1] = NULL;
65 -+ if ( ipcCreate(IPC_FIFO,&(lf->path[1]),args,"logger",NULL,&(lf->fd),NULL) <= 0 ) {
66 -+ fatalf("Cannot start log listener %s :s", lf->path, xstrerror());
67 -+ }
68 -+}
69 -diff -ru squid-2.6.5/src/structs.h squid-2.6.5-pipelog/src/structs.h
70 -+++ squid-2.6.5-pipelog/src/structs.h 2007-01-16 16:46:59.000000000 +0000
71 -@@ -2439,6 +2439,7 @@
72 - struct {
73 - unsigned int fatal;
74 - unsigned int syslog;
75 -+ unsigned int isipc;
76 - } flags;
77 - int syslog_priority;
78 - };
Index: trunk/debs/squid/debian/patches/00list
——@@ -5,4 +5,4 @@
66 06-unlinkd
77 20-wikimedia-errors
88 21-nomanglerequestheaders
9 -22-pipelog
 9+22-udplog
Index: trunk/debs/squid/debian/patches/22-udplog.dpatch
——@@ -0,0 +1,367 @@
 2+#! /bin/sh /usr/share/dpatch/dpatch-run
 3+## 22-udplog.dpatch by Tim Starling <tstarling@wikimedia.org>
 4+##
 5+## All lines beginning with `## DP:' are a description of the patch.
 6+## DP: No description.
 7+
 8+@DPATCH@
 9+diff -ru squid-2.6.5/src/access_log.c squid-2.6.5-pipelog/src/access_log.c
 10+--- squid-2.6.5/src/access_log.c 2006-09-28 21:55:37.000000000 +0000
 11+@@ -341,6 +341,8 @@
 12+
 13+ LFT_EXT_LOG,
 14+
 15++ LFT_SEQUENCE_NUMBER,
 16++
 17+ LFT_PERCENT /* special string cases for escaped chars */
 18+ } logformat_bcode_t;
 19+
 20+@@ -443,6 +445,8 @@
 21+ /*{ "<sb", LFT_REPLY_SIZE_BODY }, */
 22+ /*{ "<sB", LFT_REPLY_SIZE_BODY_NO_TE }, */
 23+
 24++ {"sn", LFT_SEQUENCE_NUMBER},
 25++
 26+ {"%", LFT_PERCENT},
 27+
 28+ {NULL, LFT_NONE} /* this must be last */
 29+@@ -457,7 +461,9 @@
 30+ static MemBuf mb = MemBufNULL;
 31+ char tmp[1024];
 32+ String sb = StringNull;
 33++ static long int sequence_number = 0;
 34+
 35++ sequence_number++;
 36+ memBufReset(&mb);
 37+
 38+ lf = log->logFormat;
 39+@@ -678,6 +684,10 @@
 40+ quote = 1;
 41+ break;
 42+
 43++ case LFT_SEQUENCE_NUMBER:
 44++ outint = sequence_number;
 45++ doint = 1;
 46++ break;
 47+ case LFT_PERCENT:
 48+ out = "%";
 49+ break;
 50+@@ -1151,7 +1161,7 @@
 51+ fatalf("Unknown log format %d\n", log->type);
 52+ break;
 53+ }
 54+- logfileFlush(log->logfile);
 55++ /*logfileFlush(log->logfile);*/
 56+ if (!checklist)
 57+ break;
 58+ }
 59+diff -ru squid-2.6.5/src/cache_cf.c squid-2.6.5-pipelog/src/cache_cf.c
 60+--- squid-2.6.5/src/cache_cf.c 2006-09-30 21:01:08.000000000 +0000
 61+@@ -2602,42 +2602,16 @@
 62+ return PEER_SIBLING;
 63+ }
 64+
 65++
 66+ #if USE_WCCPv2
 67+ static void
 68+ parse_sockaddr_in_list(sockaddr_in_list ** head)
 69+ {
 70+ char *token;
 71+- char *t;
 72+- char *host;
 73+- char *tmp;
 74+- const struct hostent *hp;
 75+- unsigned short port = 0;
 76+ sockaddr_in_list *s;
 77+ while ((token = strtok(NULL, w_space))) {
 78+- host = NULL;
 79+- port = 0;
 80+- if ((t = strchr(token, ':'))) {
 81+- /* host:port */
 82+- host = token;
 83+- *t = '\0';
 84+- port = xatos(t + 1);
 85+- if (0 == port)
 86+- self_destruct();
 87+- } else if ((port = strtol(token, &tmp, 10)), !*tmp) {
 88+- /* port */
 89+- } else {
 90+- host = token;
 91+- port = 0;
 92+- }
 93+ s = xcalloc(1, sizeof(*s));
 94+- s->s.sin_port = htons(port);
 95+- if (NULL == host)
 96+- s->s.sin_addr = any_addr;
 97+- else if (1 == safe_inet_addr(host, &s->s.sin_addr))
 98+- (void) 0;
 99+- else if ((hp = gethostbyname(host))) /* dont use ipcache */
 100+- s->s.sin_addr = inaddrFromHostent(hp);
 101+- else
 102++ if (!parse_sockaddr(token, &s->s))
 103+ self_destruct();
 104+ while (*head)
 105+ head = &(*head)->next;
 106+diff -ru squid-2.6.5/src/logfile.c squid-2.6.5-pipelog/src/logfile.c
 107+--- squid-2.6.5/src/logfile.c 2007-01-20 00:07:19.000000000 +0000
 108+@@ -35,6 +35,8 @@
 109+ #include "squid.h"
 110+
 111+ static void logfileWriteWrapper(Logfile * lf, const void *buf, size_t len);
 112++static int logfileOpenPipe(Logfile * lf);
 113++static int logfileOpenUDP(Logfile * lf);
 114+
 115+ #if HAVE_SYSLOG
 116+
 117+@@ -120,10 +122,16 @@
 118+ {
 119+ Logfile *lf = xcalloc(1, sizeof(*lf));
 120+ xstrncpy(lf->path, path, MAXPATHLEN);
 121++ if (bufsz > 0) {
 122++ lf->bufsz = bufsz;
 123++ }
 124++ if (fatal_flag)
 125++ lf->flags.fatal = 1;
 126+ #if HAVE_SYSLOG
 127+ if (strcmp(path, "syslog") == 0 || strncmp(path, "syslog:", 7) == 0) {
 128+ lf->flags.syslog = 1;
 129+ lf->fd = -1;
 130++ lf->bufsz = 0;
 131+ if (path[6] != '\0') {
 132+ const char *priority = path + 7;
 133+ char *facility = (char *) strchr(priority, '|');
 134+@@ -137,7 +145,20 @@
 135+ lf->syslog_priority |= LOG_INFO;
 136+ } else
 137+ #endif
 138+- {
 139++ if (strncmp(path, "udp:", 4) == 0) {
 140++ lf->flags.udp = 1;
 141++ /* Open a UDP socket, may change lf->bufsize */
 142++ if (!logfileOpenUDP(lf)) {
 143++ safe_free(lf);
 144++ return NULL;
 145++ }
 146++ } else if (path[0] == '|') {
 147++ lf->flags.pipe = 1;
 148++ if (!logfileOpenPipe(lf)) {
 149++ safe_free(lf);
 150++ return NULL;
 151++ }
 152++ } else {
 153+ int fd = file_open(path, O_WRONLY | O_CREAT | O_TEXT);
 154+ if (DISK_ERROR == fd) {
 155+ if (ENOENT == errno && fatal_flag) {
 156+@@ -156,13 +177,10 @@
 157+ }
 158+ }
 159+ lf->fd = fd;
 160+- if (bufsz > 0) {
 161+- lf->buf = xmalloc(bufsz);
 162+- lf->bufsz = bufsz;
 163+- }
 164+ }
 165+- if (fatal_flag)
 166+- lf->flags.fatal = 1;
 167++ if (lf->bufsz > 0) {
 168++ lf->buf = xmalloc(lf->bufsz);
 169++ }
 170+ return lf;
 171+ }
 172+
 173+@@ -186,9 +204,17 @@
 174+ int i;
 175+ char from[MAXPATHLEN];
 176+ char to[MAXPATHLEN];
 177+- assert(lf->path);
 178+- if (lf->flags.syslog)
 179++
 180++ if (lf->flags.syslog || lf->flags.udp)
 181+ return;
 182++
 183++ if (lf->flags.pipe) {
 184++ file_close(lf->fd);
 185++ logfileOpenPipe(lf);
 186++ return;
 187++ }
 188++
 189++ assert(lf->path);
 190+ #ifdef S_ISREG
 191+ if (stat(lf->path, &sb) == 0)
 192+ if (S_ISREG(sb.st_mode) == 0)
 193+@@ -298,7 +324,100 @@
 194+ fd_bytes(lf->fd, s, FD_WRITE);
 195+ if (s == len)
 196+ return;
 197++ if (errno == EAGAIN) {
 198++ /* This happens sometimes at high request rates when pipe logging
 199++ * is enabled. The pipe buffer is very small.
 200++ */
 201++ debug(50, 1)("Log line lost due to full transmit buffer\n");
 202++ return;
 203++ }
 204++ if (lf->flags.udp) {
 205++ /*
 206++ * Ignore network errors
 207++ */
 208++ return;
 209++ }
 210++
 211+ if (!lf->flags.fatal)
 212+ return;
 213+ fatalf("logfileWrite: %s: %s\n", lf->path, xstrerror());
 214+ }
 215++
 216++/*
 217++ * Open a pipe for logging
 218++ */
 219++static int
 220++logfileOpenPipe(Logfile * lf)
 221++{
 222++ const char *args[2];
 223++ args[0] = "(logger)";
 224++ args[1] = NULL;
 225++ if ( ipcCreate(IPC_FIFO,&(lf->path[1]),args,"logger",NULL,&(lf->fd),NULL) <= 0 ) {
 226++ if (lf->flags.fatal) {
 227++ fatalf("Cannot start log listener %s: %s\n", lf->path, xstrerror());
 228++ } else {
 229++ debug(50,1)("Cannot start log listener %s: %s\n", lf->path, xstrerror());
 230++ return FALSE;
 231++ }
 232++ }
 233++ return TRUE;
 234++}
 235++
 236++/**
 237++ * Open a UDP socket for logging
 238++ * May change lf->bufsize
 239++ */
 240++static int
 241++logfileOpenUDP(Logfile * lf)
 242++{
 243++ struct sockaddr_in addr;
 244++ char *strAddr;
 245++ if (strncmp(lf->path + 4, "//", 2) == 0) {
 246++ strAddr = xstrdup(lf->path + 6);
 247++ } else {
 248++ strAddr = xstrdup(lf->path + 4);
 249++ }
 250++ if (!parse_sockaddr(strAddr, &addr)) {
 251++ if (lf->flags.fatal) {
 252++ fatalf("Invalid UDP logging address '%s'\n", lf->path);
 253++ } else {
 254++ debug(50,1)("Invalid UDP logging address '%s'\n", lf->path);
 255++ safe_free(strAddr);
 256++ return FALSE;
 257++ }
 258++ }
 259++ safe_free(strAddr);
 260++
 261++ lf->fd = comm_open(SOCK_DGRAM,
 262++ IPPROTO_UDP,
 263++ no_addr,
 264++ 0,
 265++ COMM_NONBLOCKING,
 266++ "UDP log socket");
 267++ if (lf->fd < 0) {
 268++ if (lf->flags.fatal) {
 269++ fatalf("Unable to open UDP socket for logging\n");
 270++ } else {
 271++ debug(50,1)("Unable to open UDP socket for logging\n");
 272++ return FALSE;
 273++ }
 274++ }
 275++ if (comm_connect_addr(lf->fd, &addr)) {
 276++ if (lf->flags.fatal) {
 277++ fatalf("Unable to connect to %s for UDP log: %s\n", lf->path, xstrerror());
 278++ } else {
 279++ debug(50,1)("Unable to connect to %s for UDP log: %s\n", lf->path, xstrerror());
 280++ return FALSE;
 281++ }
 282++ }
 283++
 284++ /* Set the buffer size roughly equal to the MTU
 285++ * This will merge log lines into as few packets as possible
 286++ *
 287++ * TODO: make this configurable, some users may want it to be zero
 288++ */
 289++ lf->bufsz = 1450;
 290++ return TRUE;
 291++}
 292++
 293++
 294+diff -ru squid-2.6.5/src/protos.h squid-2.6.5-pipelog/src/protos.h
 295+--- squid-2.6.5/src/protos.h 2006-10-23 11:22:21.000000000 +0000
 296+@@ -1198,6 +1198,7 @@
 297+ void setUmask(mode_t mask);
 298+ int xusleep(unsigned int usec);
 299+ void keepCapabilities(void);
 300++extern int parse_sockaddr(char *s, struct sockaddr_in *addr);
 301+
 302+ #if USE_HTCP
 303+ extern void htcpInit(void);
 304+diff -ru squid-2.6.5/src/structs.h squid-2.6.5-pipelog/src/structs.h
 305+--- squid-2.6.5/src/structs.h 2007-01-20 00:07:19.000000000 +0000
 306+@@ -2439,6 +2439,8 @@
 307+ struct {
 308+ unsigned int fatal;
 309+ unsigned int syslog;
 310++ unsigned int pipe;
 311++ unsigned int udp;
 312+ } flags;
 313+ int syslog_priority;
 314+ };
 315+diff -ru squid-2.6.5/src/tools.c squid-2.6.5-pipelog/src/tools.c
 316+--- squid-2.6.5/src/tools.c 2006-11-01 20:58:52.000000000 +0000
 317+@@ -1346,3 +1346,43 @@
 318+ }
 319+ #endif
 320+ }
 321++
 322++/**
 323++ * Parse a socket address (host:port), fill the given sockaddr_in structure
 324++ * Returns FALSE on failure, TRUE on success
 325++ * Destroys s
 326++ */
 327++int parse_sockaddr(char *s, struct sockaddr_in *addr)
 328++{
 329++ char *host, *tmp, *colon;
 330++ unsigned short port = 0;
 331++ const struct hostent *hp;
 332++
 333++ host = NULL;
 334++ port = 0;
 335++ if ((colon = strchr(s, ':'))) {
 336++ /* host:port */
 337++ host = s;
 338++ *colon = '\0';
 339++ port = xatos(colon + 1);
 340++ if (0 == port)
 341++ return FALSE;
 342++ } else if ((port = strtol(s, &tmp, 10)), !*tmp) {
 343++ /* port */
 344++ } else {
 345++ host = s;
 346++ port = 0;
 347++ }
 348++ addr->sin_port = htons(port);
 349++ if (NULL == host)
 350++ addr->sin_addr = any_addr;
 351++ else if (1 == safe_inet_addr(host, &addr->sin_addr))
 352++ (void) 0;
 353++ else if ((hp = gethostbyname(host))) /* dont use ipcache */
 354++ addr->sin_addr = inaddrFromHostent(hp);
 355++ else
 356++ return FALSE;
 357++ addr->sin_family = AF_INET;
 358++ return TRUE;
 359++}
 360++
 361+
 362+
Property changes on: trunk/debs/squid/debian/patches/22-udplog.dpatch
___________________________________________________________________
Added: svn:executable
1363 + *
Index: trunk/debs/squid/debian/changelog
——@@ -1,3 +1,10 @@
 2+squid (2.6.7-1wm4) edgy; urgency=low
 3+
 4+ * Fixed EAGAIN issue in pipe logging
 5+ * Added UDP logging
 6+
 7+ -- Tim Starling <tstarling@wikimedia.org> Sat, 20 Jan 2007 00:30:22 +0000
 8+
29 squid (2.6.7-1wm3) edgy; urgency=low
310
411 * Added pipe logging patch
Index: trunk/debs/squid/debian/rules
——@@ -310,9 +310,6 @@
311311 $(INSTALL_FILE) debian/squid.overrides \
312312 $(tmp)/usr/share/lintian/overrides/squid
313313 #
314 - # Wikimedia-specific configuration
315 - $(INSTALL_PROGRAM) debian/log2udp_wm $(tmp)/usr/bin/log2udp_wm
316 - #
317314 dpkg-shlibdeps $(tmp)/usr/sbin/* $(tmp)$(libexecdir)/*
318315 dpkg-gencontrol -isp -p$(package)
319316 dpkg --build $(tmp) ..