r17688 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r17687‎ | r17688 | r17689 >
Date:04:34, 15 November 2006
Author:river
Status:old
Tags:
Comment:
experimental support for in-memory caching
Modified paths:
  • /trunk/willow/src/include/confparse.h (modified) (history)
  • /trunk/willow/src/include/flowio.h (modified) (history)
  • /trunk/willow/src/include/wconfig.h (modified) (history)
  • /trunk/willow/src/include/whttp_header.h (modified) (history)
  • /trunk/willow/src/include/wthread.h (modified) (history)
  • /trunk/willow/src/willow/Makefile.in (modified) (history)
  • /trunk/willow/src/willow/lexer.l (modified) (history)
  • /trunk/willow/src/willow/parser.y (modified) (history)
  • /trunk/willow/src/willow/wconfig.cc (modified) (history)
  • /trunk/willow/src/willow/whttp.cc (modified) (history)
  • /trunk/willow/src/willow/willow.cc (modified) (history)
  • /trunk/willow/willow.conf.example (modified) (history)

Diff [purge]

Index: trunk/willow/src/include/wconfig.h
@@ -58,10 +58,6 @@
5959 bool udp_log;
6060 string udplog_host;
6161 int udplog_port;
62 -struct cachedir *caches;
63 - int ncaches;
64 - time_t cache_expevery;
65 - int cache_expthresh;
6662 string suid, sgid;
6763 bool compress;
6864 int complevel;
@@ -78,6 +74,8 @@
7975 bool x_follow;
8076 int max_redirects;
8177 bool use_dio;
 78+ long cache_memory;
 79+ long max_entity_size;
8280 vector<pair<string, string> > stats_hosts;
8381 } config;
8482
Index: trunk/willow/src/include/wthread.h
@@ -116,16 +116,28 @@
117117 v ^= o;
118118 return *this;
119119 }
120 - atomic &operator++ (void) {
 120+ atomic &operator++ (void) { /* prefix */
121121 HOLDING(m);
122122 v++;
123123 return *this;
124124 }
125 - T operator++ (int) {
126 - atomic u (*this);
127 - u.v++;
128 - return u;
 125+ T operator++ (int) { /* postfix */
 126+ HOLDING(m);
 127+ T u = v;
 128+ v++;
 129+ return v;
129130 }
 131+ atomic &operator-- (void) {
 132+ HOLDING(m);
 133+ v--;
 134+ return *this;
 135+ }
 136+ T operator-- (int) {
 137+ HOLDING(m);
 138+ T u = v;
 139+ v--;
 140+ return v;
 141+ }
130142 };
131143
132144 template<typename T1, typename T2>
@@ -217,6 +229,7 @@
218230 HOLDING(v1.m);
219231 return !v1.v;
220232 }
 233+
221234 template<typename T1>
222235 T1 operator ~ (atomic<T1> const &v1) {
223236 HOLDING(v1.m);
Index: trunk/willow/src/include/flowio.h
@@ -471,7 +471,7 @@
472472 return res;
473473
474474 if (_left == 0)
475 - return sink_result_finished;
 475+ return _sp_sink->data_empty();
476476 return res;
477477 }
478478
@@ -485,7 +485,7 @@
486486 return res;
487487
488488 if (_left == 0)
489 - return sink_result_finished;
 489+ return _sp_sink->data_empty();
490490 return res;
491491 }
492492
Index: trunk/willow/src/include/whttp_header.h
@@ -122,6 +122,13 @@
123123 char *build (void);
124124
125125 /*
 126+ * Return the length of the string build() would return.
 127+ */
 128+ int length (void) const {
 129+ return hl_len;
 130+ }
 131+
 132+ /*
126133 * Find a specific header in the list. Returns NULL if no such header
127134 * exists.
128135 */
Index: trunk/willow/src/include/confparse.h
@@ -226,6 +226,7 @@
227227 typedef simple_value<cv_int> simple_int_t;
228228 typedef simple_value<cv_yesno> simple_yesno_t;
229229 typedef simple_value<cv_time> simple_time_t;
 230+
230231 extern simple_int_t simple_int;
231232 extern simple_yesno_t simple_yesno;
232233 extern simple_time_t simple_time;
@@ -332,6 +333,7 @@
333334 typedef set_simple<time_t> set_time;
334335 typedef set_simple<bool> set_yesno;
335336 typedef set_simple<int> set_int;
 337+typedef set_simple<long> set_long;
336338 typedef set_simple<atomic<time_t> > set_atime;
337339 typedef set_simple<atomic<bool> > set_abool;
338340 typedef set_simple<atomic<int> > set_aint;
Index: trunk/willow/src/willow/wconfig.cc
@@ -155,20 +155,6 @@
156156 {
157157 }
158158
159 -static void
160 -set_cache(tree_entry &e)
161 -{
162 -value *v;
163 - v = e/"size";
164 - config.caches = (cachedir *)wrealloc(config.caches, sizeof(*config.caches) * (config.ncaches + 1));
165 - config.caches[config.ncaches].dir = wstrdup(e.item_key.c_str());
166 - config.caches[config.ncaches].maxsize = v->cv_values[0].av_intval;
167 - wlog(WLOG_NOTICE, format("cache dir \"%s\", size %d bytes")
168 - % config.caches[config.ncaches].dir
169 - % config.caches[config.ncaches].maxsize);
170 - config.ncaches++;
171 -}
172 -
173159 static bool
174160 v_udp_log(tree_entry &e, value &v)
175161 {
@@ -377,8 +363,8 @@
378364 .value("udp-host", nonempty_qstring, set_string(config.udplog_host))
379365
380366 .block("cache")
381 - .value("expire-every", simple_time, set_time(config.cache_expevery))
382 - .value("expire-threshold", simple_range(0, 100), set_int(config.cache_expthresh))
 367+ .value("cache-memory", simple_time, set_long(config.cache_memory))
 368+ .value("max-entity-size", simple_time, set_long(config.max_entity_size))
383369
384370 .block("http")
385371 .value("compress", simple_yesno, set_yesno(config.compress))
@@ -405,10 +391,6 @@
406392 .value("enable", simple_yesno, set_yesno(config.udp_stats))
407393 .value("listen", ip_address_list, add_ip(config.stats_hosts))
408394
409 - .block("cache-dir", require_name)
410 - .end(func(set_cache))
411 - .value("size", simple_time, ignore)
412 -
413395 .block("listen", require_name)
414396 .end(func(set_listen))
415397 .value("port", simple_range(1, 65535), ignore)
@@ -453,6 +435,7 @@
454436 config.max_redirects = 1;
455437 config.use_dio = false;
456438 config.x_follow = false;
 439+ config.cache_memory = 0;
457440
458441 conf.set(*t);
459442 whttp_reconfigure();
Index: trunk/willow/src/willow/Makefile.in
@@ -3,19 +3,19 @@
44 CPPFLAGS = -I../include
55
66 BASESRCS = \
7 - willow.cc \
 7+ cache.cc \
 8+ chunking.cc \
 9+ confparse.cc \
 10+ flowio.cc \
 11+ format.cc \
 12+ radix.cc \
 13+ wbackend.cc \
814 wconfig.cc \
915 whttp.cc \
10 - chunking.cc \
11 - wbackend.cc \
12 - wcache.cc \
13 - confparse.cc \
 16+ whttp_header.cc \
 17+ willow.cc \
 18+ wlog.cc \
1419 wnet.cc \
15 - wlog.cc \
16 - whttp_header.cc \
17 - radix.cc \
18 - format.cc \
19 - flowio.cc
2020
2121 OBJS= $(BASESRCS:.cc=.o)
2222
Index: trunk/willow/src/willow/whttp.cc
@@ -48,11 +48,11 @@
4949 #include "wlogwriter.h"
5050 #include "whttp_entity.h"
5151 #include "wlog.h"
52 -#include "wcache.h"
5352 #include "radix.h"
5453 #include "chunking.h"
5554 #include "flowio.h"
5655 #include "format.h"
 56+#include "cache.h"
5757
5858 using namespace wnet;
5959
@@ -164,6 +164,9 @@
165165 error_transform_filter *_error_filter;
166166 chunking_filter *_chunking_filter;
167167 io::size_limiting_filter *_size_limit;
 168+ cachedentity *_cachedent;
 169+ caching_filter *_cache_filter;
 170+ cached_spigot *_cache_spigot;
168171
169172 backend_list *_blist;
170173 bool _denied;
@@ -193,6 +196,9 @@
194197 , _error_filter(NULL)
195198 , _chunking_filter(NULL)
196199 , _size_limit(NULL)
 200+ , _cachedent(NULL)
 201+ , _cache_filter(NULL)
 202+ , _cache_spigot(NULL)
197203 , _blist(NULL)
198204 , _denied(false)
199205 , _group(gr)
@@ -265,6 +271,13 @@
266272 _size_limit = NULL;
267273 delete _header_parser;
268274 _header_parser = NULL;
 275+ delete _cache_filter;
 276+ _cache_filter = NULL;
 277+ delete _cache_spigot;
 278+ _cache_spigot = NULL;
 279+ if (_cachedent)
 280+ entitycache.release(_cachedent);
 281+ _cachedent = NULL;
269282
270283 /*
271284 * Return the backend to the keepalive pool, if we can.
@@ -331,6 +344,41 @@
332345 }
333346
334347 _client_spigot->sp_disconnect();
 348+
 349+ /*
 350+ * See if this entity has been cached.
 351+ */
 352+ if (_header_parser->_http_reqtype != REQTYPE_POST) {
 353+ bool created = false;
 354+ string url = format("http://%s%s") % _header_parser->_http_host
 355+ % _header_parser->_http_path;
 356+ _cachedent = entitycache.find_cached(imstring(url), true, created);
 357+ if (_cachedent && _cachedent->complete()) {
 358+ /* yes - complete object is available */
 359+ _cache_spigot = new cached_spigot(_cachedent);
 360+ if (_header_parser->_force_keepalive)
 361+ _cache_spigot->keepalive(true);
 362+
 363+ if (!_client_sink)
 364+ _client_sink = new io::socket_sink(_client_socket);
 365+ _client_socket->cork();
 366+ _cache_spigot->error_callee(this, &httpcllr::send_body_to_client_error);
 367+ _cache_spigot->completed_callee(this, &httpcllr::send_body_to_client_done);
 368+ _cache_spigot->sp_connect(_client_sink);
 369+ _cache_spigot->sp_uncork();
 370+ return;
 371+ }
 372+
 373+ /*
 374+ * If the entity already exists but is not complete, we don't care
 375+ * about it.
 376+ */
 377+ if (_cachedent && !created && !_cachedent->complete()) {
 378+ entitycache.release(_cachedent);
 379+ _cachedent = NULL;
 380+ }
 381+ }
 382+
335383 map<string,int>::iterator it;
336384 map<imstring,int>::iterator mit;
337385 pair<bool, uint16_t> acheck;
@@ -469,7 +517,6 @@
470518 httpcllr::backend_write_headers_done(void)
471519 {
472520 if (_header_parser->_http_reqtype == REQTYPE_POST) {
473 -std::cout<<"POST "<<_header_parser->_content_length<<" bytes\n";
474521 /*
475522 * Connect the client to the backend and read the POST data.
476523 */
@@ -515,7 +562,6 @@
516563 * Check for X-Willow-Follow-Redirect header, which means we should
517564 * follow the redirect.
518565 */
519 -
520566 if (config.x_follow &&
521567 _backend_headers->_follow_redirect &&
522568 _backend_headers->_location.size() &&
@@ -530,6 +576,15 @@
531577 return;
532578 }
533579
 580+ /*
 581+ * If we're caching this entity, store the headers.
 582+ */
 583+ if (_cachedent) {
 584+ string status = format("HTTP/1.1 %s\r\n") % _backend_headers->_http_path;
 585+ _cachedent->store_status(status);
 586+ _cachedent->store_headers(_backend_headers->_headers);
 587+ }
 588+
534589 if (_backend_headers->_content_length == -1 && !_backend_headers->_flags.f_chunked
535590 && _header_parser->_http_vers == http11 && !(config.msie_hack && _header_parser->_is_msie))
536591 /* we will chunk the request later */
@@ -572,6 +627,7 @@
573628 void
574629 httpcllr::send_headers_to_client_done(void)
575630 {
 631+bool cache = false;
576632 /*
577633 * Now connect the backend directly to the client.
578634 */
@@ -579,6 +635,14 @@
580636 _backend_spigot->completed_callee(this, &httpcllr::send_body_to_client_done);
581637
582638 /*
 639+ * See if we can cache this entity.
 640+ */
 641+ if (_cachedent) {
 642+ cache = true;
 643+ _cache_filter = new caching_filter(_cachedent);
 644+ }
 645+
 646+ /*
583647 * If the server is sending chunked data and the client is
584648 * HTTP 1.0, insert a dechunking filter.
585649 */
@@ -593,7 +657,11 @@
594658 if (_backend_headers->_flags.f_chunked && _header_parser->_http_vers == http10) {
595659 _dechunking_filter = new dechunking_filter;
596660 _backend_spigot->sp_connect(_dechunking_filter);
597 - _dechunking_filter->sp_connect(_client_sink);
 661+ if (cache) {
 662+ _dechunking_filter->sp_connect(_cache_filter);
 663+ _cache_filter->sp_connect(_client_sink);
 664+ } else
 665+ _dechunking_filter->sp_connect(_client_sink);
598666 } else if (_backend_headers->_content_length == -1 && !_backend_headers->_flags.f_chunked
599667 && _header_parser->_http_vers == http11 && !(config.msie_hack && _header_parser->_is_msie)) {
600668 /*
@@ -602,12 +670,20 @@
603671 * didn't send enough data.
604672 */
605673 _chunking_filter = new chunking_filter;
606 - _backend_spigot->sp_connect(_chunking_filter);
 674+ if (cache) {
 675+ _backend_spigot->sp_connect(_cache_filter);
 676+ _cache_filter->sp_connect(_chunking_filter);
 677+ } else
 678+ _backend_spigot->sp_connect(_chunking_filter);
607679 _chunking_filter->sp_connect(_client_sink);
608680 } else if (_backend_headers->_flags.f_chunked && config.msie_hack && _header_parser->_is_msie) {
609681 _dechunking_filter = new dechunking_filter;
610682 _backend_spigot->sp_connect(_dechunking_filter);
611 - _dechunking_filter->sp_connect(_client_sink);
 683+ if (cache) {
 684+ _dechunking_filter->sp_connect(_cache_filter);
 685+ _cache_filter->sp_connect(_client_sink);
 686+ } else
 687+ _dechunking_filter->sp_connect(_client_sink);
612688 } else {
613689 /*
614690 * For a keep-alive request, we need a size limiting filter to prevent
@@ -617,9 +693,17 @@
618694 delete _size_limit;
619695 _size_limit = new io::size_limiting_filter(_backend_headers->_content_length);
620696 _backend_spigot->sp_connect(_size_limit);
621 - _size_limit->sp_connect(_client_sink);
 697+ if (cache) {
 698+ _size_limit->sp_connect(_cache_filter);
 699+ _cache_filter->sp_connect(_client_sink);
 700+ } else
 701+ _size_limit->sp_connect(_client_sink);
622702 } else {
623 - _backend_spigot->sp_connect(_client_sink);
 703+ if (cache) {
 704+ _backend_spigot->sp_connect(_cache_filter);
 705+ _cache_filter->sp_connect(_client_sink);
 706+ } else
 707+ _backend_spigot->sp_connect(_client_sink);
624708 }
625709 _client_sink->_counter = 0;
626710 }
Index: trunk/willow/src/willow/willow.cc
@@ -71,7 +71,6 @@
7272 fprintf(stderr, "usage: %s [-hfzv] [-D cond[=value]]\n"
7373 " -h print this message\n"
7474 " -f run in foreground (don't detach)\n"
75 -" -z create cache directory structure and exit\n"
7675 " -v print version number and exit\n"
7776 " -D cond[=value] set 'cond' to 'value' (which should be 'true' or\n"
7877 " 'false') in the configuration parser. if 'value'\n"
@@ -104,20 +103,16 @@
105104 main(int argc, char *argv[])
106105 {
107106 int i;
108 -int zflag = 0;
109107 char *cfg = NULL;
110108 char *dval;
111109
112110 progname = argv[0];
113111
114 - while ((i = getopt(argc, argv, "fzvc:D:h")) != -1) {
 112+ while ((i = getopt(argc, argv, "fvc:D:h")) != -1) {
115113 switch (i) {
116114 case 'h':
117115 usage();
118116 return 0;
119 - case 'z':
120 - zflag++;
121 - /*FALLTHRU*/
122117 case 'f':
123118 config.foreground = 1;
124119 break;
@@ -186,16 +181,11 @@
187182 }
188183
189184 wlog_init();
190 - if (zflag) {
191 - wcache_setupfs();
192 - exit(0);
193 - }
194185
195186 make_event_base();
196187 ioloop = new ioloop_t;
197188 checkexit_sched();
198189 whttp_init();
199 - wcache_init(1);
200190 stats_init();
201191
202192 wlog(WLOG_NOTICE, "running");
@@ -206,7 +196,6 @@
207197 ioloop->run();
208198 wlog(WLOG_NOTICE, "shutting down");
209199 wlog_close();
210 - wcache_shutdown();
211200 whttp_shutdown();
212201
213202 pthread_exit(NULL);
Index: trunk/willow/src/willow/parser.y
@@ -62,8 +62,8 @@
6363
6464 for (i = 0; conf_times[i].name; i++)
6565 {
66 - if (conf_times[i].name ==name ||
67 - (conf_times[i].plural && conf_times[i].plural == name))
 66+ if (!strcasecmp(conf_times[i].name, name.c_str()) ||
 67+ (conf_times[i].plural && !strcasecmp(conf_times[i].plural, name.c_str())))
6868 return conf_times[i].val;
6969 }
7070
Index: trunk/willow/src/willow/lexer.l
@@ -150,7 +150,7 @@
151151 return STRING;
152152 }
153153 {number} {
154 - yylval.number = atoi(yytext);
 154+ yylval.number = strtol(yytext, NULL, 0);
155155 conf::curpos += yyleng;
156156 return NUMBER;
157157 }
Index: trunk/willow/willow.conf.example
@@ -131,13 +131,20 @@
132132 */
133133 cache {
134134 /*
135 - * Every expire_every seconds, the oldest expire_threshold%
136 - * objects will be removed from the cache, using an LRU
137 - * algorithm. This only occurs if the cache is more than
138 - * (100 - expire_theshold)% full.
 135+ * How much memory to reserve for entity caching. Once this
 136+ * much memory has been used, old objects will be removed from
 137+ * the cache to make room for new ones.
 138+ *
 139+ * Set to 0 to disable caching.
139140 */
140 - expire-every = 1 hour;
141 - expire-threshold = 5;
 141+ cache-memory = 5MB;
 142+
 143+ /*
 144+ * The largest object to cache. An object larger than this
 145+ * will never be cached. Set to 0 (the default) to cache
 146+ * objects of any size up to cache-memory.
 147+ */
 148+ max-entity-size = 256KB;
142149 };
143150
144151 /*
@@ -219,19 +226,6 @@
220227 max-redirects = 1;
221228 };
222229
223 -/*
224 - * For now, only one cache directory is allowed.
225 - * Run "willow -z" to create the initial directory. Make sure it
226 - * doesn't exist first.
227 - *
228 - * If you specify no cache directories, Willow will switch to a more
229 - * efficient proxy-only mode. This is most useful in combination with
230 - * CARP hashing in front of another cache.
231 - */
232 -cache-dir "/home/kate/mediawiki/willow/cache" {
233 - size = 10 mb;
234 -};
235 -
236230 log {
237231 /*
238232 * Levels: