Index: trunk/willow/src/include/wconfig.h |
— | — | @@ -58,10 +58,6 @@ |
59 | 59 | bool udp_log; |
60 | 60 | string udplog_host; |
61 | 61 | int udplog_port; |
62 | | -struct cachedir *caches; |
63 | | - int ncaches; |
64 | | - time_t cache_expevery; |
65 | | - int cache_expthresh; |
66 | 62 | string suid, sgid; |
67 | 63 | bool compress; |
68 | 64 | int complevel; |
— | — | @@ -78,6 +74,8 @@ |
79 | 75 | bool x_follow; |
80 | 76 | int max_redirects; |
81 | 77 | bool use_dio; |
| 78 | + long cache_memory; |
| 79 | + long max_entity_size; |
82 | 80 | vector<pair<string, string> > stats_hosts; |
83 | 81 | } config; |
84 | 82 | |
Index: trunk/willow/src/include/wthread.h |
— | — | @@ -116,16 +116,28 @@ |
117 | 117 | v ^= o; |
118 | 118 | return *this; |
119 | 119 | } |
120 | | - atomic &operator++ (void) { |
| 120 | + atomic &operator++ (void) { /* prefix */ |
121 | 121 | HOLDING(m); |
122 | 122 | v++; |
123 | 123 | return *this; |
124 | 124 | } |
125 | | - T operator++ (int) { |
126 | | - atomic u (*this); |
127 | | - u.v++; |
128 | | - return u; |
| 125 | + T operator++ (int) { /* postfix */ |
| 126 | + HOLDING(m); |
| 127 | + T u = v; |
| 128 | + v++; |
| 129 | + return v; |
129 | 130 | } |
| 131 | + atomic &operator-- (void) { |
| 132 | + HOLDING(m); |
| 133 | + v--; |
| 134 | + return *this; |
| 135 | + } |
| 136 | + T operator-- (int) { |
| 137 | + HOLDING(m); |
| 138 | + T u = v; |
| 139 | + v--; |
| 140 | + return v; |
| 141 | + } |
130 | 142 | }; |
131 | 143 | |
132 | 144 | template<typename T1, typename T2> |
— | — | @@ -217,6 +229,7 @@ |
218 | 230 | HOLDING(v1.m); |
219 | 231 | return !v1.v; |
220 | 232 | } |
| 233 | + |
221 | 234 | template<typename T1> |
222 | 235 | T1 operator ~ (atomic<T1> const &v1) { |
223 | 236 | HOLDING(v1.m); |
Index: trunk/willow/src/include/flowio.h |
— | — | @@ -471,7 +471,7 @@ |
472 | 472 | return res; |
473 | 473 | |
474 | 474 | if (_left == 0) |
475 | | - return sink_result_finished; |
| 475 | + return _sp_sink->data_empty(); |
476 | 476 | return res; |
477 | 477 | } |
478 | 478 | |
— | — | @@ -485,7 +485,7 @@ |
486 | 486 | return res; |
487 | 487 | |
488 | 488 | if (_left == 0) |
489 | | - return sink_result_finished; |
| 489 | + return _sp_sink->data_empty(); |
490 | 490 | return res; |
491 | 491 | } |
492 | 492 | |
Index: trunk/willow/src/include/whttp_header.h |
— | — | @@ -122,6 +122,13 @@ |
123 | 123 | char *build (void); |
124 | 124 | |
125 | 125 | /* |
| 126 | + * Return the length of the string build() would return. |
| 127 | + */ |
| 128 | + int length (void) const { |
| 129 | + return hl_len; |
| 130 | + } |
| 131 | + |
| 132 | + /* |
126 | 133 | * Find a specific header in the list. Returns NULL if no such header |
127 | 134 | * exists. |
128 | 135 | */ |
Index: trunk/willow/src/include/confparse.h |
— | — | @@ -226,6 +226,7 @@ |
227 | 227 | typedef simple_value<cv_int> simple_int_t; |
228 | 228 | typedef simple_value<cv_yesno> simple_yesno_t; |
229 | 229 | typedef simple_value<cv_time> simple_time_t; |
| 230 | + |
230 | 231 | extern simple_int_t simple_int; |
231 | 232 | extern simple_yesno_t simple_yesno; |
232 | 233 | extern simple_time_t simple_time; |
— | — | @@ -332,6 +333,7 @@ |
333 | 334 | typedef set_simple<time_t> set_time; |
334 | 335 | typedef set_simple<bool> set_yesno; |
335 | 336 | typedef set_simple<int> set_int; |
| 337 | +typedef set_simple<long> set_long; |
336 | 338 | typedef set_simple<atomic<time_t> > set_atime; |
337 | 339 | typedef set_simple<atomic<bool> > set_abool; |
338 | 340 | typedef set_simple<atomic<int> > set_aint; |
Index: trunk/willow/src/willow/wconfig.cc |
— | — | @@ -155,20 +155,6 @@ |
156 | 156 | { |
157 | 157 | } |
158 | 158 | |
159 | | -static void |
160 | | -set_cache(tree_entry &e) |
161 | | -{ |
162 | | -value *v; |
163 | | - v = e/"size"; |
164 | | - config.caches = (cachedir *)wrealloc(config.caches, sizeof(*config.caches) * (config.ncaches + 1)); |
165 | | - config.caches[config.ncaches].dir = wstrdup(e.item_key.c_str()); |
166 | | - config.caches[config.ncaches].maxsize = v->cv_values[0].av_intval; |
167 | | - wlog(WLOG_NOTICE, format("cache dir \"%s\", size %d bytes") |
168 | | - % config.caches[config.ncaches].dir |
169 | | - % config.caches[config.ncaches].maxsize); |
170 | | - config.ncaches++; |
171 | | -} |
172 | | - |
173 | 159 | static bool |
174 | 160 | v_udp_log(tree_entry &e, value &v) |
175 | 161 | { |
— | — | @@ -377,8 +363,8 @@ |
378 | 364 | .value("udp-host", nonempty_qstring, set_string(config.udplog_host)) |
379 | 365 | |
380 | 366 | .block("cache") |
381 | | - .value("expire-every", simple_time, set_time(config.cache_expevery)) |
382 | | - .value("expire-threshold", simple_range(0, 100), set_int(config.cache_expthresh)) |
| 367 | + .value("cache-memory", simple_time, set_long(config.cache_memory)) |
| 368 | + .value("max-entity-size", simple_time, set_long(config.max_entity_size)) |
383 | 369 | |
384 | 370 | .block("http") |
385 | 371 | .value("compress", simple_yesno, set_yesno(config.compress)) |
— | — | @@ -405,10 +391,6 @@ |
406 | 392 | .value("enable", simple_yesno, set_yesno(config.udp_stats)) |
407 | 393 | .value("listen", ip_address_list, add_ip(config.stats_hosts)) |
408 | 394 | |
409 | | - .block("cache-dir", require_name) |
410 | | - .end(func(set_cache)) |
411 | | - .value("size", simple_time, ignore) |
412 | | - |
413 | 395 | .block("listen", require_name) |
414 | 396 | .end(func(set_listen)) |
415 | 397 | .value("port", simple_range(1, 65535), ignore) |
— | — | @@ -453,6 +435,7 @@ |
454 | 436 | config.max_redirects = 1; |
455 | 437 | config.use_dio = false; |
456 | 438 | config.x_follow = false; |
| 439 | + config.cache_memory = 0; |
457 | 440 | |
458 | 441 | conf.set(*t); |
459 | 442 | whttp_reconfigure(); |
Index: trunk/willow/src/willow/Makefile.in |
— | — | @@ -3,19 +3,19 @@ |
4 | 4 | CPPFLAGS = -I../include |
5 | 5 | |
6 | 6 | BASESRCS = \ |
7 | | - willow.cc \ |
| 7 | + cache.cc \ |
| 8 | + chunking.cc \ |
| 9 | + confparse.cc \ |
| 10 | + flowio.cc \ |
| 11 | + format.cc \ |
| 12 | + radix.cc \ |
| 13 | + wbackend.cc \ |
8 | 14 | wconfig.cc \ |
9 | 15 | whttp.cc \ |
10 | | - chunking.cc \ |
11 | | - wbackend.cc \ |
12 | | - wcache.cc \ |
13 | | - confparse.cc \ |
| 16 | + whttp_header.cc \ |
| 17 | + willow.cc \ |
| 18 | + wlog.cc \ |
14 | 19 | wnet.cc \ |
15 | | - wlog.cc \ |
16 | | - whttp_header.cc \ |
17 | | - radix.cc \ |
18 | | - format.cc \ |
19 | | - flowio.cc |
20 | 20 | |
21 | 21 | OBJS= $(BASESRCS:.cc=.o) |
22 | 22 | |
Index: trunk/willow/src/willow/whttp.cc |
— | — | @@ -48,11 +48,11 @@ |
49 | 49 | #include "wlogwriter.h" |
50 | 50 | #include "whttp_entity.h" |
51 | 51 | #include "wlog.h" |
52 | | -#include "wcache.h" |
53 | 52 | #include "radix.h" |
54 | 53 | #include "chunking.h" |
55 | 54 | #include "flowio.h" |
56 | 55 | #include "format.h" |
| 56 | +#include "cache.h" |
57 | 57 | |
58 | 58 | using namespace wnet; |
59 | 59 | |
— | — | @@ -164,6 +164,9 @@ |
165 | 165 | error_transform_filter *_error_filter; |
166 | 166 | chunking_filter *_chunking_filter; |
167 | 167 | io::size_limiting_filter *_size_limit; |
| 168 | + cachedentity *_cachedent; |
| 169 | + caching_filter *_cache_filter; |
| 170 | + cached_spigot *_cache_spigot; |
168 | 171 | |
169 | 172 | backend_list *_blist; |
170 | 173 | bool _denied; |
— | — | @@ -193,6 +196,9 @@ |
194 | 197 | , _error_filter(NULL) |
195 | 198 | , _chunking_filter(NULL) |
196 | 199 | , _size_limit(NULL) |
| 200 | + , _cachedent(NULL) |
| 201 | + , _cache_filter(NULL) |
| 202 | + , _cache_spigot(NULL) |
197 | 203 | , _blist(NULL) |
198 | 204 | , _denied(false) |
199 | 205 | , _group(gr) |
— | — | @@ -265,6 +271,13 @@ |
266 | 272 | _size_limit = NULL; |
267 | 273 | delete _header_parser; |
268 | 274 | _header_parser = NULL; |
| 275 | + delete _cache_filter; |
| 276 | + _cache_filter = NULL; |
| 277 | + delete _cache_spigot; |
| 278 | + _cache_spigot = NULL; |
| 279 | + if (_cachedent) |
| 280 | + entitycache.release(_cachedent); |
| 281 | + _cachedent = NULL; |
269 | 282 | |
270 | 283 | /* |
271 | 284 | * Return the backend to the keepalive pool, if we can. |
— | — | @@ -331,6 +344,41 @@ |
332 | 345 | } |
333 | 346 | |
334 | 347 | _client_spigot->sp_disconnect(); |
| 348 | + |
| 349 | + /* |
| 350 | + * See if this entity has been cached. |
| 351 | + */ |
| 352 | + if (_header_parser->_http_reqtype != REQTYPE_POST) { |
| 353 | + bool created = false; |
| 354 | + string url = format("http://%s%s") % _header_parser->_http_host |
| 355 | + % _header_parser->_http_path; |
| 356 | + _cachedent = entitycache.find_cached(imstring(url), true, created); |
| 357 | + if (_cachedent && _cachedent->complete()) { |
| 358 | + /* yes - complete object is available */ |
| 359 | + _cache_spigot = new cached_spigot(_cachedent); |
| 360 | + if (_header_parser->_force_keepalive) |
| 361 | + _cache_spigot->keepalive(true); |
| 362 | + |
| 363 | + if (!_client_sink) |
| 364 | + _client_sink = new io::socket_sink(_client_socket); |
| 365 | + _client_socket->cork(); |
| 366 | + _cache_spigot->error_callee(this, &httpcllr::send_body_to_client_error); |
| 367 | + _cache_spigot->completed_callee(this, &httpcllr::send_body_to_client_done); |
| 368 | + _cache_spigot->sp_connect(_client_sink); |
| 369 | + _cache_spigot->sp_uncork(); |
| 370 | + return; |
| 371 | + } |
| 372 | + |
| 373 | + /* |
| 374 | + * If the entity already exists but is not complete, we don't care |
| 375 | + * about it. |
| 376 | + */ |
| 377 | + if (_cachedent && !created && !_cachedent->complete()) { |
| 378 | + entitycache.release(_cachedent); |
| 379 | + _cachedent = NULL; |
| 380 | + } |
| 381 | + } |
| 382 | + |
335 | 383 | map<string,int>::iterator it; |
336 | 384 | map<imstring,int>::iterator mit; |
337 | 385 | pair<bool, uint16_t> acheck; |
— | — | @@ -469,7 +517,6 @@ |
470 | 518 | httpcllr::backend_write_headers_done(void) |
471 | 519 | { |
472 | 520 | if (_header_parser->_http_reqtype == REQTYPE_POST) { |
473 | | -std::cout<<"POST "<<_header_parser->_content_length<<" bytes\n"; |
474 | 521 | /* |
475 | 522 | * Connect the client to the backend and read the POST data. |
476 | 523 | */ |
— | — | @@ -515,7 +562,6 @@ |
516 | 563 | * Check for X-Willow-Follow-Redirect header, which means we should |
517 | 564 | * follow the redirect. |
518 | 565 | */ |
519 | | - |
520 | 566 | if (config.x_follow && |
521 | 567 | _backend_headers->_follow_redirect && |
522 | 568 | _backend_headers->_location.size() && |
— | — | @@ -530,6 +576,15 @@ |
531 | 577 | return; |
532 | 578 | } |
533 | 579 | |
| 580 | + /* |
| 581 | + * If we're caching this entity, store the headers. |
| 582 | + */ |
| 583 | + if (_cachedent) { |
| 584 | + string status = format("HTTP/1.1 %s\r\n") % _backend_headers->_http_path; |
| 585 | + _cachedent->store_status(status); |
| 586 | + _cachedent->store_headers(_backend_headers->_headers); |
| 587 | + } |
| 588 | + |
534 | 589 | if (_backend_headers->_content_length == -1 && !_backend_headers->_flags.f_chunked |
535 | 590 | && _header_parser->_http_vers == http11 && !(config.msie_hack && _header_parser->_is_msie)) |
536 | 591 | /* we will chunk the request later */ |
— | — | @@ -572,6 +627,7 @@ |
573 | 628 | void |
574 | 629 | httpcllr::send_headers_to_client_done(void) |
575 | 630 | { |
| 631 | +bool cache = false; |
576 | 632 | /* |
577 | 633 | * Now connect the backend directly to the client. |
578 | 634 | */ |
— | — | @@ -579,6 +635,14 @@ |
580 | 636 | _backend_spigot->completed_callee(this, &httpcllr::send_body_to_client_done); |
581 | 637 | |
582 | 638 | /* |
| 639 | + * See if we can cache this entity. |
| 640 | + */ |
| 641 | + if (_cachedent) { |
| 642 | + cache = true; |
| 643 | + _cache_filter = new caching_filter(_cachedent); |
| 644 | + } |
| 645 | + |
| 646 | + /* |
583 | 647 | * If the server is sending chunked data and the client is |
584 | 648 | * HTTP 1.0, insert a dechunking filter. |
585 | 649 | */ |
— | — | @@ -593,7 +657,11 @@ |
594 | 658 | if (_backend_headers->_flags.f_chunked && _header_parser->_http_vers == http10) { |
595 | 659 | _dechunking_filter = new dechunking_filter; |
596 | 660 | _backend_spigot->sp_connect(_dechunking_filter); |
597 | | - _dechunking_filter->sp_connect(_client_sink); |
| 661 | + if (cache) { |
| 662 | + _dechunking_filter->sp_connect(_cache_filter); |
| 663 | + _cache_filter->sp_connect(_client_sink); |
| 664 | + } else |
| 665 | + _dechunking_filter->sp_connect(_client_sink); |
598 | 666 | } else if (_backend_headers->_content_length == -1 && !_backend_headers->_flags.f_chunked |
599 | 667 | && _header_parser->_http_vers == http11 && !(config.msie_hack && _header_parser->_is_msie)) { |
600 | 668 | /* |
— | — | @@ -602,12 +670,20 @@ |
603 | 671 | * didn't send enough data. |
604 | 672 | */ |
605 | 673 | _chunking_filter = new chunking_filter; |
606 | | - _backend_spigot->sp_connect(_chunking_filter); |
| 674 | + if (cache) { |
| 675 | + _backend_spigot->sp_connect(_cache_filter); |
| 676 | + _cache_filter->sp_connect(_chunking_filter); |
| 677 | + } else |
| 678 | + _backend_spigot->sp_connect(_chunking_filter); |
607 | 679 | _chunking_filter->sp_connect(_client_sink); |
608 | 680 | } else if (_backend_headers->_flags.f_chunked && config.msie_hack && _header_parser->_is_msie) { |
609 | 681 | _dechunking_filter = new dechunking_filter; |
610 | 682 | _backend_spigot->sp_connect(_dechunking_filter); |
611 | | - _dechunking_filter->sp_connect(_client_sink); |
| 683 | + if (cache) { |
| 684 | + _dechunking_filter->sp_connect(_cache_filter); |
| 685 | + _cache_filter->sp_connect(_client_sink); |
| 686 | + } else |
| 687 | + _dechunking_filter->sp_connect(_client_sink); |
612 | 688 | } else { |
613 | 689 | /* |
614 | 690 | * For a keep-alive request, we need a size limiting filter to prevent |
— | — | @@ -617,9 +693,17 @@ |
618 | 694 | delete _size_limit; |
619 | 695 | _size_limit = new io::size_limiting_filter(_backend_headers->_content_length); |
620 | 696 | _backend_spigot->sp_connect(_size_limit); |
621 | | - _size_limit->sp_connect(_client_sink); |
| 697 | + if (cache) { |
| 698 | + _size_limit->sp_connect(_cache_filter); |
| 699 | + _cache_filter->sp_connect(_client_sink); |
| 700 | + } else |
| 701 | + _size_limit->sp_connect(_client_sink); |
622 | 702 | } else { |
623 | | - _backend_spigot->sp_connect(_client_sink); |
| 703 | + if (cache) { |
| 704 | + _backend_spigot->sp_connect(_cache_filter); |
| 705 | + _cache_filter->sp_connect(_client_sink); |
| 706 | + } else |
| 707 | + _backend_spigot->sp_connect(_client_sink); |
624 | 708 | } |
625 | 709 | _client_sink->_counter = 0; |
626 | 710 | } |
Index: trunk/willow/src/willow/willow.cc |
— | — | @@ -71,7 +71,6 @@ |
72 | 72 | fprintf(stderr, "usage: %s [-hfzv] [-D cond[=value]]\n" |
73 | 73 | " -h print this message\n" |
74 | 74 | " -f run in foreground (don't detach)\n" |
75 | | -" -z create cache directory structure and exit\n" |
76 | 75 | " -v print version number and exit\n" |
77 | 76 | " -D cond[=value] set 'cond' to 'value' (which should be 'true' or\n" |
78 | 77 | " 'false') in the configuration parser. if 'value'\n" |
— | — | @@ -104,20 +103,16 @@ |
105 | 104 | main(int argc, char *argv[]) |
106 | 105 | { |
107 | 106 | int i; |
108 | | -int zflag = 0; |
109 | 107 | char *cfg = NULL; |
110 | 108 | char *dval; |
111 | 109 | |
112 | 110 | progname = argv[0]; |
113 | 111 | |
114 | | - while ((i = getopt(argc, argv, "fzvc:D:h")) != -1) { |
| 112 | + while ((i = getopt(argc, argv, "fvc:D:h")) != -1) { |
115 | 113 | switch (i) { |
116 | 114 | case 'h': |
117 | 115 | usage(); |
118 | 116 | return 0; |
119 | | - case 'z': |
120 | | - zflag++; |
121 | | - /*FALLTHRU*/ |
122 | 117 | case 'f': |
123 | 118 | config.foreground = 1; |
124 | 119 | break; |
— | — | @@ -186,16 +181,11 @@ |
187 | 182 | } |
188 | 183 | |
189 | 184 | wlog_init(); |
190 | | - if (zflag) { |
191 | | - wcache_setupfs(); |
192 | | - exit(0); |
193 | | - } |
194 | 185 | |
195 | 186 | make_event_base(); |
196 | 187 | ioloop = new ioloop_t; |
197 | 188 | checkexit_sched(); |
198 | 189 | whttp_init(); |
199 | | - wcache_init(1); |
200 | 190 | stats_init(); |
201 | 191 | |
202 | 192 | wlog(WLOG_NOTICE, "running"); |
— | — | @@ -206,7 +196,6 @@ |
207 | 197 | ioloop->run(); |
208 | 198 | wlog(WLOG_NOTICE, "shutting down"); |
209 | 199 | wlog_close(); |
210 | | - wcache_shutdown(); |
211 | 200 | whttp_shutdown(); |
212 | 201 | |
213 | 202 | pthread_exit(NULL); |
Index: trunk/willow/src/willow/parser.y |
— | — | @@ -62,8 +62,8 @@ |
63 | 63 | |
64 | 64 | for (i = 0; conf_times[i].name; i++) |
65 | 65 | { |
66 | | - if (conf_times[i].name ==name || |
67 | | - (conf_times[i].plural && conf_times[i].plural == name)) |
| 66 | + if (!strcasecmp(conf_times[i].name, name.c_str()) || |
| 67 | + (conf_times[i].plural && !strcasecmp(conf_times[i].plural, name.c_str()))) |
68 | 68 | return conf_times[i].val; |
69 | 69 | } |
70 | 70 | |
Index: trunk/willow/src/willow/lexer.l |
— | — | @@ -150,7 +150,7 @@ |
151 | 151 | return STRING; |
152 | 152 | } |
153 | 153 | {number} { |
154 | | - yylval.number = atoi(yytext); |
| 154 | + yylval.number = strtol(yytext, NULL, 0); |
155 | 155 | conf::curpos += yyleng; |
156 | 156 | return NUMBER; |
157 | 157 | } |
Index: trunk/willow/willow.conf.example |
— | — | @@ -131,13 +131,20 @@ |
132 | 132 | */ |
133 | 133 | cache { |
134 | 134 | /* |
135 | | - * Every expire_every seconds, the oldest expire_threshold% |
136 | | - * objects will be removed from the cache, using an LRU |
137 | | - * algorithm. This only occurs if the cache is more than |
138 | | - * (100 - expire_theshold)% full. |
| 135 | + * How much memory to reserve for entity caching. Once this |
| 136 | + * much memory has been used, old objects will be removed from |
| 137 | + * the cache to make room for new ones. |
| 138 | + * |
| 139 | + * Set to 0 to disable caching. |
139 | 140 | */ |
140 | | - expire-every = 1 hour; |
141 | | - expire-threshold = 5; |
| 141 | + cache-memory = 5MB; |
| 142 | + |
| 143 | + /* |
| 144 | + * The largest object to cache. An object larger than this |
| 145 | + * will never be cached. Set to 0 (the default) to cache |
| 146 | + * objects of any size up to cache-memory. |
| 147 | + */ |
| 148 | + max-entity-size = 256KB; |
142 | 149 | }; |
143 | 150 | |
144 | 151 | /* |
— | — | @@ -219,19 +226,6 @@ |
220 | 227 | max-redirects = 1; |
221 | 228 | }; |
222 | 229 | |
223 | | -/* |
224 | | - * For now, only one cache directory is allowed. |
225 | | - * Run "willow -z" to create the initial directory. Make sure it |
226 | | - * doesn't exist first. |
227 | | - * |
228 | | - * If you specify no cache directories, Willow will switch to a more |
229 | | - * efficient proxy-only mode. This is most useful in combination with |
230 | | - * CARP hashing in front of another cache. |
231 | | - */ |
232 | | -cache-dir "/home/kate/mediawiki/willow/cache" { |
233 | | - size = 10 mb; |
234 | | -}; |
235 | | - |
236 | 230 | log { |
237 | 231 | /* |
238 | 232 | * Levels: |