r17726 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r17725‎ \| r17726 \| r17727 >
Date:	18:40, 16 November 2006
Author:	river
Status:	old
Tags:
Comment:	start on-disk backing for memory cache
Modified paths:	/trunk/willow/configure.in (modified) (history) /trunk/willow/src/include/cache.h (modified) (history) /trunk/willow/src/include/dbwrap.h (added) (history) /trunk/willow/src/include/flowio.h (modified) (history) /trunk/willow/src/include/wconfig.h (modified) (history) /trunk/willow/src/include/willow.h (modified) (history) /trunk/willow/src/willow/Makefile.in (modified) (history) /trunk/willow/src/willow/cache.cc (modified) (history) /trunk/willow/src/willow/chunking.cc (modified) (history) /trunk/willow/src/willow/dbwrap.cc (added) (history) /trunk/willow/src/willow/wconfig.cc (modified) (history) /trunk/willow/src/willow/willow.cc (modified) (history)

Diff [purge]

Index: trunk/willow/configure.in
—	—	@@ -120,6 +120,7 @@
121	121	AC_SEARCH_LIBS(sendfile, sendfile)
122	122	AC_SEARCH_LIBS(zlibVersion, z,,AC_MSG_ERROR([zlib not found]))
123	123	AC_SEARCH_LIBS(pthread_key_create, pthread)
	124	+AC_SEARCH_LIBS(db_create, db-4.3)
124	125
125	126	AC_CHECK_FUNC(strlcat,AC_DEFINE([HAVE_STRLCAT],,[Define this if you have strlcat()]),[AC_LIBOBJ(strlcat)])
126	127	AC_CHECK_FUNC(strlcpy,AC_DEFINE([HAVE_STRLCPY],,[Define this if you have strlcpy()]),[AC_LIBOBJ(strlcpy)])
Index: trunk/willow/src/include/wconfig.h
—	—	@@ -76,6 +76,7 @@
77	77	bool use_dio;
78	78	long cache_memory;
79	79	long max_entity_size;
	80	+ string cache_master;
80	81	vector<pair<string, string> > stats_hosts;
81	82	} config;
82	83
Index: trunk/willow/src/include/flowio.h
—	—	@@ -306,7 +306,7 @@
307	307	sink_result data_ready(char const *buf, size_t len, ssize_t &discard) {
308	308	sink_result result;
309	309	ssize_t d = discard;
310		~~- while ((discard - d) < len) {~~
	310	+ while ((size_t)(discard - d) < len) {
311	311	result = bf_transform(buf + (discard - d), len - (discard - d), discard);
312	312	if (result != sink_result_okay &&
313	313	result != sink_result_finished)
Index: trunk/willow/src/include/cache.h
—	—	@@ -22,10 +22,26 @@
23	23	#include "flowio.h"
24	24	#include "whttp_header.h"
25	25	#include "format.h"
	26	+#include "dbwrap.h"
26	27
27	28	struct caching_filter;
28	29	struct cached_spigot;
29	30
	31	+namespace db {
	32	+ template<>
	33	+ struct marshaller<imstring> {
	34	+ pair<char const *, uint32_t> marshall(imstring const &s) {
	35	+ char *b = new char[s.size()];
	36	+ memcpy(b, s.data(), s.size());
	37	+ return pair<char const *, uint32_t>(b, s.size());
	38	+ }
	39	+
	40	+ imstring unmarhall(pair<char const , uint32_t> const &d) {
	41	+ return new imstring(d.first, d.second);
	42	+ }
	43	+ };
	44	+};
	45	+
30	46	struct cachedentity {
31	47	~cachedentity(void);
32	48
—	—	@@ -73,55 +89,11 @@
74	90	/*
75	91	* If the object is still valid, its lifetime can increase.
76	92	*/
77		~~- _lifetime = (time(0) - _modified) * 1.25;~~
	93	+ _lifetime = (time_t) ((time(0) - _modified) * 1.25);
78	94	_revalidate_at = time(0) + _lifetime;
79	95	}
80	96
81		~~- void set_complete(void) {~~
82		~~- header *h;~~
83		~~- WDEBUG((WLOG_DEBUG, format("set_complete: void=%d") % _void));~~
84		~~- if (_void)~~
85		~~- return;~~
86		~~- _headers.remove("transfer-encoding");~~
87		~~- if (!_headers.find("content-length")) {~~
88		~~- char lenstr[64];~~
89		~~- snprintf(lenstr, sizeof lenstr, "%lu",~~
90		~~- (unsigned long) _data.size());~~
91		~~- _headers.add("Content-Length", lenstr);~~
92		~~- }~~
93		-
94		~~- if ((h = _headers.find("Expires")) != NULL) {~~
95		~~- if ((_expires = parse_date(h->value())) == -1) {~~
96		~~- _expires = time(0);~~
97		~~- }~~
98		~~- } else {~~
99		~~- _expires = 0;~~
100		~~- }~~
101		-
102		~~- if ((h = _headers.find("Last-Modified")) != NULL) {~~
103		~~- if ((_modified = parse_date(h->value())) == -1) {~~
104		~~- _modified = time(0);~~
105		~~- }~~
106		~~- } else {~~
107		~~- if ((h = _headers.find("Date")) != NULL) {~~
108		~~- if ((_modified = parse_date(h->value())) == -1) {~~
109		~~- _modified = time(0);~~
110		~~- }~~
111		~~- } else {~~
112		~~- _modified = time(0);~~
113		~~- }~~
114		~~- }~~
115		-
116		~~- _lifetime = (time(0) - _modified) * 1.25;~~
117		~~- WDEBUG((WLOG_DEBUG, format("object lifetime=%d sec.") % _lifetime));~~
118		~~- revalidated();~~
119		~~- _builthdrs = _headers.build();~~
120		~~- _builtsz = _headers.length();~~
121		~~- _data.finished();~~
122		-
123		~~- _complete = true;~~
124		~~- }~~
125		-
	97	+ void set_complete(void);
126	98	void store_status(imstring const &status) {
127	99	_status = status;
128	100	}
—	—	@@ -138,6 +110,9 @@
139	111
140	112	static time_t parse_date(char const *date);
141	113
	114	+ pair<char const *, uint32_t> marshall(void) const;
	115	+ static cachedentity unmarshall(char const , uint32_t);
	116	+
142	117	private:
143	118	friend struct httpcache;
144	119	friend struct caching_filter;
—	—	@@ -173,10 +148,14 @@
174	149	httpcache();
175	150	~httpcache();
176	151
177		~~- cachedentity *find_cached(imstring const &url, bool create, bool& wasnew);~~
178		~~- void release(cachedentity *);~~
179		~~- bool purge(imstring const &url);~~
	152	+ bool open(void);
	153	+ void close(void);
	154	+ bool create(void);
180	155
	156	+ cachedentity *find_cached(imstring const &url, bool create, bool& wasnew);
	157	+ void release(cachedentity *);
	158	+ bool purge(imstring const &url);
	159	+
181	160	private:
182	161	friend struct cachedentity;
183	162	friend struct caching_filter;
—	—	@@ -191,16 +170,22 @@
192	171
193	172	typedef multiset<entmap::iterator, lru_comparator> lruset;
194	173
195		~~- void _remove(cachedentity *ent);~~
196		~~- void _remove_unlocked(cachedentity *ent);~~
	174	+ void _remove(cachedentity *ent);
	175	+ void _remove_unlocked(cachedentity *ent);
	176	+ void _swap_out(cachedentity *);
	177	+ cachedentity *_swap_in(imstring const &url);
197	178
198	179	entmap _entities;
199	180	lruset _lru;
200	181	lockable _lock, _memlock;
201	182	size_t _cache_mem;
202	183
	184	+ db::environment *_env;
	185	+ db::database<imstring, cachedentity>
	186	+ *_db;
	187	+
203	188	void cache_mem_reduce(size_t);
204		~~- bool cache_mem_increase(size_t);~~
	189	+ bool cache_mem_increase(size_t, cachedentity *);
205	190	};
206	191
207	192	struct caching_filter : io::sink, io::spigot {
—	—	@@ -239,8 +224,8 @@
240	225	, _done(false)
241	226	, _keepalive(false)
242	227	, _doneheaders(false)
243		~~- , _inited(false)~~
244	228	, _corked(true)
	229	+ , _inited(false)
245	230	, _off(0) {}
246	231
247	232	~cached_spigot() {
—	—	@@ -314,7 +299,7 @@
315	300	WDEBUG((WLOG_DEBUG, "all finished"));
316	301	return;
317	302	case io::sink_result_okay:
318		~~- if (_off = _ent->_data.size()) {~~
	303	+ if (_off == _ent->_data.size()) {
319	304	_sp_completed_callee();
320	305	return;
321	306	}
Index: trunk/willow/src/include/willow.h
—	—	@@ -476,15 +476,15 @@
477	477
478	478	void resize(size_t newsize);
479	479
480		~~- char *ptr(void) {~~
	480	+ char *ptr(void) const {
481	481	return _buf;
482	482	}
483	483
484		~~- size_t size(void) {~~
	484	+ size_t size(void) const {
485	485	return _size;
486	486	}
487	487
488		~~- int fd(void) {~~
	488	+ int fd(void) const {
489	489	return _fd;
490	490	}
491	491
Index: trunk/willow/src/include/dbwrap.h
—	—	@@ -0,0 +1,392 @@
	2	+/* @(#) $Id$ */
	3	+/* This source code is in the public domain. */
	4	+/*
	5	+ * Willow: Lightweight HTTP reverse-proxy.
	6	+ * dbwrap: C++ Berkeley DB wrapper.
	7	+ */
	8	+
	9	+#ifndef DBWRAP_H
	10	+#define DBWRAP_H
	11	+
	12	+#if defined __SUNPRO_C \|\| defined __DECC \|\| defined __HP_cc
	13	+# pragma ident "@(#)$Id$"
	14	+#endif
	15	+
	16	+#include <sys/types.h>
	17	+
	18	+#include <algorithm>
	19	+using std::back_inserter;
	20	+
	21	+#include <db.h>
	22	+
	23	+#include "willow.h"
	24	+#include "util.h"
	25	+
	26	+namespace db {
	27	+
	28	+template<typename Key, typename Value>
	29	+struct database;
	30	+struct transaction;
	31	+
	32	+struct environment : noncopyable {
	33	+ static environment *open(string const &path);
	34	+ static environment *create(string const &path);
	35	+ ~environment();
	36	+
	37	+ template<typename Key, typename Value>
	38	+ database<Key, Value> *open_database(string const &path);
	39	+
	40	+ template<typename Key, typename Value>
	41	+ database<Key, Value> *create_database(string const &path);
	42	+
	43	+ int error (void) const;
	44	+ string strerror (void) const;
	45	+ void close (void);
	46	+
	47	+ struct transaction *transaction(void);
	48	+
	49	+private:
	50	+ template<typename Key, typename Value>
	51	+ friend struct database;
	52	+ friend struct transaction;
	53	+
	54	+ explicit environment(string const &path, uint32_t flags);
	55	+
	56	+ DB_ENV *_env;
	57	+ int _error;
	58	+};
	59	+
	60	+template<typename Key, typename Value>
	61	+struct database : noncopyable {
	62	+ int error (void) const;
	63	+ string strerror (void) const;
	64	+ void close (void);
	65	+
	66	+ ~database();
	67	+
	68	+ bool put(Key const &key, Value const &value, transaction *);
	69	+ bool put(Key const &key, Value const &value);
	70	+ Value get(Key const &key, transaction );
	71	+ Value *get(Key const &key);
	72	+
	73	+private:
	74	+ friend struct environment;
	75	+
	76	+ explicit database(environment *env, string const &path, uint32_t flags);
	77	+ static void errcall(DB_ENV const , char const pfx, char const *msg);
	78	+
	79	+ DB *_db;
	80	+ environment *_env;
	81	+ int _error;
	82	+};
	83	+
	84	+struct transaction {
	85	+ ~transaction();
	86	+
	87	+ bool commit(void);
	88	+ bool rollback(void);
	89	+
	90	+ int error(void) const;
	91	+ string strerror(void) const;
	92	+
	93	+private:
	94	+ friend struct environment;
	95	+ template<typename Key, typename Value>
	96	+ friend struct database;
	97	+
	98	+ transaction(environment *);
	99	+ DB_TXN *_txn;
	100	+ int _error;
	101	+};
	102	+
	103	+struct marshalling_buffer {
	104	+ marshalling_buffer()
	105	+ : _buf(NULL)
	106	+ , _size(0)
	107	+ , _bufsz(0)
	108	+ {}
	109	+
	110	+ marshalling_buffer(char const *buf, uint32_t sz)
	111	+ : _buf(const_cast<char *>(buf))
	112	+ , _size(0)
	113	+ , _bufsz(sz)
	114	+ {}
	115	+
	116	+ ~marshalling_buffer(void) {
	117	+ }
	118	+
	119	+ void reserve(size_t size) {
	120	+ _bufsz = size;
	121	+ _buf = new char[size];
	122	+ }
	123	+
	124	+ template<typename T>
	125	+ void append(T const &);
	126	+
	127	+ template<typename charT, typename traits, typename allocator>
	128	+ void append(basic_string<charT, traits, allocator> const &);
	129	+
	130	+ void append_bytes(char const *buf, size_t s) {
	131	+std::cout<<"appending "<<s<<" bytes\n";
	132	+ assert(_size + s <= _bufsz);
	133	+ memcpy(_buf + _size, buf, s);
	134	+ _size += s;
	135	+ }
	136	+
	137	+ char const *buffer(void) const {
	138	+ return _buf;
	139	+ }
	140	+
	141	+ size_t size(void) const {
	142	+ return _size;
	143	+ }
	144	+
	145	+ template<typename T>
	146	+ bool extract(T &);
	147	+
	148	+ template<typename charT, typename traits, typename allocator>
	149	+ bool extract(basic_string<charT, traits, allocator> &);
	150	+
	151	+ bool extract_bytes(char *b, size_t s) {
	152	+std::cout<<"extracting "<<s<<" bytes\n";
	153	+ if (_size + s > _bufsz)
	154	+ return false;
	155	+ memcpy(b, _buf + _size, s);
	156	+ _size += s;
	157	+ return true;
	158	+ }
	159	+
	160	+private:
	161	+ char *_buf;
	162	+ size_t _size;
	163	+ size_t _bufsz;
	164	+ bool _delete;
	165	+};
	166	+
	167	+template<typename T>
	168	+void
	169	+marshalling_buffer::append(T const &o) {
	170	+ append_bytes((char const *)&o, sizeof(o));
	171	+}
	172	+
	173	+template<>
	174	+void
	175	+marshalling_buffer::append<imstring>(imstring const &o);
	176	+
	177	+template<typename charT, typename traits, typename allocator>
	178	+void
	179	+marshalling_buffer::append(basic_string<charT, traits, allocator> const &s) {
	180	+ append<size_t>(s.size() * sizeof(charT));
	181	+ append_bytes(s.data(), s.size() * sizeof(charT));
	182	+}
	183	+
	184	+template<typename T>
	185	+bool
	186	+marshalling_buffer::extract(T &o)
	187	+{
	188	+ return extract_bytes((char *) &o, sizeof(o));
	189	+}
	190	+
	191	+template<typename charT, typename traits, typename allocator>
	192	+bool
	193	+marshalling_buffer::extract(basic_string<charT, traits, allocator> &s)
	194	+{
	195	+size_t sz;
	196	+ if (!extract<size_t>(sz))
	197	+ return false;
	198	+ if (_size + sz > _bufsz)
	199	+ return false;
	200	+ s.reserve(sz);
	201	+ copy(_buf + _size, _buf + _size + sz, back_inserter<charT>(s));
	202	+ _size += sz;
	203	+ return true;
	204	+}
	205	+
	206	+template<typename T>
	207	+struct marshaller {
	208	+};
	209	+
	210	+template<>
	211	+struct marshaller<char> {
	212	+ pair<char const *, uint32_t> marshall(char c) {
	213	+ return make_pair(&c, sizeof(c));
	214	+ }
	215	+};
	216	+
	217	+template<>
	218	+struct marshaller<int> {
	219	+ pair<char const *, uint32_t> marshall(int c) {
	220	+ return make_pair((char const *)&c, sizeof(c));
	221	+ }
	222	+};
	223	+
	224	+template<>
	225	+struct marshaller<long> {
	226	+ pair<char const *, uint32_t> marshall(long c) {
	227	+ return make_pair((char const *)&c, sizeof(c));
	228	+ }
	229	+};
	230	+
	231	+template<>
	232	+struct marshaller<unsigned long> {
	233	+ pair<char const *, uint32_t> marshall(unsigned long c) {
	234	+ return make_pair((char const *)&c, sizeof(c));
	235	+ }
	236	+};
	237	+
	238	+template<>
	239	+struct marshaller<unsigned int> {
	240	+ pair<char const *, uint32_t> marshall(unsigned int c) {
	241	+ return make_pair((char const *)&c, sizeof(c));
	242	+ }
	243	+};
	244	+
	245	+template<>
	246	+struct marshaller<string> {
	247	+ pair<char const *, uint32_t> marshall(string const &s) {
	248	+ return make_pair(s.data(), s.size());
	249	+ }
	250	+};
	251	+
	252	+template<typename Key, typename Value>
	253	+database<Key, Value> *
	254	+environment::open_database(string const &name)
	255	+{
	256	+ return new database<Key, Value>(this, name, 0);
	257	+}
	258	+
	259	+template<typename Key, typename Value>
	260	+database<Key, Value> *
	261	+environment::create_database(string const &name)
	262	+{
	263	+ return new database<Key, Value>(this, name, DB_CREATE);
	264	+}
	265	+
	266	+template<typename Key, typename Value>
	267	+database<Key, Value>::database(environment *env, string const &path, uint32_t flags)
	268	+ : _env(env)
	269	+{
	270	+ _error = db_create(&_db, env->_env, 0);
	271	+ if (_error != 0) {
	272	+ _db->close(_db, 0);
	273	+ _db = NULL;
	274	+ return;
	275	+ }
	276	+
	277	+ _error = _db->open(_db, NULL, path.c_str(), NULL, DB_HASH,
	278	+ DB_THREAD \| DB_AUTO_COMMIT \| flags, 0);
	279	+ if (_error != 0) {
	280	+ _db->close(_db, 0);
	281	+ _db = NULL;
	282	+ return;
	283	+ }
	284	+
	285	+ _db->set_errcall(_db, &database::errcall);
	286	+}
	287	+
	288	+template<typename Key, typename Value>
	289	+void
	290	+database<Key, Value>::errcall(DB_ENV const , char const pfx, char const *msg)
	291	+{
	292	+ if (pfx)
	293	+ wlog(WLOG_WARNING, format("%s: %s") % pfx % msg);
	294	+ else
	295	+ wlog(WLOG_WARNING, msg);
	296	+}
	297	+
	298	+template<typename Key, typename Value>
	299	+int
	300	+database<Key, Value>::error(void) const
	301	+{
	302	+ return _error;
	303	+}
	304	+
	305	+template<typename Key, typename Value>
	306	+string
	307	+database<Key, Value>::strerror(void) const
	308	+{
	309	+ return db_strerror(_error);
	310	+}
	311	+
	312	+template<typename Key, typename Value>
	313	+database<Key, Value>::~database(void)
	314	+{
	315	+ if (_db)
	316	+ _db->close(_db, 0);
	317	+}
	318	+
	319	+template<typename Key, typename Value>
	320	+bool
	321	+database<Key, Value>::put(Key const &key, Value const &value)
	322	+{
	323	+ return put (key, value, NULL);
	324	+}
	325	+
	326	+template<typename Key, typename Value>
	327	+bool
	328	+database<Key, Value>::put(Key const &key, Value const &value, transaction *txn)
	329	+{
	330	+pair<char const *, uint32_t> mkey, mvalue;
	331	+DBT dbkey, dbvalue;
	332	+marshaller<Key> keymarsh;
	333	+marshaller<Value> valuemarsh;
	334	+ memset(&dbkey, 0, sizeof(dbkey));
	335	+ memset(&dbvalue, 0, sizeof(dbvalue));
	336	+ mkey = keymarsh.marshall(key);
	337	+ mvalue = valuemarsh.marshall(value);
	338	+ dbkey.data = (void *) mkey.first;
	339	+ dbkey.size = mkey.second;
	340	+ dbvalue.data = (void *) mvalue.first;
	341	+ dbvalue.size = mvalue.second;
	342	+
	343	+ _error = _db->put(_db, txn ? txn->_txn : NULL, &dbkey, &dbvalue,
	344	+ DB_NOOVERWRITE \| (txn ? 0 : DB_AUTO_COMMIT));
	345	+ delete[] mkey.first;
	346	+ delete[] mvalue.first;
	347	+ if (_error != 0)
	348	+ return false;
	349	+ return true;
	350	+}
	351	+
	352	+template<typename Key, typename Value>
	353	+Value *
	354	+database<Key, Value>::get(Key const &key)
	355	+{
	356	+ return get(key, NULL);
	357	+}
	358	+
	359	+template<typename Key, typename Value>
	360	+Value *
	361	+database<Key, Value>::get(Key const &key, transaction *txn)
	362	+{
	363	+pair<char const *, uint32_t> mkey;
	364	+DBT dbkey, dbvalue;
	365	+marshaller<Key> keymarsh;
	366	+marshaller<Value> vmarsh;
	367	+ memset(&dbkey, 0, sizeof(dbkey));
	368	+ memset(&dbvalue, 0, sizeof(dbvalue));
	369	+ mkey = keymarsh.marshall(key);
	370	+ dbkey.data = (void *) mkey.first;
	371	+ dbkey.size = mkey.second;
	372	+ dbvalue.flags = DB_DBT_MALLOC;
	373	+ _error = _db->get(_db, txn ? txn->_txn : NULL, &dbkey, &dbvalue, 0);
	374	+ if (_error != 0)
	375	+ return NULL;
	376	+Value *ret;
	377	+ ret = vmarsh.unmarshall(pair<char const *, uint32_t>(
	378	+ (char const *) dbvalue.data, dbvalue.size));
	379	+ free(dbvalue.data);
	380	+ return ret;
	381	+}
	382	+
	383	+template<typename Key, typename Value>
	384	+void
	385	+database<Key, Value>::close(void)
	386	+{
	387	+ _db->close(_db, 0);
	388	+ _db = NULL;
	389	+}
	390	+
	391	+} // namespace db
	392	+
	393	+#endif
Property changes on: trunk/willow/src/include/dbwrap.h
___________________________________________________________________
Added: svn:keywords
1	394	+ Id Revision
Index: trunk/willow/src/willow/wconfig.cc
—	—	@@ -365,6 +365,7 @@
366	366	.block("cache")
367	367	.value("cache-memory", simple_time, set_long(config.cache_memory))
368	368	.value("max-entity-size", simple_time, set_long(config.max_entity_size))
	369	+ .value("master-state", nonempty_qstring, set_string(config.cache_master))
369	370
370	371	.block("http")
371	372	.value("compress", simple_yesno, set_yesno(config.compress))
Index: trunk/willow/src/willow/Makefile.in
—	—	@@ -6,6 +6,7 @@
7	7	cache.cc \
8	8	chunking.cc \
9	9	confparse.cc \
	10	+ dbwrap.cc \
10	11	flowio.cc \
11	12	format.cc \
12	13	radix.cc \
Index: trunk/willow/src/willow/cache.cc
—	—	@@ -16,15 +16,33 @@
17	17	#include "format.h"
18	18	#include "wconfig.h"
19	19
	20	+namespace db {
	21	+
	22	+template<>
	23	+struct marshaller<cachedentity> {
	24	+ pair<char const *, uint32_t> marshall(cachedentity const &e) {
	25	+ return e.marshall();
	26	+ }
	27	+
	28	+ cachedentity unmarshall(pair<char const , uint32_t> const &d) {
	29	+ return cachedentity::unmarshall(d.first, d.second);
	30	+ }
	31	+};
	32	+
	33	+} // namespace db
	34	+
20	35	httpcache entitycache;
21	36
22	37	httpcache::httpcache(void)
23	38	: _cache_mem(0)
	39	+ , _env(NULL)
	40	+ , _db(NULL)
24	41	{
25	42	}
26	43
27	44	httpcache::~httpcache(void)
28	45	{
	46	+ close();
29	47	}
30	48
31	49	cachedentity *
—	—	@@ -51,6 +69,22 @@
52	70	return ret;
53	71	}
54	72
	73	+ /*
	74	+ * Maybe it's in the disk cache
	75	+ */
	76	+ ret = _db->get(url);
	77	+ if (ret != NULL) {
	78	+ WDEBUG((WLOG_DEBUG, format("found [%s] in disk cache") %url));
	79	+ ret->ref();
	80	+ _lru.insert(_entities.insert(make_pair(url, ret)).first);
	81	+ wasnew = false;
	82	+ return ret;
	83	+ }
	84	+ if (_db->error()) {
	85	+ wlog(WLOG_WARNING, format("fetching cached data: %s")
	86	+ % _db->strerror());
	87	+ }
	88	+
55	89	WDEBUG((WLOG_DEBUG, format("[%s] not cached") % url));
56	90	if (!create)
57	91	return NULL;
—	—	@@ -100,20 +134,25 @@
101	135	}
102	136
103	137	bool
104		~~-httpcache::cache_mem_increase(size_t n)~~
	138	+httpcache::cache_mem_increase(size_t n, cachedentity *self)
105	139	{
106	140	for (;;) {
107	141	{ HOLDING(_memlock);
108		~~- if (_cache_mem + n <= config.cache_memory)~~
	142	+ if ((long) (_cache_mem + n) <= config.cache_memory)
109	143	break;
110	144	}
111	145	cachedentity *ent;
112	146	{ HOLDING(_lock);
113		~~- lruset::iterator it;~~
114		~~- if ((it = _lru.begin()) == _lru.end())~~
	147	+ lruset::iterator it = _lru.begin();
	148	+ for (; it != _lru.end(); ++it) {
	149	+ ent = (*it)->second;
	150	+ if (ent == self)
	151	+ continue;
	152	+ ent->deref();
	153	+ break;
	154	+ }
	155	+ if (ent == self)
115	156	return false;
116		~~- ent = (*it)->second;~~
117		~~- ent->deref();~~
118	157	}
119	158	}
120	159
—	—	@@ -122,6 +161,19 @@
123	162	return true;
124	163	}
125	164
	165	+void
	166	+httpcache::_swap_out(cachedentity *ent)
	167	+{
	168	+ WDEBUG((WLOG_DEBUG, format("swapping out %s") % ent->url()));
	169	+ if (!_db)
	170	+ return;
	171	+ _db->put(ent->url(), *ent);
	172	+ if (_db->error()) {
	173	+ wlog(WLOG_WARNING, format("storing cached data: %s")
	174	+ % _db->strerror());
	175	+ }
	176	+}
	177	+
126	178	bool
127	179	httpcache::purge(imstring const &url)
128	180	{
—	—	@@ -141,6 +193,90 @@
142	194	return true;
143	195	}
144	196
	197	+bool
	198	+httpcache::open(void)
	199	+{
	200	+ if (config.cache_master.empty())
	201	+ return true;
	202	+
	203	+ _env = db::environment::open(config.cache_master);
	204	+ if (_env->error()) {
	205	+ wlog(WLOG_ERROR,
	206	+ format("cannot open cache master environment \"%s\": %s")
	207	+ % config.cache_master % _env->strerror());
	208	+ delete _env;
	209	+ _env = NULL;
	210	+ return false;
	211	+ }
	212	+
	213	+ _db = _env->open_database<imstring, cachedentity>("objects");
	214	+ if (_db->error()) {
	215	+ wlog(WLOG_ERROR,
	216	+ format("cannot open cache master database \"%s\": %s")
	217	+ % config.cache_master % _db->strerror());
	218	+ _env->close();
	219	+ delete _env;
	220	+ delete _db;
	221	+ _env = NULL;
	222	+ _db = NULL;
	223	+ return false;
	224	+ }
	225	+ return true;
	226	+}
	227	+
	228	+bool
	229	+httpcache::create(void)
	230	+{
	231	+ if (config.cache_master.empty()) {
	232	+ wlog(WLOG_ERROR, "no cache master to create");
	233	+ return false;
	234	+ }
	235	+
	236	+ if (mkdir(config.cache_master.c_str(), 0700) < 0) {
	237	+ wlog(WLOG_ERROR, format("cannot create cache master \"%s\": %e")
	238	+ % config.cache_master);
	239	+ return false;
	240	+ }
	241	+
	242	+ _env = db::environment::create(config.cache_master);
	243	+ if (_env->error()) {
	244	+ wlog(WLOG_ERROR,
	245	+ format("cannot create cache master environmet \"%s\": %s")
	246	+ % config.cache_master % _env->strerror());
	247	+ delete _env;
	248	+ _env = NULL;
	249	+ return false;
	250	+ }
	251	+
	252	+ _db = _env->create_database<imstring, cachedentity>("objects");
	253	+ if (_db->error()) {
	254	+ wlog(WLOG_ERROR,
	255	+ format("cannot create cache master database \"%s\": %s")
	256	+ % config.cache_master % _db->strerror());
	257	+ _env->close();
	258	+ delete _env;
	259	+ delete _db;
	260	+ _env = NULL;
	261	+ _db = NULL;
	262	+ return false;
	263	+ }
	264	+ return true;
	265	+}
	266	+
	267	+void
	268	+httpcache::close(void)
	269	+{
	270	+ if (_db) {
	271	+ _db->close();
	272	+ _db = NULL;
	273	+ }
	274	+
	275	+ if (_env) {
	276	+ _env->close();
	277	+ _env = NULL;
	278	+ }
	279	+}
	280	+
145	281	cachedentity::cachedentity(imstring const &url, size_t hint)
146	282	: _url(url)
147	283	, _data(hint ? hint : 4096)
—	—	@@ -168,12 +304,12 @@
169	305	if (_void)
170	306	return;
171	307
172		~~- if (_data.size() + size > config.max_entity_size) {~~
	308	+ if ((long) (_data.size() + size) > config.max_entity_size) {
173	309	_void = true;
174	310	return;
175	311	}
176	312
177		~~- if (!entitycache.cache_mem_increase(size)) {~~
	313	+ if (!entitycache.cache_mem_increase(size, this)) {
178	314	WDEBUG((WLOG_DEBUG, "object is too large, voiding cache"));
179	315	_void = true;
180	316	return;
—	—	@@ -190,3 +326,142 @@
191	327	return (time_t) -1;
192	328	return mktime(&tm);
193	329	}
	330	+
	331	+void
	332	+cachedentity::set_complete(void)
	333	+{
	334	+header *h;
	335	+ WDEBUG((WLOG_DEBUG, format("set_complete: void=%d") % _void));
	336	+ if (_void)
	337	+ return;
	338	+ _headers.remove("transfer-encoding");
	339	+ if (!_headers.find("content-length")) {
	340	+ char lenstr[64];
	341	+ snprintf(lenstr, sizeof lenstr, "%lu",
	342	+ (unsigned long) _data.size());
	343	+ _headers.add("Content-Length", lenstr);
	344	+ }
	345	+
	346	+ if ((h = _headers.find("Expires")) != NULL) {
	347	+ if ((_expires = parse_date(h->value())) == -1) {
	348	+ _expires = time(0);
	349	+ }
	350	+ } else {
	351	+ _expires = 0;
	352	+ }
	353	+
	354	+ if ((h = _headers.find("Last-Modified")) != NULL) {
	355	+ if ((_modified = parse_date(h->value())) == -1) {
	356	+ _modified = time(0);
	357	+ }
	358	+ } else {
	359	+ if ((h = _headers.find("Date")) != NULL) {
	360	+ if ((_modified = parse_date(h->value())) == -1) {
	361	+ _modified = time(0);
	362	+ }
	363	+ } else {
	364	+ _modified = time(0);
	365	+ }
	366	+ }
	367	+
	368	+ _lifetime = (time_t) ((time(0) - _modified) * 1.25);
	369	+ WDEBUG((WLOG_DEBUG, format("object lifetime=%d sec.") % _lifetime));
	370	+ revalidated();
	371	+ _builthdrs = _headers.build();
	372	+ _builtsz = _headers.length();
	373	+ _data.finished();
	374	+ entitycache._swap_out(this);
	375	+ _complete = true;
	376	+}
	377	+
	378	+pair<char const *, uint32_t>
	379	+cachedentity::marshall(void) const
	380	+{
	381	+db::marshalling_buffer buf;
	382	+ buf.reserve(
	383	+ sizeof(size_t) + _url.size() +
	384	+ sizeof(size_t) + _status.size() +
	385	+ sizeof(size_t) + _data.size() +
	386	+ sizeof(size_t) + _builtsz +
	387	+ sizeof(time_t) * 5);
	388	+ buf.append<imstring>(_url);
	389	+ buf.append<imstring>(_status);
	390	+ buf.append<size_t>(_data.size());
	391	+ buf.append_bytes(_data.ptr(), _data.size());
	392	+ buf.append<size_t>(_builtsz);
	393	+ buf.append_bytes(_builthdrs, _builtsz);
	394	+ buf.append<time_t>(_lastuse);
	395	+ buf.append<time_t>(_expires);
	396	+ buf.append<time_t>(_modified);
	397	+ buf.append<time_t>(_lifetime);
	398	+ buf.append<time_t>(_revalidate_at);
	399	+ return make_pair(buf.buffer(), buf.size());
	400	+}
	401	+
	402	+cachedentity *
	403	+cachedentity::unmarshall(char const *d, uint32_t s)
	404	+{
	405	+cachedentity *ret;
	406	+db::marshalling_buffer buf(d, s);
	407	+imstring url;
	408	+char *hdrbuf;
	409	+size_t bufsz;
	410	+ if (!buf.extract<imstring>(url))
	411	+ return NULL;
	412	+ ret = new cachedentity(url);
	413	+ if (!buf.extract<imstring>(ret->_status)) {
	414	+ delete ret;
	415	+ return NULL;
	416	+ }
	417	+
	418	+ if (!buf.extract<size_t>(bufsz)) {
	419	+ delete ret;
	420	+ return NULL;
	421	+ }
	422	+
	423	+ ret->_data.resize(bufsz);
	424	+
	425	+ if (!buf.extract_bytes(ret->_data.ptr(), bufsz)) {
	426	+ delete ret;
	427	+ return NULL;
	428	+ }
	429	+
	430	+ if (!buf.extract<size_t>(bufsz)) {
	431	+ delete ret;
	432	+ return NULL;
	433	+ }
	434	+
	435	+ ret->_builthdrs = new char[bufsz + 1];
	436	+
	437	+ if (!buf.extract_bytes(ret->_builthdrs, bufsz)) {
	438	+ delete ret;
	439	+ return NULL;
	440	+ }
	441	+
	442	+ if (!buf.extract<time_t>(ret->_lastuse)) {
	443	+ delete ret;
	444	+ return NULL;
	445	+ }
	446	+
	447	+ if (!buf.extract<time_t>(ret->_expires)) {
	448	+ delete ret;
	449	+ return NULL;
	450	+ }
	451	+
	452	+ if (!buf.extract<time_t>(ret->_modified)) {
	453	+ delete ret;
	454	+ return NULL;
	455	+ }
	456	+
	457	+ if (!buf.extract<time_t>(ret->_lifetime)) {
	458	+ delete ret;
	459	+ return NULL;
	460	+ }
	461	+
	462	+ if (!buf.extract<time_t>(ret->_revalidate_at)) {
	463	+ delete ret;
	464	+ return NULL;
	465	+ }
	466	+ ret->_refs = 1;
	467	+ return ret;
	468	+}
Index: trunk/willow/src/willow/chunking.cc
—	—	@@ -45,11 +45,11 @@
46	46	}
47	47
48	48	dechunking_filter::dechunking_filter()
49		~~- : _current_chunk_size(0)~~
	49	+ : _state(s_start)
	50	+ , _current_chunk_size(0)
50	51	, _counter(0)
51	52	, _atend(false)
52	53	, _first(true)
53		~~- , _state(s_start)~~
54	54	{
55	55	_cbuf[0] = '\0';
56	56	}
Index: trunk/willow/src/willow/willow.cc
—	—	@@ -29,7 +29,7 @@
30	30	#include "wconfig.h"
31	31	#include "willow.h"
32	32	#include "whttp.h"
33		~~-#include "wcache.h"~~
	33	+#include "cache.h"
34	34	#include "confparse.h"
35	35	#include "radix.h"
36	36	#include "format.h"
—	—	@@ -76,6 +76,7 @@
77	77	" -h print this message\n"
78	78	" -f run in foreground (don't detach)\n"
79	79	" -v print version number and exit\n"
	80	+" -z create initial cache directories and exit\n"
80	81	" -D cond[=value] set 'cond' to 'value' (which should be 'true' or\n"
81	82	" 'false') in the configuration parser. if 'value'\n"
82	83	" is not specified, defaults to true\n"
—	—	@@ -109,11 +110,11 @@
110	111	int i;
111	112	char *cfg = NULL;
112	113	char *dval;
113		-
	114	+bool zflag = false;
114	115	progname = argv[0];
115	116	pagesize = sysconf(_SC_PAGESIZE);
116	117
117		~~- while ((i = getopt(argc, argv, "fvc:D:h")) != -1) {~~
	118	+ while ((i = getopt(argc, argv, "fvc:D:hz")) != -1) {
118	119	switch (i) {
119	120	case 'h':
120	121	usage();
—	—	@@ -127,6 +128,10 @@
128	129	case 'c':
129	130	cfg = optarg;
130	131	break;
	132	+ case 'z':
	133	+ zflag = true;
	134	+ config.foreground = true;
	135	+ break;
131	136	case 'D':
132	137	dval = NULL;
133	138	if ((dval = strchr(optarg, '=')) != NULL) {
—	—	@@ -187,9 +192,18 @@
188	193
189	194	wlog_init();
190	195
	196	+ if (zflag) {
	197	+ if (!entitycache.create())
	198	+ return 1;
	199	+ return 0;
	200	+ }
	201	+
191	202	make_event_base();
192	203	ioloop = new ioloop_t;
193	204	checkexit_sched();
	205	+ if (!entitycache.open())
	206	+ return 1;
	207	+
194	208	whttp_init();
195	209	stats_init();
196	210
—	—	@@ -202,6 +216,7 @@
203	217	wlog(WLOG_NOTICE, "shutting down");
204	218	wlog_close();
205	219	whttp_shutdown();
	220	+ entitycache.close();
206	221
207	222	pthread_exit(NULL);
208	223	return EXIT_SUCCESS;
—	—	@@ -513,8 +528,6 @@
514	529	void
515	530	diobuf::finished(void)
516	531	{
517		~~- munmap(_buf, _reserved);~~
518		~~- _buf = NULL;~~
519	532	}
520	533
521	534	diobuf::~diobuf(void)
—	—	@@ -529,8 +542,8 @@
530	543	}
531	544
532	545	diobuf::diobuf(size_t size)
533		~~- : _buf(0)~~
534		~~- , _fd(-1)~~
	546	+ : _fd(-1)
	547	+ , _buf(0)
535	548	, _size(0)
536	549	, _reserved(pagesize * (size / pagesize + 1))
537	550	{
Index: trunk/willow/src/willow/dbwrap.cc
—	—	@@ -0,0 +1,146 @@
	2	+/* @(#) $Id$ */
	3	+/* This source code is in the public domain. */
	4	+/*
	5	+ * Willow: Lightweight HTTP reverse-proxy.
	6	+ * dbwrap: C++ Berkeley DB wrapper.
	7	+ */
	8	+
	9	+#if defined __SUNPRO_C \|\| defined __DECC \|\| defined __HP_cc
	10	+# pragma ident "@(#)$Id$"
	11	+#endif
	12	+
	13	+#include "dbwrap.h"
	14	+
	15	+namespace db {
	16	+
	17	+environment *
	18	+environment::open(string const &path)
	19	+{
	20	+ return new environment(path, 0);
	21	+}
	22	+
	23	+environment *
	24	+environment::create(string const &path)
	25	+{
	26	+ return new environment(path, DB_CREATE);
	27	+}
	28	+
	29	+environment::environment(string const &path, uint32_t flags)
	30	+ : _env(NULL)
	31	+ , _error(0)
	32	+{
	33	+ _error = db_env_create(&_env, 0);
	34	+ if (_error != 0) {
	35	+ _env = NULL;
	36	+ return;
	37	+ }
	38	+
	39	+ _env->set_errfile(_env, stderr);
	40	+
	41	+ _error = _env->open(_env, path.c_str(),
	42	+ DB_INIT_LOCK \| DB_INIT_LOG \| DB_INIT_TXN \| DB_RECOVER \|
	43	+ DB_INIT_MPOOL \| DB_THREAD \| DB_CREATE \| DB_RECOVER \| flags, 0);
	44	+ if (_error != 0) {
	45	+ _env = NULL;
	46	+ return;
	47	+ }
	48	+}
	49	+
	50	+int
	51	+environment::error(void) const
	52	+{
	53	+ return _error;
	54	+}
	55	+
	56	+string
	57	+environment::strerror(void) const
	58	+{
	59	+ return db_strerror(_error);
	60	+}
	61	+
	62	+environment::~environment(void)
	63	+{
	64	+ if (_env)
	65	+ _env->close(_env, 0);
	66	+}
	67	+
	68	+void
	69	+environment::close(void)
	70	+{
	71	+ _env->close(_env, 0);
	72	+ _env = NULL;
	73	+}
	74	+
	75	+transaction *
	76	+environment::transaction(void)
	77	+{
	78	+ return new struct transaction(this);
	79	+}
	80	+
	81	+transaction::transaction(environment *env)
	82	+{
	83	+ _error = env->_env->txn_begin(env->_env, NULL, &_txn, 0);
	84	+ if (_error != 0) {
	85	+ wlog(WLOG_WARNING, format("error starting transaction: %s")
	86	+ % strerror());
	87	+ _txn = NULL;
	88	+ }
	89	+}
	90	+
	91	+bool
	92	+transaction::commit(void)
	93	+{
	94	+ _error = _txn->commit(_txn, 0);
	95	+ _txn = NULL;
	96	+ return _error == 0;
	97	+}
	98	+
	99	+bool
	100	+transaction::rollback(void)
	101	+{
	102	+ _error = _txn->abort(_txn);
	103	+ _txn = NULL;
	104	+ return _error == 0;
	105	+}
	106	+
	107	+transaction::~transaction(void)
	108	+{
	109	+ assert(!_txn);
	110	+}
	111	+
	112	+int
	113	+transaction::error(void) const
	114	+{
	115	+ return _error;
	116	+}
	117	+
	118	+string
	119	+transaction::strerror(void) const
	120	+{
	121	+ return db_strerror(_error);
	122	+}
	123	+
	124	+template<>
	125	+void
	126	+marshalling_buffer::append<imstring>(imstring const &o) {
	127	+ append<size_t>(o.size());
	128	+ append_bytes(o.data(), o.size());
	129	+}
	130	+
	131	+template<>
	132	+bool
	133	+marshalling_buffer::extract<imstring>(imstring &s)
	134	+{
	135	+size_t sz = 0;
	136	+ if (!extract<size_t>(sz))
	137	+ return false;
	138	+ if (_size + sz > _bufsz)
	139	+ return false;
	140	+ s.reserve(sz);
	141	+ memcpy(s.data(), _buf + _size + sz, sz);
	142	+ _size += sz;
	143	+ return true;
	144	+}
	145	+
	146	+
	147	+} // namespace db
Property changes on: trunk/willow/src/willow/dbwrap.cc
___________________________________________________________________
Added: svn:keywords
1	148	+ Id Revision