Index: branches/concurrency/maintenance/archives/patch-concurrencycheck.sql |
— | — | @@ -0,0 +1,25 @@ |
| 2 | +-- |
| 3 | +-- Store atomic locking information for web resources, to permit |
| 4 | +-- UI that warns users when concurrently editing things that aren't |
| 5 | +-- concurrently editable. |
| 6 | +-- |
| 7 | +CREATE TABLE /*_*/concurrencycheck ( |
| 8 | + -- a string describing the resource or application being checked out. |
| 9 | + cc_resource_type varchar(255) NOT NULL, |
| 10 | + |
| 11 | + -- the (numeric) ID of the record that's being checked out. |
| 12 | + cc_record int unsigned NOT NULL, |
| 13 | + |
| 14 | + -- the user who has control of the resource |
| 15 | + cc_user int unsigned NOT NULL, |
| 16 | + |
| 17 | + -- the date/time on which this record expires |
| 18 | + cc_expiration varbinary(14) not null |
| 19 | + |
| 20 | +) /*$wgDBTableOptions*/; |
| 21 | +-- composite pk. |
| 22 | +CREATE UNIQUE INDEX /*i*/cc_resource_record ON /*_*/concurrencycheck (cc_resource_type, cc_record); |
| 23 | +-- sometimes there's a delete based on userid. |
| 24 | +CREATE INDEX /*i*/cc_user ON /*_*/concurrencycheck (cc_user); |
| 25 | +-- sometimes there's a delete based on expiration |
| 26 | +CREATE INDEX /*i*/cc_expiration ON /*_*/concurrencycheck (cc_expiration); |
Property changes on: branches/concurrency/maintenance/archives/patch-concurrencycheck.sql |
___________________________________________________________________ |
Added: svn:eol-style |
1 | 27 | + native |
Index: branches/concurrency/maintenance/tables.sql |
— | — | @@ -1483,4 +1483,31 @@ |
1484 | 1484 | -- Should cover *most* configuration - strings, ints, bools, etc. |
1485 | 1485 | CREATE INDEX /*i*/cf_name_value ON /*_*/config (cf_name,cf_value(255)); |
1486 | 1486 | |
| 1487 | +-- |
| 1488 | +-- Store atomic locking information for web resources, to permit |
| 1489 | +-- UI that warns users when concurrently editing things that aren't |
| 1490 | +-- concurrently editable. |
| 1491 | +-- |
| 1492 | +CREATE TABLE /*_*/concurrencycheck ( |
| 1493 | + -- a string describing the resource or application being checked out. |
| 1494 | + cc_resource_type varchar(255) NOT NULL, |
| 1495 | + |
| 1496 | + -- the (numeric) ID of the record that's being checked out. |
| 1497 | + cc_record int unsigned NOT NULL, |
| 1498 | + |
| 1499 | + -- the user who has control of the resource |
| 1500 | + cc_user int unsigned NOT NULL, |
| 1501 | + |
| 1502 | + -- the date/time on which this record expires |
| 1503 | + cc_expiration varbinary(14) not null |
| 1504 | + |
| 1505 | +) /*$wgDBTableOptions*/; |
| 1506 | +-- composite pk. |
| 1507 | +CREATE UNIQUE INDEX /*i*/cc_resource_record ON /*_*/concurrencycheck (cc_resource_type, cc_record); |
| 1508 | +-- sometimes there's a delete based on userid. |
| 1509 | +CREATE INDEX /*i*/cc_user ON /*_*/concurrencycheck (cc_user); |
| 1510 | +-- sometimes there's a delete based on expiration |
| 1511 | +CREATE INDEX /*i*/cc_expiration ON /*_*/concurrencycheck (cc_expiration); |
| 1512 | + |
| 1513 | + |
1487 | 1514 | -- vim: sw=2 sts=2 et |
Index: branches/concurrency/tests/phpunit/includes/ConcurrencyCheckTest.php |
— | — | @@ -0,0 +1,93 @@ |
| 2 | +<?php |
| 3 | + |
| 4 | +class ConcurrencyCheckTest extends MediaWikiTestCase { |
| 5 | + /** |
| 6 | + * @var Array of test users |
| 7 | + */ |
| 8 | + public static $users; |
| 9 | + |
| 10 | + // Prepare test environment |
| 11 | + |
| 12 | + public function setUp() { |
| 13 | + parent::setUp(); |
| 14 | + |
| 15 | + self::$users = array( |
| 16 | + 'user1' => new ApiTestUser( |
| 17 | + 'Concurrencychecktestuser1', |
| 18 | + 'ConcurrencyCheck Test User 1', |
| 19 | + 'concurrency_check_test_user_1@example.com', |
| 20 | + array() |
| 21 | + ), |
| 22 | + 'user2' => new ApiTestUser( |
| 23 | + 'Concurrencychecktestuser2', |
| 24 | + 'ConcurrencyCheck Test User 2', |
| 25 | + 'concurrency_check_test_user_2@example.com', |
| 26 | + array() |
| 27 | + ), |
| 28 | + ); |
| 29 | + } |
| 30 | + |
| 31 | + public function tearDown() { |
| 32 | + parent::tearDown(); |
| 33 | + |
| 34 | + // perhaps clean up all the cruft left in the db |
| 35 | + } |
| 36 | + |
| 37 | + // Actual tests from here down |
| 38 | + |
| 39 | + public function testCheckoutCheckin() { |
| 40 | + $first = new ConcurrencyCheck( 'CCUnitTest', self::$users['user1']->user ); |
| 41 | + $second = new ConcurrencyCheck( 'CCUnitTest', self::$users['user2']->user ); |
| 42 | + $testKey = 1337; |
| 43 | + |
| 44 | + // clean up after any previously failed tests |
| 45 | + $first->checkin($testKey); |
| 46 | + $second->checkin($testKey); |
| 47 | + |
| 48 | + // tests |
| 49 | + $this->assertTrue( $first->checkout($testKey), "Initial checkout" ); |
| 50 | + $this->assertFalse( $second->checkout($testKey), "Checkout of locked resource fails as different user" ); |
| 51 | + $this->assertTrue( $first->checkout($testKey), "Checkout of locked resource succeeds as original user" ); |
| 52 | + $this->assertFalse( $second->checkin($testKey), "Checkin of locked resource fails as different user" ); |
| 53 | + $this->assertTrue( $first->checkin($testKey), "Checkin of locked resource succeeds as original user" ); |
| 54 | + $second->setExpirationTime(-5); |
| 55 | + $this->assertTrue( $second->checkout($testKey), "Checked-in resource is now available to second user" ); |
| 56 | + $second->setExpirationTime(); |
| 57 | + $this->assertTrue( $first->checkout($testKey), "Checkout of expired resource succeeds as first user"); |
| 58 | + $this->assertTrue( $second->checkout($testKey, true), "Checkout override" ); |
| 59 | + $this->assertFalse( $first->checkout($testKey), "Checkout of overriden resource fails as different user" ); |
| 60 | + |
| 61 | + // cleanup |
| 62 | + $this->assertTrue( $second->checkin($testKey), "Checkin of record with changed ownership" ); |
| 63 | + } |
| 64 | + |
| 65 | + public function testExpire() { |
| 66 | + $cc = new ConcurrencyCheck( 'CCUnitTest', self::$users['user1']->user ); |
| 67 | + $cc->setExpirationTime(-1); |
| 68 | + $cc->checkout( 1338 ); // these numbers are test record ids. |
| 69 | + $cc->checkout( 1339 ); |
| 70 | + $cc->setExpirationTime(); |
| 71 | + $cc->checkout( 13310 ); |
| 72 | + |
| 73 | + // tests |
| 74 | + $this->assertEquals( 2, $cc->expire(), "Resource expiration" ); |
| 75 | + $this->assertTrue( $cc->checkin( 13310 ), "Checkin succeeds after expiration" ); |
| 76 | + } |
| 77 | + |
| 78 | + public function testStatus() { |
| 79 | + $cc = new ConcurrencyCheck( 'CCUnitTest', self::$users['user1']->user ); |
| 80 | + $cc->checkout( 1337 ); |
| 81 | + $cc->checkout( 1338 ); |
| 82 | + $cc->setExpirationTime(-5); |
| 83 | + $cc->checkout( 1339 ); |
| 84 | + $cc->setExpirationTime(); |
| 85 | + |
| 86 | + // tests |
| 87 | + $output = $cc->status( array( 1337, 1338, 1339, 13310 ) ); |
| 88 | + $this->assertEquals( true, is_array( $output ), "Status returns values" ); |
| 89 | + $this->assertEquals( 4, count( $output ), "Output has the correct number of records" ); |
| 90 | + $this->assertEquals( 'valid', $output[1337]['status'], "Current checkouts are listed as valid"); |
| 91 | + $this->assertEquals( 'invalid', $output[1339]['status'], "Expired checkouts are invalid"); |
| 92 | + $this->assertEquals( 'invalid', $output[13310]['status'], "Missing checkouts are invalid"); |
| 93 | + } |
| 94 | +} |
\ No newline at end of file |
Property changes on: branches/concurrency/tests/phpunit/includes/ConcurrencyCheckTest.php |
___________________________________________________________________ |
Added: svn:eol-style |
1 | 95 | + native |
Index: branches/concurrency/includes/installer/MysqlUpdater.php |
— | — | @@ -192,6 +192,7 @@ |
193 | 193 | array( 'modifyField', 'user', 'ug_group', 'patch-ug_group-length-increase.sql' ), |
194 | 194 | array( 'addField', 'uploadstash', 'us_chunk_inx', 'patch-uploadstash_chunk.sql' ), |
195 | 195 | array( 'addfield', 'job', 'job_timestamp', 'patch-jobs-add-timestamp.sql' ), |
| 196 | + array( 'addTable', 'concurrencycheck', 'patch-concurrencycheck.sql'), |
196 | 197 | ); |
197 | 198 | } |
198 | 199 | |
Index: branches/concurrency/includes/AutoLoader.php |
— | — | @@ -43,6 +43,7 @@ |
44 | 44 | 'ChannelFeed' => 'includes/Feed.php', |
45 | 45 | 'Collation' => 'includes/Collation.php', |
46 | 46 | 'ConcatenatedGzipHistoryBlob' => 'includes/HistoryBlob.php', |
| 47 | + 'ConcurrencyCheck' => 'includes/ConcurrencyCheck.php', |
47 | 48 | 'ConfEditor' => 'includes/ConfEditor.php', |
48 | 49 | 'ConfEditorParseError' => 'includes/ConfEditor.php', |
49 | 50 | 'ConfEditorToken' => 'includes/ConfEditor.php', |
— | — | @@ -52,7 +53,6 @@ |
53 | 54 | 'DeferredUpdates' => 'includes/DeferredUpdates.php', |
54 | 55 | 'DerivativeRequest' => 'includes/WebRequest.php', |
55 | 56 | 'DiffHistoryBlob' => 'includes/HistoryBlob.php', |
56 | | - |
57 | 57 | 'DoubleReplacer' => 'includes/StringUtils.php', |
58 | 58 | 'DummyLinker' => 'includes/Linker.php', |
59 | 59 | 'Dump7ZipOutput' => 'includes/Export.php', |
Index: branches/concurrency/includes/ConcurrencyCheck.php |
— | — | @@ -0,0 +1,304 @@ |
| 2 | +<?php |
| 3 | + |
| 4 | +/** |
| 5 | + * Class for cooperative locking of web resources |
| 6 | + * |
| 7 | + * Each resource is identified by a combination of the "resource type" (the application, the type |
| 8 | + * of content, etc), and the resource's primary key or some other unique numeric ID. |
| 9 | + * |
| 10 | + * Currently, a resource can only be checked out by a single user. Other attempts to check it out result |
| 11 | + * in the checkout failing. In the future, an option for multiple simulataneous checkouts could be added |
| 12 | + * without much trouble. |
| 13 | + * |
| 14 | + * This could be done with named locks, except then it would be impossible to build a list of all the |
| 15 | + * resources currently checked out for a given application. There's no good way to construct a query |
| 16 | + * that answers the question, "What locks do you have starting with [foo]" This could be done really well |
| 17 | + * with a concurrent, reliable, distributed key/value store, but we don't have one of those right now. |
| 18 | + * |
| 19 | + * @author Ian Baker <ian@wikimedia.org> |
| 20 | + */ |
| 21 | +class ConcurrencyCheck { |
| 22 | + |
| 23 | + |
| 24 | + // TODO: docblock |
| 25 | + public function __construct( $resourceType, $user, $expirationTime = null ) { |
| 26 | + |
| 27 | + // All database calls are to the master, since the whole point of this class is maintaining |
| 28 | + // concurrency. Most reads should come from cache anyway. |
| 29 | + $this->dbw = wfGetDb( DB_MASTER ); |
| 30 | + |
| 31 | + $this->user = $user; |
| 32 | + $this->resourceType = $resourceType; |
| 33 | + $this->setExpirationTime( $expirationTime ); |
| 34 | + } |
| 35 | + |
| 36 | + /** |
| 37 | + * Check out a resource. This establishes an atomically generated, cooperative lock |
| 38 | + * on a key. The lock is tied to the current user. |
| 39 | + * |
| 40 | + * @var $record Integer containing the record id to check out |
| 41 | + * @var $override Boolean (optional) describing whether to override an existing checkout |
| 42 | + * @return boolean |
| 43 | + */ |
| 44 | + public function checkout( $record, $override = null ) { |
| 45 | + $this->validateId( $record ); |
| 46 | + $dbw = $this->dbw; |
| 47 | + $userId = $this->user->getId(); |
| 48 | + |
| 49 | + // attempt an insert, check success (this is atomic) |
| 50 | + $insertError = null; |
| 51 | + $res = $dbw->insert( |
| 52 | + 'concurrencycheck', |
| 53 | + array( |
| 54 | + 'cc_resource_type' => $this->resourceType, |
| 55 | + 'cc_record' => $record, |
| 56 | + 'cc_user' => $userId, |
| 57 | + 'cc_expiration' => time() + $this->expirationTime, |
| 58 | + ), |
| 59 | + __METHOD__, |
| 60 | + array('IGNORE') |
| 61 | + ); |
| 62 | + |
| 63 | + // if the insert succeeded, checkout is done. |
| 64 | + if( $dbw->affectedRows() === 1 ) { |
| 65 | + //TODO: delete cache key |
| 66 | + return true; |
| 67 | + } |
| 68 | + |
| 69 | + $dbw->begin(); |
| 70 | + $row = $dbw->selectRow( |
| 71 | + 'concurrencycheck', |
| 72 | + array( 'cc_user', 'cc_expiration' ), |
| 73 | + array( |
| 74 | + 'cc_resource_type' => $this->resourceType, |
| 75 | + 'cc_record' => $record, |
| 76 | + ), |
| 77 | + __METHOD__, |
| 78 | + array() |
| 79 | + ); |
| 80 | + |
| 81 | + // not checked out by current user, checkout is unexpired, override is unset |
| 82 | + if( ! ( $override || $row->cc_user == $userId || $row->cc_expiration <= time() ) ) { |
| 83 | + $dbw->rollback(); |
| 84 | + return false; |
| 85 | + } |
| 86 | + |
| 87 | + // execute a replace |
| 88 | + $res = $dbw->replace( |
| 89 | + 'concurrencycheck', |
| 90 | + array( array('cc_resource_type', 'cc_record') ), |
| 91 | + array( |
| 92 | + 'cc_resource_type' => $this->resourceType, |
| 93 | + 'cc_record' => $record, |
| 94 | + 'cc_user' => $userId, |
| 95 | + 'cc_expiration' => time() + $this->expirationTime, |
| 96 | + ), |
| 97 | + __METHOD__ |
| 98 | + ); |
| 99 | + |
| 100 | + // TODO cache the result. |
| 101 | + |
| 102 | + $dbw->commit(); |
| 103 | + return true; |
| 104 | + |
| 105 | + // if insert succeeds, delete the cache key. don't make a new one since they have to be created atomically. |
| 106 | + // |
| 107 | + // if insert fails: |
| 108 | + // begin transaction |
| 109 | + // select where key=key and expiration > now() |
| 110 | + // if row is missing or user matches: |
| 111 | + // execute a replace() |
| 112 | + // overwrite the cache key (might as well, since this is inside a transaction) |
| 113 | + // commit |
| 114 | + // if select returned an unexpired row owned by someone else, return failure. |
| 115 | + |
| 116 | + // optional: check to see if the current user already has the resource checked out, and if so, |
| 117 | + // return that checkout information instead. (does anyone want that?) |
| 118 | + } |
| 119 | + |
| 120 | + /** |
| 121 | + * Check in a resource. Only works if the resource is checked out by the current user. |
| 122 | + * |
| 123 | + * @var $record Integer containing the record id to checkin |
| 124 | + * @return Boolean |
| 125 | + */ |
| 126 | + public function checkin( $record ) { |
| 127 | + $this->validateId( $record ); |
| 128 | + $dbw = $this->dbw; |
| 129 | + $userId = $this->user->getId(); |
| 130 | + |
| 131 | + $dbw->delete( |
| 132 | + 'concurrencycheck', |
| 133 | + array( |
| 134 | + 'cc_resource_type' => $this->resourceType, |
| 135 | + 'cc_record' => $record, |
| 136 | + 'cc_user' => $userId, // only the owner can perform a checkin |
| 137 | + ), |
| 138 | + __METHOD__, |
| 139 | + array() |
| 140 | + ); |
| 141 | + |
| 142 | + // check row count (this is atomic, select would not be) |
| 143 | + if( $dbw->affectedRows() === 1 ) { |
| 144 | + // TODO: remove record from cache |
| 145 | + return true; |
| 146 | + } |
| 147 | + |
| 148 | + return false; |
| 149 | + |
| 150 | + // delete the row, specifying the username in the where clause (keeps users from checking in stuff that's not theirs). |
| 151 | + // if a row was deleted: |
| 152 | + // remove the record from memcache. (removing cache key doesn't require atomicity) |
| 153 | + // return true |
| 154 | + // else |
| 155 | + // return false |
| 156 | + |
| 157 | + } |
| 158 | + |
| 159 | + /** |
| 160 | + * Remove all expired checkouts. |
| 161 | + * |
| 162 | + * @return Integer describing the number of records expired. |
| 163 | + */ |
| 164 | + public function expire() { |
| 165 | + $dbw = $this->dbw; |
| 166 | + |
| 167 | + $now = time(); |
| 168 | + |
| 169 | + // get the rows to remove from cache. |
| 170 | + $res = $dbw->select( |
| 171 | + 'concurrencycheck', |
| 172 | + array( '*' ), |
| 173 | + array( |
| 174 | + 'cc_expiration <= ' . $now, |
| 175 | + ), |
| 176 | + __METHOD__, |
| 177 | + array() |
| 178 | + ); |
| 179 | + |
| 180 | + // remove the rows from the db |
| 181 | + $dbw->delete( |
| 182 | + 'concurrencycheck', |
| 183 | + array( |
| 184 | + 'cc_expiration <= ' . $now, |
| 185 | + ), |
| 186 | + __METHOD__, |
| 187 | + array() |
| 188 | + ); |
| 189 | + |
| 190 | + // TODO: fetch the rows here, remove them from cache. |
| 191 | + |
| 192 | + // return the number of rows removed. |
| 193 | + return $dbw->affectedRows(); |
| 194 | + |
| 195 | + // grab a unixtime. |
| 196 | + // select all rows where expiration < time |
| 197 | + // delete all rows where expiration < time |
| 198 | + // remove selected rows from memcache |
| 199 | + // |
| 200 | + // previous idea, probably wrong: |
| 201 | + // select all expired rows. |
| 202 | + // foreach( expired ) |
| 203 | + // delete row where id=id and expiration < now() (accounts for updates) |
| 204 | + // if delete succeeded, remove cache key (txn not required, since removing cache key doesn't require atomicity) |
| 205 | + // (sadly, this must be many deletes to coordinate removal from memcache) |
| 206 | + // (is it necessary to remove expired cache entries?) |
| 207 | + } |
| 208 | + |
| 209 | + public function status( $keys ) { |
| 210 | + $dbw = $this->dbw; |
| 211 | + |
| 212 | + // validate keys. |
| 213 | + foreach( $keys as $key ) { |
| 214 | + $this->validateId( $key ); |
| 215 | + } |
| 216 | + |
| 217 | + $checkouts = array(); |
| 218 | + |
| 219 | + // TODO: check cache before selecting |
| 220 | + |
| 221 | + // check for each of $keys in cache (also check expiration) |
| 222 | + // build a list of the missing ones. |
| 223 | + // run the below select with that list. |
| 224 | + // when finished, re-add any missing keys with the 'invalid' status. |
| 225 | + |
| 226 | + $this->expire(); |
| 227 | + |
| 228 | + $dbw->begin(); |
| 229 | + $res = $dbw->select( |
| 230 | + 'concurrencycheck', |
| 231 | + array( '*' ), |
| 232 | + array( |
| 233 | + 'cc_resource_type' => $this->resourceType, |
| 234 | + 'cc_record IN (' . implode( ',', $keys ) . ')', |
| 235 | + 'cc_expiration > unix_timestamp(now())' |
| 236 | + ), |
| 237 | + __METHOD__, |
| 238 | + array() |
| 239 | + ); |
| 240 | + |
| 241 | + while( $res && $record = $res->fetchRow() ) { |
| 242 | + $record['status'] = 'valid'; |
| 243 | + # cache the row. |
| 244 | + $checkouts[ $record['cc_record'] ] = $record; |
| 245 | + } |
| 246 | + |
| 247 | + // if a key was passed in but has no (unexpired) checkout, include it in the |
| 248 | + // result set to make things easier and more consistent on the client-side. |
| 249 | + foreach( $keys as $key ) { |
| 250 | + if( ! array_key_exists( $key, $checkouts ) ) { |
| 251 | + $checkouts[$key]['status'] = 'invalid'; |
| 252 | + } |
| 253 | + } |
| 254 | + |
| 255 | + return $checkouts; |
| 256 | + |
| 257 | + // fetch keys from cache or db (keys are an array) |
| 258 | + // |
| 259 | + // for all unexpired keys present in cache, store cached return value for returning later. |
| 260 | + // |
| 261 | + // if some keys remain (missing from cache or expired): |
| 262 | + // execute expire() to make sure db records are cleared |
| 263 | + // for all remaining keys: |
| 264 | + // begin transaction |
| 265 | + // select rows where key in (keys) and expiration > now() |
| 266 | + // overwrite any memcache entry |
| 267 | + // commit |
| 268 | + // return values that were added to cache, plus values pulled from cache |
| 269 | + } |
| 270 | + |
| 271 | + public function list() { |
| 272 | + |
| 273 | + } |
| 274 | + |
| 275 | + public function setUser ( $user ) { |
| 276 | + $this->user = $user; |
| 277 | + } |
| 278 | + |
| 279 | + public function setExpirationTime ( $expirationTime = null ) { |
| 280 | + // check to make sure the time is digits only, so it can be used in queries |
| 281 | + // negative number are allowed, though mostly only used for testing |
| 282 | + // TODO: define maximum and minimum times in configuration, to prevent DoS |
| 283 | + if( $expirationTime && preg_match('/^[\d-]+$/', $expirationTime) ) { |
| 284 | + $this->expirationTime = $expirationTime; // the amount of time before a checkout expires. |
| 285 | + } else { |
| 286 | + $this->expirationTime = 60 * 15; // 15 mins. TODO: make a configurable default for this. |
| 287 | + } |
| 288 | + } |
| 289 | + |
| 290 | + /** |
| 291 | + * Check to make sure a record ID is numeric, throw an exception if not. |
| 292 | + * |
| 293 | + * @var $record Integer |
| 294 | + * @throws ConcurrencyCheckBadRecordIdException |
| 295 | + * @return boolean |
| 296 | + */ |
| 297 | + private static function validateId ( $record ) { |
| 298 | + if( ! preg_match('/^\d+$/', $record) ) { |
| 299 | + throw new ConcurrencyCheckBadRecordIdException( 'Record ID ' . $record . ' must be a positive integer' ); |
| 300 | + } |
| 301 | + return true; |
| 302 | + } |
| 303 | +} |
| 304 | + |
| 305 | +class ConcurrencyCheckBadRecordIdException extends MWException {}; |
Property changes on: branches/concurrency/includes/ConcurrencyCheck.php |
___________________________________________________________________ |
Added: svn:eol-style |
1 | 306 | + native |