r108302 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r108301‎ | r108302 | r108303 >
Date:01:53, 7 January 2012
Author:raindrift
Status:deferred
Tags:
Comment:
Adding concurrency checking backend class
Note: This is not done! That's why it's in a branch!
Modified paths:
  • /branches/concurrency/includes/AutoLoader.php (modified) (history)
  • /branches/concurrency/includes/ConcurrencyCheck.php (added) (history)
  • /branches/concurrency/includes/installer/MysqlUpdater.php (modified) (history)
  • /branches/concurrency/maintenance/archives/patch-concurrencycheck.sql (added) (history)
  • /branches/concurrency/maintenance/tables.sql (modified) (history)
  • /branches/concurrency/tests/phpunit/includes/ConcurrencyCheckTest.php (added) (history)

Diff [purge]

Index: branches/concurrency/maintenance/archives/patch-concurrencycheck.sql
@@ -0,0 +1,25 @@
 2+--
 3+-- Store atomic locking information for web resources, to permit
 4+-- UI that warns users when concurrently editing things that aren't
 5+-- concurrently editable.
 6+--
 7+CREATE TABLE /*_*/concurrencycheck (
 8+ -- a string describing the resource or application being checked out.
 9+ cc_resource_type varchar(255) NOT NULL,
 10+
 11+ -- the (numeric) ID of the record that's being checked out.
 12+ cc_record int unsigned NOT NULL,
 13+
 14+ -- the user who has control of the resource
 15+ cc_user int unsigned NOT NULL,
 16+
 17+ -- the date/time on which this record expires
 18+ cc_expiration varbinary(14) not null
 19+
 20+) /*$wgDBTableOptions*/;
 21+-- composite pk.
 22+CREATE UNIQUE INDEX /*i*/cc_resource_record ON /*_*/concurrencycheck (cc_resource_type, cc_record);
 23+-- sometimes there's a delete based on userid.
 24+CREATE INDEX /*i*/cc_user ON /*_*/concurrencycheck (cc_user);
 25+-- sometimes there's a delete based on expiration
 26+CREATE INDEX /*i*/cc_expiration ON /*_*/concurrencycheck (cc_expiration);
Property changes on: branches/concurrency/maintenance/archives/patch-concurrencycheck.sql
___________________________________________________________________
Added: svn:eol-style
127 + native
Index: branches/concurrency/maintenance/tables.sql
@@ -1483,4 +1483,31 @@
14841484 -- Should cover *most* configuration - strings, ints, bools, etc.
14851485 CREATE INDEX /*i*/cf_name_value ON /*_*/config (cf_name,cf_value(255));
14861486
 1487+--
 1488+-- Store atomic locking information for web resources, to permit
 1489+-- UI that warns users when concurrently editing things that aren't
 1490+-- concurrently editable.
 1491+--
 1492+CREATE TABLE /*_*/concurrencycheck (
 1493+ -- a string describing the resource or application being checked out.
 1494+ cc_resource_type varchar(255) NOT NULL,
 1495+
 1496+ -- the (numeric) ID of the record that's being checked out.
 1497+ cc_record int unsigned NOT NULL,
 1498+
 1499+ -- the user who has control of the resource
 1500+ cc_user int unsigned NOT NULL,
 1501+
 1502+ -- the date/time on which this record expires
 1503+ cc_expiration varbinary(14) not null
 1504+
 1505+) /*$wgDBTableOptions*/;
 1506+-- composite pk.
 1507+CREATE UNIQUE INDEX /*i*/cc_resource_record ON /*_*/concurrencycheck (cc_resource_type, cc_record);
 1508+-- sometimes there's a delete based on userid.
 1509+CREATE INDEX /*i*/cc_user ON /*_*/concurrencycheck (cc_user);
 1510+-- sometimes there's a delete based on expiration
 1511+CREATE INDEX /*i*/cc_expiration ON /*_*/concurrencycheck (cc_expiration);
 1512+
 1513+
14871514 -- vim: sw=2 sts=2 et
Index: branches/concurrency/tests/phpunit/includes/ConcurrencyCheckTest.php
@@ -0,0 +1,93 @@
 2+<?php
 3+
 4+class ConcurrencyCheckTest extends MediaWikiTestCase {
 5+ /**
 6+ * @var Array of test users
 7+ */
 8+ public static $users;
 9+
 10+ // Prepare test environment
 11+
 12+ public function setUp() {
 13+ parent::setUp();
 14+
 15+ self::$users = array(
 16+ 'user1' => new ApiTestUser(
 17+ 'Concurrencychecktestuser1',
 18+ 'ConcurrencyCheck Test User 1',
 19+ 'concurrency_check_test_user_1@example.com',
 20+ array()
 21+ ),
 22+ 'user2' => new ApiTestUser(
 23+ 'Concurrencychecktestuser2',
 24+ 'ConcurrencyCheck Test User 2',
 25+ 'concurrency_check_test_user_2@example.com',
 26+ array()
 27+ ),
 28+ );
 29+ }
 30+
 31+ public function tearDown() {
 32+ parent::tearDown();
 33+
 34+ // perhaps clean up all the cruft left in the db
 35+ }
 36+
 37+ // Actual tests from here down
 38+
 39+ public function testCheckoutCheckin() {
 40+ $first = new ConcurrencyCheck( 'CCUnitTest', self::$users['user1']->user );
 41+ $second = new ConcurrencyCheck( 'CCUnitTest', self::$users['user2']->user );
 42+ $testKey = 1337;
 43+
 44+ // clean up after any previously failed tests
 45+ $first->checkin($testKey);
 46+ $second->checkin($testKey);
 47+
 48+ // tests
 49+ $this->assertTrue( $first->checkout($testKey), "Initial checkout" );
 50+ $this->assertFalse( $second->checkout($testKey), "Checkout of locked resource fails as different user" );
 51+ $this->assertTrue( $first->checkout($testKey), "Checkout of locked resource succeeds as original user" );
 52+ $this->assertFalse( $second->checkin($testKey), "Checkin of locked resource fails as different user" );
 53+ $this->assertTrue( $first->checkin($testKey), "Checkin of locked resource succeeds as original user" );
 54+ $second->setExpirationTime(-5);
 55+ $this->assertTrue( $second->checkout($testKey), "Checked-in resource is now available to second user" );
 56+ $second->setExpirationTime();
 57+ $this->assertTrue( $first->checkout($testKey), "Checkout of expired resource succeeds as first user");
 58+ $this->assertTrue( $second->checkout($testKey, true), "Checkout override" );
 59+ $this->assertFalse( $first->checkout($testKey), "Checkout of overriden resource fails as different user" );
 60+
 61+ // cleanup
 62+ $this->assertTrue( $second->checkin($testKey), "Checkin of record with changed ownership" );
 63+ }
 64+
 65+ public function testExpire() {
 66+ $cc = new ConcurrencyCheck( 'CCUnitTest', self::$users['user1']->user );
 67+ $cc->setExpirationTime(-1);
 68+ $cc->checkout( 1338 ); // these numbers are test record ids.
 69+ $cc->checkout( 1339 );
 70+ $cc->setExpirationTime();
 71+ $cc->checkout( 13310 );
 72+
 73+ // tests
 74+ $this->assertEquals( 2, $cc->expire(), "Resource expiration" );
 75+ $this->assertTrue( $cc->checkin( 13310 ), "Checkin succeeds after expiration" );
 76+ }
 77+
 78+ public function testStatus() {
 79+ $cc = new ConcurrencyCheck( 'CCUnitTest', self::$users['user1']->user );
 80+ $cc->checkout( 1337 );
 81+ $cc->checkout( 1338 );
 82+ $cc->setExpirationTime(-5);
 83+ $cc->checkout( 1339 );
 84+ $cc->setExpirationTime();
 85+
 86+ // tests
 87+ $output = $cc->status( array( 1337, 1338, 1339, 13310 ) );
 88+ $this->assertEquals( true, is_array( $output ), "Status returns values" );
 89+ $this->assertEquals( 4, count( $output ), "Output has the correct number of records" );
 90+ $this->assertEquals( 'valid', $output[1337]['status'], "Current checkouts are listed as valid");
 91+ $this->assertEquals( 'invalid', $output[1339]['status'], "Expired checkouts are invalid");
 92+ $this->assertEquals( 'invalid', $output[13310]['status'], "Missing checkouts are invalid");
 93+ }
 94+}
\ No newline at end of file
Property changes on: branches/concurrency/tests/phpunit/includes/ConcurrencyCheckTest.php
___________________________________________________________________
Added: svn:eol-style
195 + native
Index: branches/concurrency/includes/installer/MysqlUpdater.php
@@ -192,6 +192,7 @@
193193 array( 'modifyField', 'user', 'ug_group', 'patch-ug_group-length-increase.sql' ),
194194 array( 'addField', 'uploadstash', 'us_chunk_inx', 'patch-uploadstash_chunk.sql' ),
195195 array( 'addfield', 'job', 'job_timestamp', 'patch-jobs-add-timestamp.sql' ),
 196+ array( 'addTable', 'concurrencycheck', 'patch-concurrencycheck.sql'),
196197 );
197198 }
198199
Index: branches/concurrency/includes/AutoLoader.php
@@ -43,6 +43,7 @@
4444 'ChannelFeed' => 'includes/Feed.php',
4545 'Collation' => 'includes/Collation.php',
4646 'ConcatenatedGzipHistoryBlob' => 'includes/HistoryBlob.php',
 47+ 'ConcurrencyCheck' => 'includes/ConcurrencyCheck.php',
4748 'ConfEditor' => 'includes/ConfEditor.php',
4849 'ConfEditorParseError' => 'includes/ConfEditor.php',
4950 'ConfEditorToken' => 'includes/ConfEditor.php',
@@ -52,7 +53,6 @@
5354 'DeferredUpdates' => 'includes/DeferredUpdates.php',
5455 'DerivativeRequest' => 'includes/WebRequest.php',
5556 'DiffHistoryBlob' => 'includes/HistoryBlob.php',
56 -
5757 'DoubleReplacer' => 'includes/StringUtils.php',
5858 'DummyLinker' => 'includes/Linker.php',
5959 'Dump7ZipOutput' => 'includes/Export.php',
Index: branches/concurrency/includes/ConcurrencyCheck.php
@@ -0,0 +1,304 @@
 2+<?php
 3+
 4+/**
 5+ * Class for cooperative locking of web resources
 6+ *
 7+ * Each resource is identified by a combination of the "resource type" (the application, the type
 8+ * of content, etc), and the resource's primary key or some other unique numeric ID.
 9+ *
 10+ * Currently, a resource can only be checked out by a single user. Other attempts to check it out result
 11+ * in the checkout failing. In the future, an option for multiple simulataneous checkouts could be added
 12+ * without much trouble.
 13+ *
 14+ * This could be done with named locks, except then it would be impossible to build a list of all the
 15+ * resources currently checked out for a given application. There's no good way to construct a query
 16+ * that answers the question, "What locks do you have starting with [foo]" This could be done really well
 17+ * with a concurrent, reliable, distributed key/value store, but we don't have one of those right now.
 18+ *
 19+ * @author Ian Baker <ian@wikimedia.org>
 20+ */
 21+class ConcurrencyCheck {
 22+
 23+
 24+ // TODO: docblock
 25+ public function __construct( $resourceType, $user, $expirationTime = null ) {
 26+
 27+ // All database calls are to the master, since the whole point of this class is maintaining
 28+ // concurrency. Most reads should come from cache anyway.
 29+ $this->dbw = wfGetDb( DB_MASTER );
 30+
 31+ $this->user = $user;
 32+ $this->resourceType = $resourceType;
 33+ $this->setExpirationTime( $expirationTime );
 34+ }
 35+
 36+ /**
 37+ * Check out a resource. This establishes an atomically generated, cooperative lock
 38+ * on a key. The lock is tied to the current user.
 39+ *
 40+ * @var $record Integer containing the record id to check out
 41+ * @var $override Boolean (optional) describing whether to override an existing checkout
 42+ * @return boolean
 43+ */
 44+ public function checkout( $record, $override = null ) {
 45+ $this->validateId( $record );
 46+ $dbw = $this->dbw;
 47+ $userId = $this->user->getId();
 48+
 49+ // attempt an insert, check success (this is atomic)
 50+ $insertError = null;
 51+ $res = $dbw->insert(
 52+ 'concurrencycheck',
 53+ array(
 54+ 'cc_resource_type' => $this->resourceType,
 55+ 'cc_record' => $record,
 56+ 'cc_user' => $userId,
 57+ 'cc_expiration' => time() + $this->expirationTime,
 58+ ),
 59+ __METHOD__,
 60+ array('IGNORE')
 61+ );
 62+
 63+ // if the insert succeeded, checkout is done.
 64+ if( $dbw->affectedRows() === 1 ) {
 65+ //TODO: delete cache key
 66+ return true;
 67+ }
 68+
 69+ $dbw->begin();
 70+ $row = $dbw->selectRow(
 71+ 'concurrencycheck',
 72+ array( 'cc_user', 'cc_expiration' ),
 73+ array(
 74+ 'cc_resource_type' => $this->resourceType,
 75+ 'cc_record' => $record,
 76+ ),
 77+ __METHOD__,
 78+ array()
 79+ );
 80+
 81+ // not checked out by current user, checkout is unexpired, override is unset
 82+ if( ! ( $override || $row->cc_user == $userId || $row->cc_expiration <= time() ) ) {
 83+ $dbw->rollback();
 84+ return false;
 85+ }
 86+
 87+ // execute a replace
 88+ $res = $dbw->replace(
 89+ 'concurrencycheck',
 90+ array( array('cc_resource_type', 'cc_record') ),
 91+ array(
 92+ 'cc_resource_type' => $this->resourceType,
 93+ 'cc_record' => $record,
 94+ 'cc_user' => $userId,
 95+ 'cc_expiration' => time() + $this->expirationTime,
 96+ ),
 97+ __METHOD__
 98+ );
 99+
 100+ // TODO cache the result.
 101+
 102+ $dbw->commit();
 103+ return true;
 104+
 105+ // if insert succeeds, delete the cache key. don't make a new one since they have to be created atomically.
 106+ //
 107+ // if insert fails:
 108+ // begin transaction
 109+ // select where key=key and expiration > now()
 110+ // if row is missing or user matches:
 111+ // execute a replace()
 112+ // overwrite the cache key (might as well, since this is inside a transaction)
 113+ // commit
 114+ // if select returned an unexpired row owned by someone else, return failure.
 115+
 116+ // optional: check to see if the current user already has the resource checked out, and if so,
 117+ // return that checkout information instead. (does anyone want that?)
 118+ }
 119+
 120+ /**
 121+ * Check in a resource. Only works if the resource is checked out by the current user.
 122+ *
 123+ * @var $record Integer containing the record id to checkin
 124+ * @return Boolean
 125+ */
 126+ public function checkin( $record ) {
 127+ $this->validateId( $record );
 128+ $dbw = $this->dbw;
 129+ $userId = $this->user->getId();
 130+
 131+ $dbw->delete(
 132+ 'concurrencycheck',
 133+ array(
 134+ 'cc_resource_type' => $this->resourceType,
 135+ 'cc_record' => $record,
 136+ 'cc_user' => $userId, // only the owner can perform a checkin
 137+ ),
 138+ __METHOD__,
 139+ array()
 140+ );
 141+
 142+ // check row count (this is atomic, select would not be)
 143+ if( $dbw->affectedRows() === 1 ) {
 144+ // TODO: remove record from cache
 145+ return true;
 146+ }
 147+
 148+ return false;
 149+
 150+ // delete the row, specifying the username in the where clause (keeps users from checking in stuff that's not theirs).
 151+ // if a row was deleted:
 152+ // remove the record from memcache. (removing cache key doesn't require atomicity)
 153+ // return true
 154+ // else
 155+ // return false
 156+
 157+ }
 158+
 159+ /**
 160+ * Remove all expired checkouts.
 161+ *
 162+ * @return Integer describing the number of records expired.
 163+ */
 164+ public function expire() {
 165+ $dbw = $this->dbw;
 166+
 167+ $now = time();
 168+
 169+ // get the rows to remove from cache.
 170+ $res = $dbw->select(
 171+ 'concurrencycheck',
 172+ array( '*' ),
 173+ array(
 174+ 'cc_expiration <= ' . $now,
 175+ ),
 176+ __METHOD__,
 177+ array()
 178+ );
 179+
 180+ // remove the rows from the db
 181+ $dbw->delete(
 182+ 'concurrencycheck',
 183+ array(
 184+ 'cc_expiration <= ' . $now,
 185+ ),
 186+ __METHOD__,
 187+ array()
 188+ );
 189+
 190+ // TODO: fetch the rows here, remove them from cache.
 191+
 192+ // return the number of rows removed.
 193+ return $dbw->affectedRows();
 194+
 195+ // grab a unixtime.
 196+ // select all rows where expiration < time
 197+ // delete all rows where expiration < time
 198+ // remove selected rows from memcache
 199+ //
 200+ // previous idea, probably wrong:
 201+ // select all expired rows.
 202+ // foreach( expired )
 203+ // delete row where id=id and expiration < now() (accounts for updates)
 204+ // if delete succeeded, remove cache key (txn not required, since removing cache key doesn't require atomicity)
 205+ // (sadly, this must be many deletes to coordinate removal from memcache)
 206+ // (is it necessary to remove expired cache entries?)
 207+ }
 208+
 209+ public function status( $keys ) {
 210+ $dbw = $this->dbw;
 211+
 212+ // validate keys.
 213+ foreach( $keys as $key ) {
 214+ $this->validateId( $key );
 215+ }
 216+
 217+ $checkouts = array();
 218+
 219+ // TODO: check cache before selecting
 220+
 221+ // check for each of $keys in cache (also check expiration)
 222+ // build a list of the missing ones.
 223+ // run the below select with that list.
 224+ // when finished, re-add any missing keys with the 'invalid' status.
 225+
 226+ $this->expire();
 227+
 228+ $dbw->begin();
 229+ $res = $dbw->select(
 230+ 'concurrencycheck',
 231+ array( '*' ),
 232+ array(
 233+ 'cc_resource_type' => $this->resourceType,
 234+ 'cc_record IN (' . implode( ',', $keys ) . ')',
 235+ 'cc_expiration > unix_timestamp(now())'
 236+ ),
 237+ __METHOD__,
 238+ array()
 239+ );
 240+
 241+ while( $res && $record = $res->fetchRow() ) {
 242+ $record['status'] = 'valid';
 243+ # cache the row.
 244+ $checkouts[ $record['cc_record'] ] = $record;
 245+ }
 246+
 247+ // if a key was passed in but has no (unexpired) checkout, include it in the
 248+ // result set to make things easier and more consistent on the client-side.
 249+ foreach( $keys as $key ) {
 250+ if( ! array_key_exists( $key, $checkouts ) ) {
 251+ $checkouts[$key]['status'] = 'invalid';
 252+ }
 253+ }
 254+
 255+ return $checkouts;
 256+
 257+ // fetch keys from cache or db (keys are an array)
 258+ //
 259+ // for all unexpired keys present in cache, store cached return value for returning later.
 260+ //
 261+ // if some keys remain (missing from cache or expired):
 262+ // execute expire() to make sure db records are cleared
 263+ // for all remaining keys:
 264+ // begin transaction
 265+ // select rows where key in (keys) and expiration > now()
 266+ // overwrite any memcache entry
 267+ // commit
 268+ // return values that were added to cache, plus values pulled from cache
 269+ }
 270+
 271+ public function list() {
 272+
 273+ }
 274+
 275+ public function setUser ( $user ) {
 276+ $this->user = $user;
 277+ }
 278+
 279+ public function setExpirationTime ( $expirationTime = null ) {
 280+ // check to make sure the time is digits only, so it can be used in queries
 281+ // negative number are allowed, though mostly only used for testing
 282+ // TODO: define maximum and minimum times in configuration, to prevent DoS
 283+ if( $expirationTime && preg_match('/^[\d-]+$/', $expirationTime) ) {
 284+ $this->expirationTime = $expirationTime; // the amount of time before a checkout expires.
 285+ } else {
 286+ $this->expirationTime = 60 * 15; // 15 mins. TODO: make a configurable default for this.
 287+ }
 288+ }
 289+
 290+ /**
 291+ * Check to make sure a record ID is numeric, throw an exception if not.
 292+ *
 293+ * @var $record Integer
 294+ * @throws ConcurrencyCheckBadRecordIdException
 295+ * @return boolean
 296+ */
 297+ private static function validateId ( $record ) {
 298+ if( ! preg_match('/^\d+$/', $record) ) {
 299+ throw new ConcurrencyCheckBadRecordIdException( 'Record ID ' . $record . ' must be a positive integer' );
 300+ }
 301+ return true;
 302+ }
 303+}
 304+
 305+class ConcurrencyCheckBadRecordIdException extends MWException {};
Property changes on: branches/concurrency/includes/ConcurrencyCheck.php
___________________________________________________________________
Added: svn:eol-style
1306 + native

Status & tagging log