| Index: branches/FileBackend/phase3/maintenance/locking/file_locks.sql |
| — | — | @@ -1,10 +1,11 @@ |
| 2 | 2 | -- Table to handle resource locking (EX) with row-level locking |
| 3 | | -CREATE TABLE /*_*/file_locks_exclusive ( |
| | 3 | +CREATE TABLE /*_*/filelocks_exclusive ( |
| 4 | 4 | fle_key binary(40) NOT NULL default '' PRIMARY KEY |
| 5 | | -) ENGINE=InnoDB, DEFAULT CHARSET=binary; |
| | 5 | +) ENGINE=InnoDB, CHECKSUM=0; |
| 6 | 6 | |
| 7 | 7 | -- Table to handle resource locking (SH) with row-level locking |
| 8 | | -CREATE TABLE /*_*/file_locks_shared ( |
| 9 | | - fls_key binary(40) NOT NULL default '' |
| 10 | | -) ENGINE=InnoDB, DEFAULT CHARSET=binary; |
| 11 | | -CREATE INDEX /*i*/fls_key ON /*_*/file_locks_shared (fls_key); |
| | 8 | +CREATE TABLE /*_*/filelocks_shared ( |
| | 9 | + fls_key binary(40) NOT NULL default '', |
| | 10 | + fls_session integer unsigned NOT NULL default 0, |
| | 11 | + PRIMARY KEY (fls_key,fls_session) |
| | 12 | +) ENGINE=InnoDB, CHECKSUM=0; |
| Index: branches/FileBackend/phase3/includes/filerepo/backend/LockManager.php |
| — | — | @@ -76,7 +76,8 @@ |
| 77 | 77 | } |
| 78 | 78 | |
| 79 | 79 | /** |
| 80 | | - * Simple version of LockManager based on using FS lock files |
| | 80 | + * Simple version of LockManager based on using FS lock files. |
| | 81 | + * All locks are non-blocking, which avoids deadlocks. |
| 81 | 82 | * |
| 82 | 83 | * This should work fine for small sites running off one server. |
| 83 | 84 | * Do not use this with 'lockDir' set to an NFS mount unless the |
| — | — | @@ -272,74 +273,80 @@ |
| 273 | 274 | |
| 274 | 275 | /** |
| 275 | 276 | * Version of LockManager based on using DB table locks. |
| 276 | | - * This is meant for multi-wiki systems that may share share files. |
| | 277 | + * This is meant for multi-wiki systems that may share files. |
| | 278 | + * All locks are blocking, so it might be useful to set a small |
| | 279 | + * lock-wait timeout via server config to curtail deadlocks. |
| 277 | 280 | * |
| 278 | 281 | * All lock requests for a resource, identified by a hash string, will map |
| 279 | | - * to one bucket. Each bucket maps to one or several peer DBs, on different |
| 280 | | - * servers, each having the file_locks.sql tables (with row-level locking). |
| 281 | | - * This does not use GET_LOCK() per http://bugs.mysql.com/bug.php?id=1118. |
| | 282 | + * to one bucket. Each bucket maps to one or several peer DBs, each on their |
| | 283 | + * own server, all having the filelocks.sql tables (with row-level locking). |
| | 284 | + * A majority of peer DBs must agree for a lock to be acquired. |
| 282 | 285 | * |
| 283 | | - * A majority of peer DBs must agree for a lock to be acquired. |
| 284 | | - * As long as one peer DB is up, lock requests will not be blocked |
| 285 | | - * just because another peer DB cannot be contacted. A global status |
| 286 | | - * cache can be setup to track DBs that recently missed queries; such |
| 287 | | - * DBs will not be trusted for obtaining locks. |
| 288 | | - * |
| 289 | | - * For performance, a small lock-wait timeout should be set via server config. |
| 290 | | - * In innoDB, this can done via the innodb_lock_wait_timeout setting. |
| | 286 | + * Caching is used to avoid hitting servers that are down. |
| 291 | 287 | */ |
| 292 | 288 | class DBLockManager extends LockManager { |
| 293 | | - /** @var Array Map of bucket indexes to peer sets */ |
| | 289 | + /** @var Array Map of DB names to server config */ |
| | 290 | + protected $dbServers; // (DB name => server config array) |
| | 291 | + /** @var Array Map of bucket indexes to peer DB lists */ |
| 294 | 292 | protected $dbsByBucket; // (bucket index => (ldb1, ldb2, ...)) |
| 295 | 293 | /** @var BagOStuff */ |
| 296 | 294 | protected $statusCache; |
| 297 | 295 | |
| 298 | | - protected $webTimeout; // integer number of seconds |
| 299 | | - protected $cliTimeout; // integer number of seconds |
| | 296 | + protected $lockExpiry; // integer number of seconds |
| 300 | 297 | protected $safeDelay; // integer number of seconds |
| 301 | 298 | |
| | 299 | + protected $session = 0; // random integer |
| 302 | 300 | /** @var Array Map of (locked key => lock type => count) */ |
| 303 | 301 | protected $locksHeld = array(); |
| 304 | | - /** @var Array Map Lock-active database connections (DB name => Database) */ |
| 305 | | - protected $activeConns = array(); |
| | 302 | + /** @var Array Map Database connections (DB name => Database) */ |
| | 303 | + protected $conns = array(); |
| 306 | 304 | |
| 307 | 305 | /** |
| 308 | 306 | * Construct a new instance from configuration. |
| 309 | 307 | * $config paramaters include: |
| 310 | | - * 'dbsByBucket' : Array of 1-16 consecutive integer keys, starting from 0, with |
| 311 | | - * a list of DB names (peers) as values. Each list should have |
| 312 | | - * an odd number of items and each DB should have its own server. |
| 313 | | - * 'webTimeout' : Lock timeout (seconds) for non-CLI scripts. [optional] |
| | 308 | + * 'dbServers' : Associative array of DB names to server configuration. |
| | 309 | + * Configuration is an associative array that includes: |
| | 310 | + * 'host' - DB server name |
| | 311 | + * 'dbname' - DB name |
| | 312 | + * 'type' - DB type (mysql,postgres,...) |
| | 313 | + * 'user' - DB user |
| | 314 | + * 'password' - DB user password |
| | 315 | + * 'tablePrefix' - DB table prefix |
| | 316 | + * 'flags' - DB flags (see DatabaseBase) |
| | 317 | + * 'dbsByBucket' : Array of 1-16 consecutive integer keys, starting from 0, |
| | 318 | + * each having an odd-numbered list of DB names (peers) as values. |
| | 319 | + * 'lockExpiry' : Lock timeout (seconds) for dropped connections. [optional] |
| 314 | 320 | * This tells the DB server how long to wait before assuming |
| 315 | 321 | * connection failure and releasing all the locks for a session. |
| 316 | | - * 'cliTimeout' : Lock timeout (seconds) for CLI scripts. [optional] |
| 317 | | - * This tells the DB server how long to wait before assuming |
| 318 | | - * connection failure and releasing all the locks for a session. |
| 319 | | - * 'safeDelay' : Seconds to mistrust a DB after restart/query loss. [optional] |
| 320 | | - * This should reflect the highest max_execution_time that PHP |
| 321 | | - * scripts might use on a wiki. Locks are lost on DB server restart. |
| 322 | 322 | * |
| 323 | 323 | * @param Array $config |
| 324 | 324 | */ |
| 325 | 325 | public function __construct( array $config ) { |
| | 326 | + $this->dbServers = $config['dbServers']; |
| 326 | 327 | // Sanitize dbsByBucket config to prevent PHP errors |
| 327 | 328 | $this->dbsByBucket = array_filter( $config['dbsByBucket'], 'is_array' ); |
| 328 | 329 | $this->dbsByBucket = array_values( $this->dbsByBucket ); // consecutive |
| 329 | 330 | |
| 330 | | - if ( isset( $config['webTimeout'] ) ) { |
| 331 | | - $this->webTimeout = $config['webTimeout']; |
| | 331 | + if ( isset( $config['lockExpiry'] ) ) { |
| | 332 | + $this->lockExpiry = $config['lockExpiry']; |
| 332 | 333 | } else { |
| 333 | 334 | $met = ini_get( 'max_execution_time' ); |
| 334 | | - $this->webTimeout = $met ? $met : 60; // use some same amount if 0 |
| | 335 | + $this->lockExpiry = $met ? $met : 60; // use some sane amount if 0 |
| 335 | 336 | } |
| 336 | | - $this->cliTimeout = isset( $config['cliTimeout'] ) |
| 337 | | - ? $config['cliTimeout'] |
| 338 | | - : 60; // some sane amount |
| 339 | | - $this->safeDelay = isset( $config['safeDelay'] ) |
| 340 | | - ? $config['safeDelay'] |
| 341 | | - : max( $this->cliTimeout, $this->webTimeout ); // cover worst case |
| | 337 | + $this->safeDelay = ( $this->lockExpiry <= 0 ) |
| | 338 | + ? 60 // pick a safe-ish number to match DB timeout default |
| | 339 | + : $this->lockExpiry; // cover worst case |
| 342 | 340 | |
| 343 | | - $this->statusCache = wfGetMainCache(); // tracks peers that couldn't be queried recently |
| | 341 | + foreach ( $this->dbsByBucket as $bucket ) { |
| | 342 | + if ( count( $bucket ) > 1 ) { |
| | 343 | + // Tracks peers that couldn't be queried recently to avoid lengthy |
| | 344 | + // connection timeouts. This is useless if each bucket has one peer. |
| | 345 | + $this->statusCache = wfGetMainCache(); |
| | 346 | + break; |
| | 347 | + } |
| | 348 | + } |
| | 349 | + |
| | 350 | + $this->session = mt_rand( 0, 2147483647 ); |
| 344 | 351 | } |
| 345 | 352 | |
| 346 | 353 | protected function doLock( array $keys, $type ) { |
| — | — | @@ -415,7 +422,8 @@ |
| 416 | 423 | } |
| 417 | 424 | |
| 418 | 425 | /** |
| 419 | | - * Get a connection to a lock DB and acquire locks on $keys |
| | 426 | + * Get a connection to a lock DB and acquire locks on $keys. |
| | 427 | + * This does not use GET_LOCK() per http://bugs.mysql.com/bug.php?id=1118. |
| 420 | 428 | * |
| 421 | 429 | * @param $lockDb string |
| 422 | 430 | * @param $keys Array |
| — | — | @@ -426,13 +434,15 @@ |
| 427 | 435 | protected function doLockingQuery( $lockDb, array $keys, $type ) { |
| 428 | 436 | if ( $type == self::LOCK_EX ) { // writer locks |
| 429 | 437 | $db = $this->getConnection( $lockDb ); |
| 430 | | - # Actually do the locking queries... |
| | 438 | + if ( !$db ) { |
| | 439 | + return false; // bad config |
| | 440 | + } |
| 431 | 441 | $data = array(); |
| 432 | 442 | foreach ( $keys as $key ) { |
| 433 | 443 | $data[] = array( 'fle_key' => $key ); |
| 434 | 444 | } |
| 435 | 445 | # Wait on any existing writers and block new ones if we get in |
| 436 | | - $db->insert( 'file_locks_exclusive', $data, __METHOD__ ); |
| | 446 | + $db->insert( 'filelocks_exclusive', $data, __METHOD__ ); |
| 437 | 447 | } |
| 438 | 448 | return true; |
| 439 | 449 | } |
| — | — | @@ -486,40 +496,40 @@ |
| 487 | 497 | } |
| 488 | 498 | |
| 489 | 499 | /** |
| 490 | | - * Get a new connection to a lock DB |
| | 500 | + * Get (or reuse) a connection to a lock DB |
| 491 | 501 | * |
| 492 | 502 | * @param $lockDb string |
| 493 | 503 | * @return Database |
| 494 | 504 | * @throws DBError |
| 495 | 505 | */ |
| 496 | 506 | protected function getConnection( $lockDb ) { |
| 497 | | - if ( !isset( $this->activeConns[$lockDb] ) ) { |
| 498 | | - $this->activeConns[$lockDb] = wfGetDB( DB_MASTER, array(), $lockDb ); |
| 499 | | - $this->activeConns[$lockDb]->begin(); // start transaction |
| | 507 | + if ( !isset( $this->conns[$lockDb] ) ) { |
| | 508 | + $config = $this->dbServers[$lockDb]; |
| | 509 | + $this->conns[$lockDb] = DatabaseBase::factory( $config['type'], $config ); |
| | 510 | + if ( !$this->conns[$lockDb] ) { |
| | 511 | + return null; // config error? |
| | 512 | + } |
| 500 | 513 | # If the connection drops, try to avoid letting the DB rollback |
| 501 | 514 | # and release the locks before the file operations are finished. |
| 502 | | - # This won't handle the case of DB server reboots however. |
| | 515 | + # This won't handle the case of DB server restarts however. |
| 503 | 516 | $options = array(); |
| 504 | | - if ( php_sapi_name() == 'cli' ) { // maintenance scripts |
| 505 | | - if ( $this->cliTimeout > 0 ) { |
| 506 | | - $options['connTimeout'] = $this->cliTimeout; |
| 507 | | - } |
| 508 | | - } else { // web requests |
| 509 | | - if ( $this->webTimeout > 0 ) { |
| 510 | | - $options['connTimeout'] = $this->webTimeout; |
| 511 | | - } |
| | 517 | + if ( $this->lockExpiry > 0 ) { |
| | 518 | + $options['connTimeout'] = $this->lockExpiry; |
| 512 | 519 | } |
| 513 | | - $this->activeConns[$lockDb]->setSessionOptions( $options ); |
| 514 | | - $this->initConnection( $lockDb, $this->activeConns[$lockDb] ); |
| | 520 | + $this->conns[$lockDb]->setSessionOptions( $options ); |
| | 521 | + $this->initConnection( $lockDb, $this->conns[$lockDb] ); |
| 515 | 522 | } |
| 516 | | - return $this->activeConns[$lockDb]; |
| | 523 | + if ( !$this->conns[$lockDb]->trxLevel() ) { |
| | 524 | + $this->conns[$lockDb]->begin(); // start transaction |
| | 525 | + } |
| | 526 | + return $this->conns[$lockDb]; |
| 517 | 527 | } |
| 518 | 528 | |
| 519 | 529 | /** |
| 520 | 530 | * Do additional initialization for new lock DB connection |
| 521 | 531 | * |
| 522 | 532 | * @param $lockDb string |
| 523 | | - * @param $db Database |
| | 533 | + * @param $db DatabaseBase |
| 524 | 534 | * @return void |
| 525 | 535 | * @throws DBError |
| 526 | 536 | */ |
| — | — | @@ -533,15 +543,15 @@ |
| 534 | 544 | */ |
| 535 | 545 | protected function finishLockTransactions() { |
| 536 | 546 | $status = Status::newGood(); |
| 537 | | - foreach ( $this->activeConns as $lockDb => $db ) { |
| 538 | | - try { |
| 539 | | - $db->rollback(); // finish transaction and kill any rows |
| 540 | | - } catch ( DBError $e ) { |
| 541 | | - $status->fatal( 'lockmanager-fail-db-release', $lockDb ); |
| 542 | | - // oh well; best effort |
| | 547 | + foreach ( $this->conns as $lockDb => $db ) { |
| | 548 | + if ( $db->trxLevel() ) { // in transaction |
| | 549 | + try { |
| | 550 | + $db->rollback(); // finish transaction and kill any rows |
| | 551 | + } catch ( DBError $e ) { |
| | 552 | + $status->fatal( 'lockmanager-fail-db-release', $lockDb ); |
| | 553 | + } |
| 543 | 554 | } |
| 544 | 555 | } |
| 545 | | - $this->activeConns = array(); |
| 546 | 556 | return $status; |
| 547 | 557 | } |
| 548 | 558 | |
| — | — | @@ -554,8 +564,8 @@ |
| 555 | 565 | * @return bool |
| 556 | 566 | */ |
| 557 | 567 | protected function lastErrorIndicatesLocked( $lockDb ) { |
| 558 | | - if ( isset( $this->activeConns[$lockDb] ) ) { // sanity |
| 559 | | - $db = $this->activeConns[$lockDb]; |
| | 568 | + if ( isset( $this->conns[$lockDb] ) ) { // sanity |
| | 569 | + $db = $this->conns[$lockDb]; |
| 560 | 570 | return ( $db->wasDeadlock() || $db->wasLockTimeout() ); |
| 561 | 571 | } |
| 562 | 572 | return false; |
| — | — | @@ -570,12 +580,10 @@ |
| 571 | 581 | * @throws DBError |
| 572 | 582 | */ |
| 573 | 583 | protected function checkUptime( $lockDb ) { |
| 574 | | - if ( isset( $this->activeConns[$lockDb] ) ) { // sanity |
| | 584 | + if ( isset( $this->conns[$lockDb] ) ) { // sanity |
| 575 | 585 | if ( $this->safeDelay > 0 ) { |
| 576 | | - $db = $this->activeConns[$lockDb]; |
| 577 | | - if ( $db->getServerUptime() < $this->safeDelay ) { |
| 578 | | - return false; |
| 579 | | - } |
| | 586 | + $db = $this->conns[$lockDb]; |
| | 587 | + return ( $db->getServerUptime() > $this->safeDelay ); |
| 580 | 588 | } |
| 581 | 589 | return true; |
| 582 | 590 | } |
| — | — | @@ -648,10 +656,23 @@ |
| 649 | 657 | * Make sure remaining locks get cleared for sanity |
| 650 | 658 | */ |
| 651 | 659 | function __destruct() { |
| 652 | | - $this->finishLockTransactions(); |
| | 660 | + foreach ( $this->conns as $lockDb => $db ) { |
| | 661 | + if ( $db->trxLevel() ) { // in transaction |
| | 662 | + try { |
| | 663 | + $db->rollback(); // finish transaction and kill any rows |
| | 664 | + } catch ( DBError $e ) { |
| | 665 | + // oh well |
| | 666 | + } |
| | 667 | + } |
| | 668 | + $db->close(); |
| | 669 | + } |
| 653 | 670 | } |
| 654 | 671 | } |
| 655 | 672 | |
| | 673 | +/** |
| | 674 | + * MySQL version of DBLockManager that supports shared locks. |
| | 675 | + * All locks are non-blocking, which avoids deadlocks. |
| | 676 | + */ |
| 656 | 677 | class MySqlLockManager extends DBLockManager { |
| 657 | 678 | /** @var Array Mapping of lock types to the type actually used */ |
| 658 | 679 | protected $lockTypeMap = array( |
| — | — | @@ -660,51 +681,55 @@ |
| 661 | 682 | self::LOCK_EX => self::LOCK_EX |
| 662 | 683 | ); |
| 663 | 684 | |
| 664 | | - /** @var Array Map of (DB name => original transaction isolation) */ |
| 665 | | - protected $trxIso = array(); |
| 666 | | - |
| 667 | 685 | protected function initConnection( $lockDb, DatabaseBase $db ) { |
| 668 | | - # Get the original transaction level for the DB server |
| 669 | | - $row = $db->query( "SELECT @@tx_isolation AS tx_iso;" )->fetchObject(); |
| 670 | | - # Convert "REPEATABLE-READ" => "REPEATABLE READ" for SET query |
| 671 | | - $this->trxIso[$lockDb] = str_replace( '-', ' ', $row->tx_iso ); |
| | 686 | + # Let this transaction see lock rows from other transactions |
| | 687 | + $db->query( "SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;" ); |
| 672 | 688 | } |
| 673 | 689 | |
| 674 | 690 | protected function doLockingQuery( $lockDb, array $keys, $type ) { |
| 675 | | - $ok = true; |
| | 691 | + $db = $this->getConnection( $lockDb ); |
| | 692 | + if ( !$db ) { |
| | 693 | + return false; |
| | 694 | + } |
| | 695 | + $data = array(); |
| | 696 | + foreach ( $keys as $key ) { |
| | 697 | + $data[] = array( 'fls_key' => $key, 'fls_session' => $this->session ); |
| | 698 | + } |
| | 699 | + # Block new writers... |
| | 700 | + $db->insert( 'filelocks_shared', $data, __METHOD__ ); |
| 676 | 701 | # Actually do the locking queries... |
| 677 | 702 | if ( $type == self::LOCK_SH ) { // reader locks |
| 678 | | - $db = $this->getConnection( $lockDb ); |
| 679 | | - $data = array(); |
| 680 | | - foreach ( $keys as $key ) { |
| 681 | | - $data[] = array( 'fls_key' => $key ); |
| 682 | | - } |
| 683 | | - # Block new writers... |
| 684 | | - $db->insert( 'file_locks_shared', $data, __METHOD__ ); |
| 685 | 703 | # Bail if there are any existing writers... |
| 686 | | - $db->query( "SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;" ); |
| 687 | | - $ok = !$db->selectField( 'file_locks_exclusive', '1', |
| | 704 | + $blocked = $db->selectField( 'filelocks_exclusive', '1', |
| 688 | 705 | array( 'fle_key' => $keys ), |
| 689 | 706 | __METHOD__ |
| 690 | 707 | ); |
| 691 | | - $db->query( "SET SESSION TRANSACTION ISOLATION LEVEL {$this->trxIso[$lockDb]};" ); |
| 692 | | - } elseif ( $type == self::LOCK_EX ) { // writer locks |
| 693 | | - $db = $this->getConnection( $lockDb ); |
| 694 | | - $data = array(); |
| 695 | | - foreach ( $keys as $key ) { |
| 696 | | - $data[] = array( 'fle_key' => $key ); |
| 697 | | - } |
| 698 | | - # Block new readers/writers and wait on any existing writers |
| 699 | | - $db->insert( 'file_locks_exclusive', $data, __METHOD__ ); |
| 700 | | - # Bail if there are any existing readers... |
| 701 | | - $db->query( "SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;" ); |
| 702 | | - $ok = !$db->selectField( 'file_locks_shared', '1', |
| 703 | | - array( 'fls_key' => $keys ), |
| | 708 | + # Prospective writers that haven't yet update filelocks_exclusive |
| | 709 | + # will recheck filelocks_shared after doing so and bail due to our entry. |
| | 710 | + } else { // writer locks |
| | 711 | + $encSession = $db->addQuotes( $this->session ); |
| | 712 | + # Bail if there are any existing writers... |
| | 713 | + # The may detect readers, but the safe check for them is below. |
| | 714 | + # Note: if two writers come at the same time, both bail :) |
| | 715 | + $blocked = $db->selectField( 'filelocks_shared', '1', |
| | 716 | + array( 'fls_key' => $keys, "fls_session != $encSession" ), |
| 704 | 717 | __METHOD__ |
| 705 | 718 | ); |
| 706 | | - $db->query( "SET SESSION TRANSACTION ISOLATION LEVEL {$this->trxIso[$lockDb]};" ); |
| | 719 | + if ( !$blocked ) { |
| | 720 | + $data = array(); |
| | 721 | + foreach ( $keys as $key ) { |
| | 722 | + $data[] = array( 'fle_key' => $key ); |
| | 723 | + } |
| | 724 | + # Block new readers/writers... |
| | 725 | + $db->insert( 'filelocks_exclusive', $data, __METHOD__ ); |
| | 726 | + # Bail if there are any existing readers... |
| | 727 | + $blocked = $db->selectField( 'filelocks_shared', '1', |
| | 728 | + array( 'fls_key' => $keys, "fls_session != $encSession" ), |
| | 729 | + __METHOD__ |
| | 730 | + ); |
| | 731 | + } |
| 707 | 732 | } |
| 708 | | - return $ok; |
| | 733 | + return !$blocked; |
| 709 | 734 | } |
| 710 | 735 | } |
| 711 | 736 | |