| Index: trunk/extensions/Translate/TranslateTasks.php |
| — | — | @@ -210,7 +210,7 @@ |
| 211 | 211 | } |
| 212 | 212 | } |
| 213 | 213 | } else { |
| 214 | | - TranslationHelpers::reportTranslationSerficeFailure( 'tmserver' ); |
| | 214 | + TranslationHelpers::reportTranslationServiceFailure( 'tmserver' ); |
| 215 | 215 | } |
| 216 | 216 | unset( $this->collection[$key] ); |
| 217 | 217 | } |
| Index: trunk/extensions/Translate/utils/TranslationHelpers.php |
| — | — | @@ -280,7 +280,7 @@ |
| 281 | 281 | } |
| 282 | 282 | } else { |
| 283 | 283 | // Assume timeout |
| 284 | | - self::reportTranslationSerficeFailure( $serviceName ); |
| | 284 | + self::reportTranslationServiceFailure( $serviceName ); |
| 285 | 285 | } |
| 286 | 286 | |
| 287 | 287 | $boxes = array_slice( $boxes, 0, 3 ); |
| — | — | @@ -356,7 +356,7 @@ |
| 357 | 357 | if ( $json === false ) { |
| 358 | 358 | wfWarn( __METHOD__ . ': Http::get failed' ); |
| 359 | 359 | // Most likely a timeout or other general error |
| 360 | | - self::reportTranslationSerficeFailure( $serviceName ); |
| | 360 | + self::reportTranslationServiceFailure( $serviceName ); |
| 361 | 361 | |
| 362 | 362 | return null; |
| 363 | 363 | } elseif ( !is_object( $response ) ) { |
| — | — | @@ -376,7 +376,7 @@ |
| 377 | 377 | $wgMemc->set( $memckey, $unsupported, 60 * 60 * 8 ); |
| 378 | 378 | } else { |
| 379 | 379 | // Unknown error, assume the worst |
| 380 | | - self::reportTranslationSerficeFailure( $serviceName ); |
| | 380 | + self::reportTranslationServiceFailure( $serviceName ); |
| 381 | 381 | wfWarn( __METHOD__ . "($serviceName): " . $response->responseDetails ); |
| 382 | 382 | error_log( __METHOD__ . "($serviceName): " . $response->responseDetails ); |
| 383 | 383 | return null; |
| — | — | @@ -425,7 +425,7 @@ |
| 426 | 426 | $response = FormatJson::decode( $json ); |
| 427 | 427 | |
| 428 | 428 | if ( $json === false ) { |
| 429 | | - self::reportTranslationSerficeFailure( $serviceName ); |
| | 429 | + self::reportTranslationServiceFailure( $serviceName ); |
| 430 | 430 | return null; |
| 431 | 431 | } elseif ( !is_object( $response ) ) { |
| 432 | 432 | error_log( __METHOD__ . ': Unable to parse reply: ' . strval( $json ) ); |
| — | — | @@ -479,7 +479,7 @@ |
| 480 | 480 | $json = Http::post( $config['url'], $options ); |
| 481 | 481 | $response = FormatJson::decode( $json ); |
| 482 | 482 | if ( $json === false || !is_object( $response ) ) { |
| 483 | | - self::reportTranslationSerficeFailure( $serviceName ); |
| | 483 | + self::reportTranslationServiceFailure( $serviceName ); |
| 484 | 484 | break; // Too slow, back off |
| 485 | 485 | } elseif ( $response->responseStatus !== 200 ) { |
| 486 | 486 | error_log( __METHOD__ . " with ($serviceName ($candidate)): " . $response->responseDetails ); |
| — | — | @@ -937,32 +937,49 @@ |
| 938 | 938 | /** |
| 939 | 939 | * Checks whether the given service has exceeded failure count */ |
| 940 | 940 | public static function checkTranslationServiceFailure( $service ) { |
| 941 | | - global $wgMemc; |
| | 941 | + global $wgMemc, $wgContLang; |
| 942 | 942 | |
| 943 | 943 | $key = wfMemckey( "translate-service-$service" ); |
| | 944 | + $value = $wgMemc->get( $key ); |
| | 945 | + if ( !is_string( $value ) ) return false; |
| | 946 | + list( $count, $failed ) = explode( '|', $value, 2 ); |
| 944 | 947 | |
| 945 | | - // Both false and null are converted to zero, which is desirable |
| 946 | | - return intval( $wgMemc->get( $key ) ) >= self::$serviceFailureCount; |
| | 948 | + if ( $failed + ( 2 * self::$serviceFailurePeriod ) < wfTimestamp() ) { |
| | 949 | + error_log( "Translation service $service (was) restored" ); |
| | 950 | + $wgMemc->delete( $key ); |
| | 951 | + return false; |
| | 952 | + } elseif ( $failed + self::$serviceFailurePeriod < wfTimestamp() ) { |
| | 953 | + /* We are in suspicious mode and one failure is enough to update |
| | 954 | + * failed timestamp. If the service works however, let's use it. |
| | 955 | + * Previous failures are forgotten after another failure period |
| | 956 | + * has passed */ |
| | 957 | + return false; |
| | 958 | + } |
| | 959 | + |
| | 960 | + return $count >= self::$serviceFailureCount; |
| 947 | 961 | } |
| 948 | 962 | |
| 949 | 963 | /** |
| 950 | 964 | * Increases the failure count for a given service */ |
| 951 | | - public static function reportTranslationSerficeFailure( $service ) { |
| | 965 | + public static function reportTranslationServiceFailure( $service ) { |
| 952 | 966 | global $wgMemc; |
| 953 | 967 | |
| 954 | 968 | $key = wfMemckey( "translate-service-$service" ); |
| 955 | | - // Both false and null are converted to zero, which is desirable. |
| 956 | | - /* FIXME: not atomic, but the default incr() implemention seems to |
| 957 | | - * ignore expiry time */ |
| 958 | | - $count = intval( $wgMemc->get( $key ) ); |
| 959 | | - $wgMemc->set( $key, $count + 1, self::$serviceFailurePeriod ); |
| | 969 | + $value = $wgMemc->get( $key ); |
| | 970 | + if ( !is_string( $value ) ) { |
| | 971 | + $count = 0; |
| | 972 | + } else { |
| | 973 | + list( $count, ) = explode( '|', $value, 2 ); |
| | 974 | + } |
| 960 | 975 | |
| 961 | | - /* By using >= we expose if something is still increasing failure |
| 962 | | - * count if we are over the limit */ |
| 963 | | - if ( $count + 1 >= self::$serviceFailureCount ) { |
| 964 | | - $language = Language::factory( 'en' ); |
| 965 | | - $period = $language->formatTimePeriod( self::$serviceFailurePeriod ); |
| 966 | | - error_log( "Translation service $service suspended for $period" ); |
| | 976 | + $count += 1; |
| | 977 | + $failed = wfTimestamp(); |
| | 978 | + $wgMemc->set( $key, "$count|$failed", self::$serviceFailurePeriod*5 ); |
| | 979 | + |
| | 980 | + if ( $count == self::$serviceFailureCount ) { |
| | 981 | + error_log( "Translation service $service suspended" ); |
| | 982 | + } elseif( $count > self::$serviceFailureCount ) { |
| | 983 | + error_log( "Translation service $service still suspended" ); |
| 967 | 984 | } |
| 968 | 985 | } |
| 969 | 986 | } |