r113577 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r113576‎ | r113577 | r113578 >
Date:18:17, 11 March 2012
Author:nikerabbit
Status:ok
Tags:i18nreview 
Comment:
* Support for shared TTMServer databases was added (only when using $wgConf)
* Output of TTMServer api module has changed
* TTMServer configs can override the link symbol by adding value for symbol in the config
* Added some simple tests
Modified paths:
  • /trunk/extensions/Translate/README (modified) (history)
  • /trunk/extensions/Translate/Translate.i18n.php (modified) (history)
  • /trunk/extensions/Translate/Translate.php (modified) (history)
  • /trunk/extensions/Translate/api/ApiTTMServer.php (modified) (history)
  • /trunk/extensions/Translate/scripts/ttmserver-export.php (modified) (history)
  • /trunk/extensions/Translate/tests/TTMServerTest.php (added) (history)
  • /trunk/extensions/Translate/utils/TTMServer.php (modified) (history)
  • /trunk/extensions/Translate/utils/TranslationHelpers.php (modified) (history)

Diff [purge]

Index: trunk/extensions/Translate/scripts/ttmserver-export.php
@@ -22,10 +22,14 @@
2323 * @since 2012-01-26
2424 */
2525 class TTMServerBootstrap extends Maintenance {
 26+ /// @var Array Configuration of requested TTMServer
 27+ protected $config;
 28+
2629 public function __construct() {
2730 parent::__construct();
2831 $this->mDescription = 'Script to bootstrap TTMServer';
2932 $this->addOption( 'threads', 'Number of threads', /*required*/false, /*has arg*/true );
 33+ $this->addOption( 'server', 'Server configuration identifier', /*required*/false, /*has arg*/true );
3034 $this->setBatchSize( 100 );
3135 $this->start = microtime( true );
3236 }
@@ -38,23 +42,55 @@
3943 }
4044
4145 public function execute() {
42 - $server = TTMServer::primary();
43 - if ( $server instanceof FakeTTMServer ) {
 46+ global $wgTranslateTranslationServices;
 47+
 48+ // TTMServer is the id of the enabled-by-default instance
 49+ $configKey = $this->getOption( 'server', 'TTMServer' );
 50+ if ( isset( $wgTranslateTranslationServices[$configKey] ) ) {
 51+ $this->config = $config = $wgTranslateTranslationServices[$configKey];
 52+ $server = new TTMServer( $config );
 53+ } else {
4454 $this->error( "Translation memory is not configured properly", 1 );
4555 }
4656
4757 $dbw = $server->getDB( DB_MASTER );
4858
49 - $this->statusLine( 'Deleting sources.. ', 1 );
50 - $dbw->delete( 'translate_tms', '*', __METHOD__ );
51 - $this->output( 'translations.. ', 1 );
52 - $dbw->delete( 'translate_tmt', '*', __METHOD__ );
53 - $this->output( 'fulltext.. ', 1 );
54 - $dbw->delete( 'translate_tmf', '*', __METHOD__ );
55 - $table = $dbw->tableName( 'translate_tmf' );
56 - $dbw->query( "DROP INDEX tmf_text ON $table" );
57 - $this->output( 'done!', 1 );
 59+ if ( $server->isShared() ) {
 60+ $wiki = array( 'tms_wiki' => wfWikiId() );
5861
 62+ $this->statusLine( 'Deleting fulltext.. ', 1 );
 63+ $dbw->deleteJoin(
 64+ 'translate_tmf', 'translate_tms',
 65+ 'tmf_sid', 'tms_sid',
 66+ $wiki, __METHOD__
 67+ );
 68+
 69+ $this->output( 'translations.. ', 1 );
 70+ $dbw->deleteJoin(
 71+ 'translate_tmt', 'translate_tms',
 72+ 'tmt_sid', 'tms_sid',
 73+ $wiki, __METHOD__
 74+ );
 75+
 76+ $this->output( 'sources.. ', 1 );
 77+ $dbw->delete( 'translate_tms', $wiki, __METHOD__ );
 78+ $this->output( 'done!', 1 );
 79+ } else {
 80+ // For dedicated databases we can just wipe out everything,
 81+ // drop the index during bootstrap and readd it later.
 82+ $this->statusLine( 'Deleting sources.. ', 1 );
 83+ $dbw->delete( 'translate_tms', '*', __METHOD__ );
 84+ $this->output( 'translations.. ', 1 );
 85+ $dbw->delete( 'translate_tmt', '*', __METHOD__ );
 86+ $this->output( 'fulltext.. ', 1 );
 87+ $dbw->delete( 'translate_tmf', '*', __METHOD__ );
 88+ $table = $dbw->tableName( 'translate_tmf' );
 89+ $dbw->ignoreErrors( true );
 90+ $dbw->query( "DROP INDEX tmf_text ON $table" );
 91+ $dbw->ignoreErrors( false );
 92+ $this->output( 'done!', 1 );
 93+ }
 94+
5995 $this->statusLine( 'Loading groups... ', 2 );
6096 $groups = MessageGroups::singleton()->getGroups();
6197 $this->output( 'done!', 2 );
@@ -99,17 +135,19 @@
100136 pcntl_waitpid( $pid, $status );
101137 }
102138
103 - $this->statusLine( 'Adding fulltext index...', 9 );
104 - $table = $dbw->tableName( 'translate_tmf' );
105 - $dbw->query( "CREATE FULLTEXT INDEX tmf_text ON $table (tmf_text)" );
106 - $this->output( ' done!', 9 );
 139+ if ( !$server->isShared() ) {
 140+ $this->statusLine( 'Adding fulltext index...', 9 );
 141+ $table = $dbw->tableName( 'translate_tmf' );
 142+ $dbw->query( "CREATE FULLTEXT INDEX tmf_text ON $table (tmf_text)" );
 143+ $this->output( ' done!', 9 );
 144+ }
107145 }
108146
109147 protected function exportGroup( MessageGroup $group, $multi = false ) {
110148 // Make sure all existing connections are dead,
111149 // we can't use them in forked children.
112150 LBFactory::destroyInstance();
113 - $server = TTMServer::primary();
 151+ $server = new TTMServer( $this->config );
114152
115153 $id = $group->getId();
116154 $sourceLanguage = $group->getSourceLanguage();
Index: trunk/extensions/Translate/tests/TTMServerTest.php
@@ -0,0 +1,115 @@
 2+<?php
 3+/**
 4+ * Tests for TTMServer
 5+ *
 6+ * @file
 7+ * @author Niklas Laxström
 8+ * @copyright Copyright © 2012, Niklas Laxström
 9+ * @license http://www.gnu.org/copyleft/gpl.html GNU General Public License 2.0 or later
 10+ */
 11+
 12+class TTMServerTest extends MediaWikiTestCase {
 13+
 14+ protected $config;
 15+
 16+ protected function setUp() {
 17+ global $wgTranslateTranslationServices;
 18+ $this->config = $wgTranslateTranslationServices;
 19+ parent::setUp();
 20+
 21+ $wgTranslateTranslationServices = array();
 22+ $wgTranslateTranslationServices['localtm'] = array(
 23+ 'url' => 'http://example.com/sandwiki/api.php',
 24+ 'displayname' => 'example.com',
 25+ 'cutoff' => 0.75,
 26+ 'type' => 'ttmserver',
 27+ );
 28+
 29+ $wgTranslateTranslationServices['apitm'] = array(
 30+ 'url' => 'http://example.com/w/api.php',
 31+ 'displayname' => 'example.com',
 32+ 'cutoff' => 0.75,
 33+ 'timeout-sync' => 4,
 34+ 'timeout-async' => 4,
 35+ 'type' => 'remote-ttmserver',
 36+ );
 37+
 38+ $wgTranslateTranslationServices['sharedtm'] = array(
 39+ 'database' => 'sharedtm',
 40+ 'cutoff' => 0.75,
 41+ 'timeout-sync' => 4,
 42+ 'timeout-async' => 4,
 43+ 'type' => 'shared-ttmserver',
 44+ );
 45+
 46+ }
 47+
 48+ protected function tearDown() {
 49+ global $wgTranslateTranslationServices;
 50+ $wgTranslateTranslationServices = $this->config;
 51+ parent::tearDown();
 52+ }
 53+
 54+ public function testConstruct() {
 55+ $server = TTMServer::primary();
 56+ $this->assertEquals(
 57+ 'FakeTTMServer',
 58+ get_class( $server ),
 59+ 'Fake server given when default server is disabled'
 60+ );
 61+ global $wgTranslateTranslationServices;
 62+ $wgTranslateTranslationServices['TTMServer'] = array(
 63+ 'database' => false, // Passed to wfGetDB
 64+ 'cutoff' => 0.75,
 65+ 'type' => 'ttmserver',
 66+ 'public' => false,
 67+ );
 68+ $server = TTMServer::primary();
 69+ $this->assertEquals(
 70+ 'TTMServer',
 71+ get_class( $server ),
 72+ 'Real server given when default server is enabled'
 73+ );
 74+ unset( $wgTranslateTranslationServices['TTMServer'] );
 75+ }
 76+
 77+ public function testFakeTTMServer() {
 78+ $server = new FakeTTMServer();
 79+ $this->assertEquals(
 80+ array(),
 81+ $server->query( 'en', 'fi', 'daa' ),
 82+ 'FakeTTMServer returns no suggestions for all queries'
 83+ );
 84+
 85+ $title = new Title();
 86+ $handle = new MessageHandle( $title );
 87+
 88+ $this->assertFalse(
 89+ $server->update( $handle, 'text' ),
 90+ 'FakeTTMServer returns false on update'
 91+ );
 92+
 93+ $this->assertFalse(
 94+ $server->insertSource( $title, 'fi', 'teksti' ),
 95+ 'FakeTTMServer returns false on insertSource'
 96+ );
 97+ }
 98+
 99+ /**
 100+ * @dataProvider dataIsShared
 101+ */
 102+ public function testIsShared( $server, $value ) {
 103+ global $wgTranslateTranslationServices;
 104+ $server = new TTMServer( $wgTranslateTranslationServices[$server] );
 105+ $this->assertEquals( $value, $server->isShared() );
 106+ }
 107+
 108+ public function dataIsShared() {
 109+ return array(
 110+ array( 'localtm', false ),
 111+ array( 'apitm', false ),
 112+ array( 'sharedtm', true ),
 113+ );
 114+ }
 115+
 116+}
Property changes on: trunk/extensions/Translate/tests/TTMServerTest.php
___________________________________________________________________
Added: svn:eol-style
1117 + native
Index: trunk/extensions/Translate/Translate.php
@@ -345,8 +345,6 @@
346346 $wgTranslateTranslationServices['TTMServer'] = array(
347347 'database' => false, // Passed to wfGetDB
348348 'cutoff' => 0.75,
349 - 'timeout-sync' => 0, // Unused
350 - 'timeout-async' => 0, // Unused
351349 'type' => 'ttmserver',
352350 'public' => false,
353351 );
Index: trunk/extensions/Translate/README
@@ -30,6 +30,11 @@
3131 https://translatewiki.net/docs/Translate/html/
3232
3333 == Change log ==
 34+* 2012-03-11
 35+- Support for shared TTMServer databases was added
 36+- Suggestions from different TTMServers are now grouped
 37+- Output of TTMServer api module has changed
 38+- TTMServer configs can override the link symbol by adding value for symbol in the config
3439 * 2012-03-07
3540 - $wgTranslateGroupStructure is no longer used. If you are using aggregate message
3641 groups, you can remove the old settings and everything still works as expected.
Index: trunk/extensions/Translate/utils/TranslationHelpers.php
@@ -258,7 +258,8 @@
259259 $source = $this->group->getSourceLanguage();
260260 $code = $this->handle->getCode();
261261 $definition = $this->getDefinition();
262 - $suggestions = TTMServer::primary()->query( $source, $code, $definition );
 262+ $TTMServer = new TTMServer( $config );
 263+ $suggestions = $TTMServer->query( $source, $code, $definition );
263264 if ( count( $suggestions ) === 0 ) {
264265 throw new TranslationHelperExpection( 'No suggestions' );
265266 }
@@ -319,34 +320,37 @@
320321 $suggestions = $wrapper['suggestions'];
321322
322323 foreach ( $suggestions as $s ) {
323 - $accuracy = wfMsgHtml( 'translate-edit-tmmatch' , sprintf( '%.2f', $s['quality'] * 100 ) );
 324+ $tooltip = wfMessage( 'translate-edit-tmmatch-source', $s['source'] )->plain();
 325+ $text = wfMessage( 'translate-edit-tmmatch', sprintf( '%.2f', $s['quality'] * 100 ) )->plain();
 326+ $accuracy = Html::element( 'span', array( 'title' => $tooltip ), $text );
324327 $legend = array( $accuracy => array() );
325328
326 - if ( $config['type'] === 'remote-ttmserver' ) {
 329+ $TTMServer = new TTMServer( $config );
 330+ if ( $TTMServer->isLocalSuggestion( $s ) ) {
 331+ $title = Title::newFromText( $s['location'] );
 332+ $symbol = isset( $config['symbol'] ) ? $config['symbol'] : '•';
 333+ $legend[$accuracy][] = self::ajaxEditLink( $title, '•' );
 334+ } else {
 335+ if ( $config['type'] === 'remote-ttmserver' ) {
 336+ $displayName = $config['displayname'];
 337+ } else {
 338+ $wiki = WikiMap::getWiki( $s['wiki'] );
 339+ $displayName = $wiki->getDisplayName() . ': ' . $s['location'];
 340+ }
 341+
327342 $params = array(
328 - 'href' => $url = $config['viewurl'] . $s['context'],
 343+ 'href' => $TTMServer->expandLocation( $s ),
329344 'target' => '_blank',
330 - 'title' => "{$config['displayname']}: {$s['context']}",
 345+ 'title' => $displayName,
331346 );
332 - $legend[$accuracy][] = Html::element( 'a', $params, '‣' );
333 - } else {
334 - $sourceTitle = Title::newFromText( $s['context'] );
335 - $handle = new MessageHandle( $sourceTitle );
336 - $targetTitle = Title::makeTitle( $sourceTitle->getNamespace(), $handle->getKey() . "/$code" );
337 - if ( $targetTitle ) {
338 - $legend[$accuracy][] = self::ajaxEditLink( $targetTitle, '•' );
339 - }
 347+
 348+ $symbol = isset( $config['symbol'] ) ? $config['symbol'] : '‣';
 349+ $legend[$accuracy][] = Html::element( 'a', $params, $symbol );
340350 }
341351
342 - // Show the source text in a tooltip
343 - if ( isset( $s['target'] ) ) {
344 - $suggestion = $s['target'];
345 - } else {
346 - $suggestion = $s['*'];
347 -
348 - }
 352+ $suggestion = $s['target'];
349353 $text = $this->suggestionField( $suggestion );
350 - $params = array( 'class' => 'mw-sp-translate-edit-tmsug', 'title' => $s['source'] );
 354+ $params = array( 'class' => 'mw-sp-translate-edit-tmsug' );
351355
352356 // Group identical suggestions together
353357 if ( isset( $sugFields[$suggestion] ) ) {
@@ -386,6 +390,7 @@
387391
388392 $handlers = array(
389393 'ttmserver' => 'getTTMServerBox',
 394+ 'shared-ttmserver' => 'getTTMServerBox',
390395 'remote-ttmserver' => 'getRemoteTTMServerBox',
391396 'microsoft' => 'getMicrosoftSuggestion',
392397 'apertium' => 'getApertiumSuggestion',
@@ -395,17 +400,19 @@
396401 $boxes = array();
397402 $TTMSSug = array();
398403 foreach ( $wgTranslateTranslationServices as $name => $config ) {
399 - if ( $async === 'async' ) {
400 - $config['timeout'] = $config['timeout-async'];
401 - } else {
402 - $config['timeout'] = $config['timeout-sync'];
 404+ $type = $config['type'];
 405+ if ( $type !== 'ttmserver' ) {
 406+ if ( $async === 'async' ) {
 407+ $config['timeout'] = $config['timeout-async'];
 408+ } else {
 409+ $config['timeout'] = $config['timeout-sync'];
 410+ }
403411 }
404412
405 - $type = $config['type'];
406413 if ( isset( $handlers[$type] ) ) {
407414 $method = $handlers[$type];
408415 try {
409 - if ( $type === 'ttmserver' || $type === 'remote-ttmserver' ) {
 416+ if ( $type === 'ttmserver' || $type === 'remote-ttmserver' || $type === 'shared-ttmserver' ) {
410417 $TTMSSug[$name] = array(
411418 'config' => $config,
412419 'suggestions' => $this->$method( $name, $config ),
Index: trunk/extensions/Translate/utils/TTMServer.php
@@ -71,6 +71,51 @@
7272 }
7373
7474 /**
 75+ * Determines if the suggestion returned by this TTMServer comes
 76+ * from this wiki or any other wiki.
 77+ * @return Bool
 78+ * @since 2012-03-11
 79+ */
 80+ public function isLocalSuggestion( array $suggestion ) {
 81+ $type = $this->config['type'];
 82+ $isLocal = $type === 'ttmserver';
 83+ if ( $type === 'shared-ttmserver' && $suggestion['wiki'] === wfWikiId() ) {
 84+ $isLocal = true;
 85+ }
 86+ return $isLocal;
 87+ }
 88+
 89+ /**
 90+ * Given suggestion returned by this TTMServer, constructs fully
 91+ * qualified URL to the location of the translation.
 92+ * @return String URL
 93+ * @since 2012-03-11
 94+ */
 95+ public function expandLocation( array $suggestion ) {
 96+ $type = $this->config['type'];
 97+ if ( $type === 'shared-ttmserver' ) {
 98+ $wiki = WikiMap::getWiki( $suggestion['wiki'] );
 99+ return $wiki->getCanonicalUrl( $suggestion['location'] );
 100+ } elseif ( $type === 'remote-ttmserver' ) {
 101+ return $suggestion['location'];
 102+ } else {
 103+ $title = Title::newFromText( $suggestion['location'] );
 104+ return $title->getCanonicalUrl();
 105+ }
 106+ }
 107+
 108+ /**
 109+ * Is this TTMServer instance using shared translation memory database.
 110+ * This affects the database layout, shared databases have an extra field
 111+ * and we have to be considerate about data originating from other wikies.
 112+ * @return Bool
 113+ * @since 2012-03-11
 114+ */
 115+ public function isShared() {
 116+ return $this->config['type'] === 'shared-ttmserver';
 117+ }
 118+
 119+ /**
75120 * @param $mode int
76121 * @return DatabaseBase
77122 */
@@ -111,6 +156,10 @@
112157 'tms_context' => $title->getPrefixedText(),
113158 'tms_text' => $definition,
114159 );
 160+ if ( $this->isShared() ) {
 161+ $conds['tms_wiki'] = wfWikiId();
 162+ }
 163+
115164 $sid = $dbw->selectField( 'translate_tms', 'tms_sid', $conds, __METHOD__ );
116165 if ( $sid === false ) {
117166 $sid = $this->insertSource( $title, $sourceLanguage, $definition );
@@ -142,6 +191,10 @@
143192 'tms_context' => $context->getPrefixedText(),
144193 );
145194
 195+ if ( $this->isShared() ) {
 196+ $row['tms_wiki'] = wfWikiId();
 197+ }
 198+
146199 $dbw = $this->getDB( DB_MASTER );
147200 $dbw->insert( 'translate_tms', $row, __METHOD__ );
148201 $sid = $dbw->insertId();
@@ -172,6 +225,10 @@
173226 $dbr = $this->getDB( DB_SLAVE );
174227 $tables = array( 'translate_tmt', 'translate_tms' );
175228 $fields = array( 'tms_context', 'tms_text', 'tmt_lang', 'tmt_text' );
 229+ if ( $this->isShared() ) {
 230+ $fields[] = 'tms_wiki';
 231+ }
 232+
176233 $conds = array(
177234 'tms_lang' => $sourceLanguage,
178235 'tmt_lang' => $targetLanguage,
@@ -189,10 +246,10 @@
190247
191248 $res = $dbr->select( $tables, $fields, $conds, __METHOD__ );
192249 wfProfileOut( __METHOD__ );
193 - return $this->processQueryResults( $res, $text );
 250+ return $this->processQueryResults( $res, $text, $sourceLanguage, $targetLanguage );
194251 }
195252
196 - protected function processQueryResults( $res, $text ) {
 253+ protected function processQueryResults( $res, $text, $sourceLanguage, $targetLanguage ) {
197254 wfProfileIn( __METHOD__ );
198255 $lenA = mb_strlen( $text );
199256 $results = array();
@@ -211,11 +268,14 @@
212269 $quality = 1 - ( $dist / $len );
213270
214271 if ( $quality >= $this->config['cutoff'] ) {
 272+ $loc = self::contextToLocation( $row->tms_context, $sourceLanguage, $targetLanguage );
215273 $results[] = array(
216274 'source' => $row->tms_text,
217275 'target' => $row->tmt_text,
218276 'context' => $row->tms_context,
 277+ 'location' => $loc,
219278 'quality' => $quality,
 279+ 'wiki' => isset( $row->tms_wiki ) ? $row->tms_wiki : false,
220280 );
221281 }
222282 }
@@ -265,6 +325,18 @@
266326 }
267327
268328 /**
 329+ * Given pagename Namespace:Page/en, replace the language code with
 330+ * the target language. In theory namespaces could be stored differently,
 331+ * but in MediaWiki this is standard practise.
 332+ * @return String
 333+ * @since 2012-03-11
 334+ */
 335+ protected static function contextToLocation( $text, $sourceLanguage, $targetLanguage ) {
 336+ $len = strlen( $sourceLanguage );
 337+ return substr( $text, 0, -$len ) . $targetLanguage;
 338+ }
 339+
 340+ /**
269341 * The native levenshtein is limited to 255 bytes.
270342 */
271343 public static function levenshtein( $str1, $str2, $length1, $length2 ) {
Index: trunk/extensions/Translate/Translate.i18n.php
@@ -83,6 +83,7 @@
8484 'translate-edit-committed' => 'Current translation in software',
8585 'translate-edit-warnings' => 'Warnings about incomplete translations',
8686 'translate-edit-tmsugs' => 'Suggestions from translation memories and machine translation',
 87+ 'translate-edit-tmmatch-source' => 'Translation source text: $1',
8788 'translate-edit-tmmatch' => '$1% match',
8889 'translate-use-suggestion' => 'Replace current translation with this suggestion.',
8990
Index: trunk/extensions/Translate/api/ApiTTMServer.php
@@ -29,8 +29,8 @@
3030
3131 $result = $this->getResult();
3232 foreach ( $suggestions as $sug ) {
33 - $result->setContent( $sug, $sug['target'] );
34 - unset( $sug['target'] );
 33+ $sug['location'] = $server->expandLocation( $sug );
 34+ unset( $sug['wiki'] );
3535 $result->addValue( $this->getModuleName(), null, $sug );
3636 }
3737

Status & tagging log