r58322 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r58321‎ \| r58322 \| r58323 >
Date:	16:19, 29 October 2009
Author:	maxsem
Status:	resolved (Comments)
Tags:
Comment:	Merge SQLite search from branches/sqlite/ to trunk
Modified paths:	/trunk/phase3 (modified) (history) /trunk/phase3/config/Installer.php (modified) (history) /trunk/phase3/includes (modified) (history) /trunk/phase3/includes/AutoLoader.php (modified) (history) /trunk/phase3/includes/db/DatabaseSqlite.php (modified) (history) /trunk/phase3/includes/search/SearchSqlite.php (added) (history) /trunk/phase3/maintenance/rebuildtextindex.php (modified) (history) /trunk/phase3/maintenance/sqlite/archives/searchindex-fts3.sql (added) (history) /trunk/phase3/maintenance/sqlite/archives/searchindex-no-fts.sql (added) (history) /trunk/phase3/maintenance/updaters.inc (modified) (history)

Diff [purge]

Index: trunk/phase3/maintenance/sqlite/archives/searchindex-no-fts.sql
—	—	@@ -0,0 +1,25 @@
	2	+-- Searchindex table definition for cases when no full-text search SQLite module is present
	3	+-- (currently, only FTS3 is supported).
	4	+-- Use it if you are moving your database from environment with FTS support
	5	+-- to environment without it.
	6	+
	7	+DROP TABLE IF EXISTS /_/searchindex;
	8	+
	9	+-- These are pieces of FTS3-enabled searchindex
	10	+DROP TABLE IF EXISTS /_/searchindex_content;
	11	+DROP TABLE IF EXISTS /_/searchindex_segdir;
	12	+DROP TABLE IF EXISTS /_/searchindex_segments;
	13	+
	14	+CREATE TABLE /_/searchindex (
	15	+ -- Key to page_id
	16	+ -- Disabled, instead we use the built-in rowid column
	17	+ -- si_page INTEGER NOT NULL,
	18	+
	19	+ -- Munged version of title
	20	+ si_title TEXT,
	21	+
	22	+ -- Munged version of body text
	23	+ si_text TEXT
	24	+);
	25	+
	26	+DELETE FROM /_/updatelog WHERE ul_key='fts3';
\ No newline at end of file
Property changes on: trunk/phase3/maintenance/sqlite/archives/searchindex-no-fts.sql
___________________________________________________________________
Name: svn:eol-style
1	27	+ native
Index: trunk/phase3/maintenance/sqlite/archives/searchindex-fts3.sql
—	—	@@ -0,0 +1,18 @@
	2	+-- Patch that introduces fulltext search capabilities to SQLite schema
	3	+-- Requires that SQLite must be compiled with FTS3 module (comes with core amalgamation).
	4	+-- See http://www.sqlite.org/cvstrac/wiki?p=FtsUsage for details of syntax.
	5	+-- Will fail if FTS3 is not present,
	6	+DROP TABLE IF EXISTS /_/searchindex;
	7	+CREATE VIRTUAL TABLE /_/searchindex USING FTS3(
	8	+ -- Key to page_id
	9	+ -- Disabled, instead we use the built-in rowid column
	10	+ -- si_page INTEGER NOT NULL,
	11	+
	12	+ -- Munged version of title
	13	+ si_title,
	14	+
	15	+ -- Munged version of body text
	16	+ si_text
	17	+);
	18	+
	19	+INSERT INTO /_/updatelog VALUES ('fts3');
\ No newline at end of file
Property changes on: trunk/phase3/maintenance/sqlite/archives/searchindex-fts3.sql
___________________________________________________________________
Name: svn:eol-style
1	20	+ native
Index: trunk/phase3/maintenance/updaters.inc
—	—	@@ -189,6 +189,9 @@
190	190	array( 'add_index', 'log_search', 'ls_field_val', 'patch-log_search-rename-index.sql' ),
191	191	array( 'add_index', 'change_tag', 'change_tag_rc_tag', 'patch-change_tag-indexes.sql' ),
192	192	array( 'add_field', 'redirect', 'rd_interwiki', 'patch-rd_interwiki.sql' ),
	193	+
	194	+ // version-independent searchindex setup, added in 1.16
	195	+ array( 'sqlite_setup_searchindex' ),
193	196	),
194	197	);
195	198
—	—	@@ -1281,6 +1284,23 @@
1282	1285	wfOut( "done\n" );
1283	1286	}
1284	1287
	1288	+function sqlite_setup_searchindex() {
	1289	+ global $wgDatabase;
	1290	+ $module = $wgDatabase->getFulltextSearchModule();
	1291	+ $fts3tTable = update_row_exists( 'fts3' );
	1292	+ if ( $fts3tTable && !$module ) {
	1293	+ wfOut( '...PHP is missing FTS3 support, downgrading tables...' );
	1294	+ $wgDatabase->sourceFile( archive( 'searchindex-no-fts.sql' ) );
	1295	+ wfOut( "done\n" );
	1296	+ } elseif ( !$fts3tTable && $module == 'FTS3' ) {
	1297	+ wfOut( '...adding FTS3 search capabilities...' );
	1298	+ $wgDatabase->sourceFile( archive( 'searchindex-fts3.sql' ) );
	1299	+ wfOut( "done\n" );
	1300	+ } else {
	1301	+ wfOut( "...fulltext search table appears to be in order.\n" );
	1302	+ }
	1303	+}
	1304	+
1285	1305	function do_unique_pl_tl_il() {
1286	1306	global $wgDatabase;
1287	1307	$info = $wgDatabase->indexInfo( 'pagelinks', 'pl_namespace' );
Index: trunk/phase3/maintenance/rebuildtextindex.php
—	—	@@ -1,9 +1,8 @@
2	2	<?php
3	3	/**
4		~~- * Rebuild search index table from scratch. This takes several~~
	4	+ * Rebuild search index table from scratch. This may take several
5	5	* hours, depending on the database size and server configuration.
6	6	*
7		~~- * This is only for MySQL (see bug 9905).~~
8	7	* Postgres is trigger-based and should never need rebuilding.
9	8	*
10	9	* This program is free software; you can redistribute it and/or modify
—	—	@@ -28,9 +27,9 @@
29	28	require_once( dirname(__FILE__) . '/Maintenance.php' );
30	29
31	30	class RebuildTextIndex extends Maintenance {
	31	+ const RTI_CHUNK_SIZE = 500;
	32	+ private $db;
32	33
33		~~- const RTI_CHUNK_SIZE = 500;~~
34		-
35	34	public function __construct() {
36	35	parent::__construct();
37	36	$this->mDescription = "Rebuild search index table from scratch";
—	—	@@ -41,46 +40,34 @@
42	41	}
43	42
44	43	public function execute() {
45		~~- global $wgTitle;~~
46		-
47		~~- // Only do this for MySQL~~
48		~~- $database = wfGetDB( DB_MASTER );~~
49		~~- if( !$database instanceof DatabaseMysql ) {~~
50		~~- $this->error( "This script is only for MySQL.", true );~~
	44	+ global $wgTitle, $wgDBtype;
	45	+
	46	+ // Shouldn't be needed for Postgres
	47	+ if ( $wgDBtype == 'postgres' ) {
	48	+ $this->error( "This script does not work with PostgreSQL.\n", true );
51	49	}
52		-
	50	+
	51	+ $this->db = wfGetDB( DB_MASTER );
53	52	$wgTitle = Title::newFromText( "Rebuild text index script" );
54	53
55		~~- $this->dropTextIndex( $database );~~
56		~~- $this->doRebuildTextIndex( $database );~~
57		~~- $this->createTextIndex( $database );~~
	54	+ if ( $wgDBtype == 'mysql' ) {
	55	+ $this->dropMysqlTextIndex();
	56	+ $this->populateSearchIndex();
	57	+ $this->createMysqlTextIndex();
	58	+ } else {
	59	+ $this->clearSearchIndex();
	60	+ $this->populateSearchIndex();
	61	+ }
58	62
59	63	$this->output( "Done.\n" );
60	64	}
61		-
62		~~- private function dropTextIndex( &$database ) {~~
63		~~- $searchindex = $database->tableName( 'searchindex' );~~
64		~~- if ( $database->indexExists( "searchindex", "si_title" ) ) {~~
65		~~- $this->output( "Dropping index...\n" );~~
66		~~- $sql = "ALTER TABLE $searchindex DROP INDEX si_title, DROP INDEX si_text";~~
67		~~- $database->query($sql, "dropTextIndex" );~~
68		~~- }~~
69		~~- }~~
70	65
71		~~- private function createTextIndex( &$database ) {~~
72		~~- $searchindex = $database->tableName( 'searchindex' );~~
73		~~- $this->output( "\nRebuild the index...\n" );~~
74		~~- $sql = "ALTER TABLE $searchindex ADD FULLTEXT si_title (si_title), " .~~
75		~~- "ADD FULLTEXT si_text (si_text)";~~
76		~~- $database->query($sql, "createTextIndex" );~~
77		~~- }~~
78		-
79		~~- private function doRebuildTextIndex( &$database ) {~~
80		~~- list ($page, $revision, $text, $searchindex) = $database->tableNamesN( 'page', 'revision', 'text', 'searchindex' );~~
81		-
82		~~- $sql = "SELECT MAX(page_id) AS count FROM $page";~~
83		~~- $res = $database->query($sql, "rebuildTextIndex" );~~
84		~~- $s = $database->fetchObject($res);~~
	66	+ /**
	67	+ * Populates the search index with content from all pages
	68	+ */
	69	+ protected function populateSearchIndex() {
	70	+ $res = $this->db->select( 'page', 'MAX(page_id) AS count' );
	71	+ $s = $this->db->fetchObject($res);
85	72	$count = $s->count;
86	73	$this->output( "Rebuilding index fields for {$count} pages...\n" );
87	74	$n = 0;
—	—	@@ -88,22 +75,54 @@
89	76	while ( $n < $count ) {
90	77	$this->output( $n . "\n" );
91	78	$end = $n + self::RTI_CHUNK_SIZE - 1;
92		~~- $sql = "SELECT page_id, page_namespace, page_title, old_flags, old_text~~
93		~~- FROM $page, $revision, $text~~
94		~~- WHERE page_id BETWEEN $n AND $end~~
95		~~- AND page_latest=rev_id~~
96		~~- AND rev_text_id=old_id";~~
97		~~- $res = $database->query($sql, "rebuildTextIndex" );~~
	79	+
	80	+ $res = $this->db->select( array( 'page', 'revision', 'text' ),
	81	+ array( 'page_id', 'page_namespace', 'page_title', 'old_flags', 'old_text' ),
	82	+ array( "page_id BETWEEN $n AND $end", 'page_latest = rev_id', 'rev_text_id = old_id' ),
	83	+ __METHOD__
	84	+ );
98	85
99	86	foreach( $res as $s ) {
100	87	$revtext = Revision::getRevisionText( $s );
101	88	$u = new SearchUpdate( $s->page_id, $s->page_title, $revtext );
102	89	$u->doUpdate();
103	90	}
104		~~- $database->freeResult( $res );~~
	91	+ $this->db->freeResult( $res );
105	92	$n += self::RTI_CHUNK_SIZE;
106	93	}
107	94	}
	95	+
	96	+ /**
	97	+ * (MySQL only) Drops fulltext index before populating the table.
	98	+ */
	99	+ private function dropMysqlTextIndex() {
	100	+ $searchindex = $this->db->tableName( 'searchindex' );
	101	+ if ( $this->db->indexExists( 'searchindex', 'si_title' ) ) {
	102	+ $this->output( "Dropping index...\n" );
	103	+ $sql = "ALTER TABLE $searchindex DROP INDEX si_title, DROP INDEX si_text";
	104	+ $this->db->query($sql, __METHOD__ );
	105	+ }
	106	+ }
	107	+
	108	+ /**
	109	+ * (MySQL only) Adds back fulltext index after populating the table.
	110	+ */
	111	+ private function createMysqlTextIndex() {
	112	+ $searchindex = $this->db->tableName( 'searchindex' );
	113	+ $this->output( "\nRebuild the index...\n" );
	114	+ $sql = "ALTER TABLE $searchindex ADD FULLTEXT si_title (si_title), " .
	115	+ "ADD FULLTEXT si_text (si_text)";
	116	+ $this->db->query( $sql, __METHOD__ );
	117	+ }
	118	+
	119	+ /**
	120	+ * Deletes everything from search index.
	121	+ */
	122	+ private function clearSearchIndex() {
	123	+ $this->output( 'Clearing searchindex table...' );
	124	+ $this->db->delete( 'searchindex', '*', __METHOD__ );
	125	+ $this->output( "Done\n" );
	126	+ }
108	127	}
109	128
110	129	$maintClass = "RebuildTextIndex";
Index: trunk/phase3/includes/search/SearchSqlite.php
—	—	@@ -0,0 +1,383 @@
	2	+<?php
	3	+# SQLite search backend, based upon SearchMysql
	4	+#
	5	+# This program is free software; you can redistribute it and/or modify
	6	+# it under the terms of the GNU General Public License as published by
	7	+# the Free Software Foundation; either version 2 of the License, or
	8	+# (at your option) any later version.
	9	+#
	10	+# This program is distributed in the hope that it will be useful,
	11	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	+# GNU General Public License for more details.
	14	+#
	15	+# You should have received a copy of the GNU General Public License along
	16	+# with this program; if not, write to the Free Software Foundation, Inc.,
	17	+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
	18	+# http://www.gnu.org/copyleft/gpl.html
	19	+
	20	+/**
	21	+ * @file
	22	+ * @ingroup Search
	23	+ */
	24	+
	25	+/**
	26	+ * Search engine hook for SQLite
	27	+ * @ingroup Search
	28	+ */
	29	+class SearchSqlite extends SearchEngine {
	30	+ var $strictMatching = true;
	31	+
	32	+ // Cached because SearchUpdate keeps recreating our class
	33	+ private static $fulltextSupported = NULL;
	34	+
	35	+ /**
	36	+ * Creates an instance of this class
	37	+ * @param $db DatabaseSqlite: database object
	38	+ */
	39	+ function __construct( $db ) {
	40	+ $this->db = $db;
	41	+ }
	42	+
	43	+ /**
	44	+ * Whether fulltext search is supported by current schema
	45	+ * @return Boolean
	46	+ */
	47	+ function fulltextSearchSupported() {
	48	+ if ( self::$fulltextSupported === NULL ) {
	49	+ $res = $this->db->selectField( 'updatelog', 'ul_key', array( 'ul_key' => 'fts3' ), __METHOD__ );
	50	+ self::$fulltextSupported = $res && $this->db->numRows( $res ) > 0;
	51	+ }
	52	+ wfDebug( "***********************************************************" . self::$fulltextSupported . "**************\n" );
	53	+ return self::$fulltextSupported;
	54	+ }
	55	+
	56	+ /**
	57	+ * Parse the user's query and transform it into an SQL fragment which will
	58	+ * become part of a WHERE clause
	59	+ */
	60	+ function parseQuery( $filteredText, $fulltext ) {
	61	+ global $wgContLang;
	62	+ $lc = SearchEngine::legalSearchChars(); // Minus format chars
	63	+ $searchon = '';
	64	+ $this->searchTerms = array();
	65	+
	66	+ # FIXME: This doesn't handle parenthetical expressions.
	67	+ $m = array();
	68	+ if( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\?)\|"[^"]")/',
	69	+ $filteredText, $m, PREG_SET_ORDER ) ) {
	70	+ foreach( $m as $bits ) {
	71	+ @list( /* all */, $modifier, $term, $nonQuoted, $wildcard ) = $bits;
	72	+
	73	+ if( $nonQuoted != '' ) {
	74	+ $term = $nonQuoted;
	75	+ $quote = '';
	76	+ } else {
	77	+ $term = str_replace( '"', '', $term );
	78	+ $quote = '"';
	79	+ }
	80	+
	81	+ if( $searchon !== '' ) $searchon .= ' ';
	82	+ if( $this->strictMatching && ($modifier == '') ) {
	83	+ // If we leave this out, boolean op defaults to OR which is rarely helpful.
	84	+ $modifier = '+';
	85	+ }
	86	+
	87	+ // Some languages such as Serbian store the input form in the search index,
	88	+ // so we may need to search for matches in multiple writing system variants.
	89	+ $convertedVariants = $wgContLang->autoConvertToAllVariants( $term );
	90	+ if( is_array( $convertedVariants ) ) {
	91	+ $variants = array_unique( array_values( $convertedVariants ) );
	92	+ } else {
	93	+ $variants = array( $term );
	94	+ }
	95	+
	96	+ // The low-level search index does some processing on input to work
	97	+ // around problems with minimum lengths and encoding in MySQL's
	98	+ // fulltext engine.
	99	+ // For Chinese this also inserts spaces between adjacent Han characters.
	100	+ $strippedVariants = array_map(
	101	+ array( $wgContLang, 'stripForSearch' ),
	102	+ $variants );
	103	+
	104	+ // Some languages such as Chinese force all variants to a canonical
	105	+ // form when stripping to the low-level search index, so to be sure
	106	+ // let's check our variants list for unique items after stripping.
	107	+ $strippedVariants = array_unique( $strippedVariants );
	108	+
	109	+ $searchon .= $modifier;
	110	+ if( count( $strippedVariants) > 1 )
	111	+ $searchon .= '(';
	112	+ foreach( $strippedVariants as $stripped ) {
	113	+ if( $nonQuoted && strpos( $stripped, ' ' ) !== false ) {
	114	+ // Hack for Chinese: we need to toss in quotes for
	115	+ // multiple-character phrases since stripForSearch()
	116	+ // added spaces between them to make word breaks.
	117	+ $stripped = '"' . trim( $stripped ) . '"';
	118	+ }
	119	+ $searchon .= "$quote$stripped$quote$wildcard ";
	120	+ }
	121	+ if( count( $strippedVariants) > 1 )
	122	+ $searchon .= ')';
	123	+
	124	+ // Match individual terms or quoted phrase in result highlighting...
	125	+ // Note that variants will be introduced in a later stage for highlighting!
	126	+ $regexp = $this->regexTerm( $term, $wildcard );
	127	+ $this->searchTerms[] = $regexp;
	128	+ }
	129	+ wfDebug( __METHOD__ . ": Would search with '$searchon'\n" );
	130	+ wfDebug( __METHOD__ . ': Match with /' . implode( '\|', $this->searchTerms ) . "/\n" );
	131	+ } else {
	132	+ wfDebug( __METHOD__ . ": Can't understand search query '{$filteredText}'\n" );
	133	+ }
	134	+
	135	+ $searchon = $this->db->strencode( $searchon );
	136	+ $field = $this->getIndexField( $fulltext );
	137	+ return " $field MATCH '$searchon' ";
	138	+ }
	139	+
	140	+ function regexTerm( $string, $wildcard ) {
	141	+ global $wgContLang;
	142	+
	143	+ $regex = preg_quote( $string, '/' );
	144	+ if( $wgContLang->hasWordBreaks() ) {
	145	+ if( $wildcard ) {
	146	+ // Don't cut off the final bit!
	147	+ $regex = "\b$regex";
	148	+ } else {
	149	+ $regex = "\b$regex\b";
	150	+ }
	151	+ } else {
	152	+ // For Chinese, words may legitimately abut other words in the text literal.
	153	+ // Don't add \b boundary checks... note this could cause false positives
	154	+ // for latin chars.
	155	+ }
	156	+ return $regex;
	157	+ }
	158	+
	159	+ public static function legalSearchChars() {
	160	+ return "\"*" . parent::legalSearchChars();
	161	+ }
	162	+
	163	+ /**
	164	+ * Perform a full text search query and return a result set.
	165	+ *
	166	+ * @param $term String: raw search term
	167	+ * @return SqliteSearchResultSet
	168	+ */
	169	+ function searchText( $term ) {
	170	+ return $this->searchInternal( $term, true );
	171	+ }
	172	+
	173	+ /**
	174	+ * Perform a title-only search query and return a result set.
	175	+ *
	176	+ * @param $term String: raw search term
	177	+ * @return SqliteSearchResultSet
	178	+ */
	179	+ function searchTitle( $term ) {
	180	+ return $this->searchInternal( $term, false );
	181	+ }
	182	+
	183	+ protected function searchInternal( $term, $fulltext ) {
	184	+ global $wgSearchMySQLTotalHits;
	185	+
	186	+ if ( !$this->fulltextSearchSupported() ) {
	187	+ return null;
	188	+ }
	189	+
	190	+ $filteredTerm = $this->filter( $term );
	191	+ $resultSet = $this->db->query( $this->getQuery( $filteredTerm, $fulltext ) );
	192	+
	193	+ $total = null;
	194	+ if( $wgSearchMySQLTotalHits ) {
	195	+ $totalResult = $this->db->query( $this->getCountQuery( $filteredTerm, $fulltext ) );
	196	+ $row = $totalResult->fetchObject();
	197	+ if( $row ) {
	198	+ $total = intval( $row->c );
	199	+ }
	200	+ $totalResult->free();
	201	+ }
	202	+
	203	+ return new SqliteSearchResultSet( $resultSet, $this->searchTerms, $total );
	204	+ }
	205	+
	206	+
	207	+ /**
	208	+ * Return a partial WHERE clause to exclude redirects, if so set
	209	+ * @return String
	210	+ */
	211	+ function queryRedirect() {
	212	+ if( $this->showRedirects ) {
	213	+ return '';
	214	+ } else {
	215	+ return 'AND page_is_redirect=0';
	216	+ }
	217	+ }
	218	+
	219	+ /**
	220	+ * Return a partial WHERE clause to limit the search to the given namespaces
	221	+ * @return String
	222	+ */
	223	+ function queryNamespaces() {
	224	+ if( is_null($this->namespaces) )
	225	+ return ''; # search all
	226	+ if ( !count( $this->namespaces ) ) {
	227	+ $namespaces = '0';
	228	+ } else {
	229	+ $namespaces = $this->db->makeList( $this->namespaces );
	230	+ }
	231	+ return 'AND page_namespace IN (' . $namespaces . ')';
	232	+ }
	233	+
	234	+ /**
	235	+ * Returns a query with limit for number of results set.
	236	+ * @param $sql String:
	237	+ * @return String
	238	+ */
	239	+ function limitResult( $sql ) {
	240	+ return $this->db->limitResult( $sql, $this->limit, $this->offset );
	241	+ }
	242	+
	243	+ /**
	244	+ * Does not do anything for generic search engine
	245	+ * subclasses may define this though
	246	+ * @return String
	247	+ */
	248	+ function queryRanking( $filteredTerm, $fulltext ) {
	249	+ return '';
	250	+ }
	251	+
	252	+ /**
	253	+ * Construct the full SQL query to do the search.
	254	+ * The guts shoulds be constructed in queryMain()
	255	+ * @param $filteredTerm String
	256	+ * @param $fulltext Boolean
	257	+ */
	258	+ function getQuery( $filteredTerm, $fulltext ) {
	259	+ return $this->limitResult(
	260	+ $this->queryMain( $filteredTerm, $fulltext ) . ' ' .
	261	+ $this->queryRedirect() . ' ' .
	262	+ $this->queryNamespaces() . ' ' .
	263	+ $this->queryRanking( $filteredTerm, $fulltext )
	264	+ );
	265	+ }
	266	+
	267	+ /**
	268	+ * Picks which field to index on, depending on what type of query.
	269	+ * @param $fulltext Boolean
	270	+ * @return String
	271	+ */
	272	+ function getIndexField( $fulltext ) {
	273	+ return $fulltext ? 'si_text' : 'si_title';
	274	+ }
	275	+
	276	+ /**
	277	+ * Get the base part of the search query.
	278	+ *
	279	+ * @param $filteredTerm String
	280	+ * @param $fulltext Boolean
	281	+ * @return String
	282	+ */
	283	+ function queryMain( $filteredTerm, $fulltext ) {
	284	+ $match = $this->parseQuery( $filteredTerm, $fulltext );
	285	+ $page = $this->db->tableName( 'page' );
	286	+ $searchindex = $this->db->tableName( 'searchindex' );
	287	+ return "SELECT $searchindex.rowid, page_namespace, page_title " .
	288	+ "FROM $page,$searchindex " .
	289	+ "WHERE page_id=$searchindex.rowid AND $match";
	290	+ }
	291	+
	292	+ function getCountQuery( $filteredTerm, $fulltext ) {
	293	+ $match = $this->parseQuery( $filteredTerm, $fulltext );
	294	+ $page = $this->db->tableName( 'page' );
	295	+ $searchindex = $this->db->tableName( 'searchindex' );
	296	+ return "SELECT COUNT(*) AS c " .
	297	+ "FROM $page,$searchindex " .
	298	+ "WHERE page_id=$searchindex.rowid AND $match" .
	299	+ $this->queryRedirect() . ' ' .
	300	+ $this->queryNamespaces();
	301	+ }
	302	+
	303	+ /**
	304	+ * Create or update the search index record for the given page.
	305	+ * Title and text should be pre-processed.
	306	+ *
	307	+ * @param $id Integer
	308	+ * @param $title String
	309	+ * @param $text String
	310	+ */
	311	+ function update( $id, $title, $text ) {
	312	+ if ( !$this->fulltextSearchSupported() ) {
	313	+ return;
	314	+ }
	315	+ // @todo: find a method to do it in a single request,
	316	+ // couldn't do it so far due to typelessness of FTS3 tables.
	317	+ $dbw = wfGetDB( DB_MASTER );
	318	+
	319	+ $dbw->delete( 'searchindex', array( 'rowid' => $id ), __METHOD__ );
	320	+
	321	+ $dbw->insert( 'searchindex',
	322	+ array(
	323	+ 'rowid' => $id,
	324	+ 'si_title' => $title,
	325	+ 'si_text' => $text
	326	+ ), __METHOD__ );
	327	+ }
	328	+
	329	+ /**
	330	+ * Update a search index record's title only.
	331	+ * Title should be pre-processed.
	332	+ *
	333	+ * @param $id Integer
	334	+ * @param $title String
	335	+ */
	336	+ function updateTitle( $id, $title ) {
	337	+ if ( !$this->fulltextSearchSupported() ) {
	338	+ return;
	339	+ }
	340	+ $dbw = wfGetDB( DB_MASTER );
	341	+
	342	+ $dbw->update( 'searchindex',
	343	+ array( 'rowid' => $id ),
	344	+ array( 'si_title' => $title ),
	345	+ __METHOD__ );
	346	+ }
	347	+}
	348	+
	349	+/**
	350	+ * @ingroup Search
	351	+ */
	352	+class SqliteSearchResultSet extends SearchResultSet {
	353	+ function SqliteSearchResultSet( $resultSet, $terms, $totalHits=null ) {
	354	+ $this->mResultSet = $resultSet;
	355	+ $this->mTerms = $terms;
	356	+ $this->mTotalHits = $totalHits;
	357	+ }
	358	+
	359	+ function termMatches() {
	360	+ return $this->mTerms;
	361	+ }
	362	+
	363	+ function numRows() {
	364	+ return $this->mResultSet->numRows();
	365	+ }
	366	+
	367	+ function next() {
	368	+ $row = $this->mResultSet->fetchObject();
	369	+ if( $row === false ) {
	370	+ return false;
	371	+ } else {
	372	+ return new SearchResult( $row );
	373	+ }
	374	+ }
	375	+
	376	+ function free() {
	377	+ $this->mResultSet->free();
	378	+ }
	379	+
	380	+
	381	+ function getTotalHits() {
	382	+ return $this->mTotalHits;
	383	+ }
	384	+}
\ No newline at end of file
Property changes on: trunk/phase3/includes/search/SearchSqlite.php
___________________________________________________________________
Name: svn:eol-style
1	385	+ native
Index: trunk/phase3/includes/db/DatabaseSqlite.php
—	—	@@ -97,6 +97,20 @@
98	98	}
99	99
100	100	/**
	101	+ * Returns version of currently supported SQLite fulltext search module or false if none present.
	102	+ * @return String
	103	+ */
	104	+ function getFulltextSearchModule() {
	105	+ $table = 'dummy_search_test';
	106	+ $this->query( "DROP TABLE IF EXISTS $table", __METHOD__ );
	107	+ if ( $this->query( "CREATE VIRTUAL TABLE $table USING FTS3(dummy_field)", __METHOD__, true ) ) {
	108	+ $this->query( "DROP TABLE IF EXISTS $table", __METHOD__ );
	109	+ return 'FTS3';
	110	+ }
	111	+ return false;
	112	+ }
	113	+
	114	+ /**
101	115	* SQLite doesn't allow buffered results or data seeking etc, so we'll use fetchAll as the result
102	116	*/
103	117	function doQuery( $sql ) {
—	—	@@ -319,15 +333,15 @@
320	334	}
321	335
322	336	function wasDeadlock() {
323		~~- return $this->lastErrno() == SQLITE_BUSY;~~
	337	+ return $this->lastErrno() == 5; // SQLITE_BUSY
324	338	}
325	339
326	340	function wasErrorReissuable() {
327		~~- return $this->lastErrno() == SQLITE_SCHEMA;~~
	341	+ return $this->lastErrno() == 17; // SQLITE_SCHEMA;
328	342	}
329	343
330	344	function wasReadOnlyError() {
331		~~- return $this->lastErrno() == SQLITE_READONLY;~~
	345	+ return $this->lastErrno() == 8; // SQLITE_READONLY;
332	346	}
333	347
334	348	/**
—	—	@@ -460,7 +474,7 @@
461	475	}
462	476
463	477	public function getSearchEngine() {
464		~~- return "SearchEngineDummy";~~
	478	+ return "SearchSqlite";
465	479	}
466	480
467	481	/**
Index: trunk/phase3/includes/AutoLoader.php
—	—	@@ -483,8 +483,10 @@
484	484	'SearchResult' => 'includes/search/SearchEngine.php',
485	485	'SearchResultSet' => 'includes/search/SearchEngine.php',
486	486	'SearchResultTooMany' => 'includes/search/SearchEngine.php',
	487	+ 'SearchSqlite' => 'includes/search/SearchSqlite.php',
487	488	'SearchUpdate' => 'includes/search/SearchUpdate.php',
488	489	'SearchUpdateMyISAM' => 'includes/search/SearchUpdate.php',
	490	+ 'SqliteSearchResultSet' => 'includes/search/SearchSqlite.php',
489	491
490	492	# includes/specials
491	493	'SpecialAllmessages' => 'includes/specials/SpecialAllmessages.php',
Property changes on: trunk/phase3/includes
___________________________________________________________________
Name: svn:mergeinfo
492	494	- /branches/REL1_15/phase3/includes:51646
/branches/wmf-deployment/includes:53381
493	495	+ /branches/REL1_15/phase3/includes:51646
/branches/sqlite/includes:58211-58321
/branches/wmf-deployment/includes:53381
Index: trunk/phase3/config/Installer.php
—	—	@@ -1268,7 +1268,7 @@
1269	1269	print " done.</li>\n";
1270	1270
1271	1271
1272		~~- if ($conf->DBtype == 'ibm_db2') {~~
	1272	+ if ( $conf->DBtype == 'ibm_db2' ) {
1273	1273	// Now that table creation is done, make sure everything is committed
1274	1274	// Do this before doing inserts through API
1275	1275	if ($wgDatabase->lastError()) {
—	—	@@ -1279,6 +1279,10 @@
1280	1280	print "<li>MediaWiki tables successfully created</li>\n";
1281	1281	$wgDatabase->commit();
1282	1282	}
	1283	+ } elseif ( $conf->DBtype == 'sqlite' ) {
	1284	+ // Ensure proper searchindex format. We have to do that separately because
	1285	+ // if SQLite is compiled without the FTS3 module, table creation syntax will be invalid.
	1286	+ sqlite_setup_searchindex();
1283	1287	}
1284	1288
1285	1289	print "<li>Initializing statistics...</li>\n";
Property changes on: trunk/phase3
___________________________________________________________________
Name: svn:mergeinfo
1286	1290	- /branches/REL1_15/phase3:51646
1287	1291	+ /branches/REL1_15/phase3:51646
/branches/sqlite:58211-58321

Follow-up revisions

Revision	Commit summary	Author	Date
r58327	Release notes for r58322	maxsem	17:52, 29 October 2009
r58360	Follow-up to r58322: removed debugging code	maxsem	13:31, 30 October 2009
r60739	Fixed insane selectField() usage from r58322	maxsem	18:52, 6 January 2010
r60860	Severely refactored the SearchResultSet descendants system, consolidating mos...	maxsem	14:30, 9 January 2010
r60884	Cleared SearchSqlite from some mysqlisms from r58322	maxsem	10:02, 10 January 2010

Comments

#Comment by Tim Starling (talk | contribs) 01:53, 5 January 2010

$res = $this->db->selectField( 'updatelog', 'ul_key', array( 'ul_key' => 'fts3' ), __METHOD__ );
self::$fulltextSupported = $res && $this->db->numRows( $res ) > 0;

selectField() returns the field value "fts3", not a result object. This only works because DatabaseSqlite::numRows() will return 1 whenever you pass it a scalar.

Obviously there's lots of code here which is duplicated from DatabaseMysql, and needs to be factored out.

I think getFulltextSearchModule() will throw an exception if the module is missing, due to a query error, I can't see how it could work as expected. That's what happened when I ran the query from eval.php with a bogus module name.

#Comment by Tim Starling (talk | contribs) 03:42, 5 January 2010

"Obviously there's lots of code here which is duplicated from DatabaseMysql": should have said SearchMysql not DatabaseMysql

#Comment by MaxSem (talk | contribs) 07:34, 5 January 2010

I think getFulltextSearchModule() will throw an exception if the module is missing

It doesn't, because I specifically instruct query() not to throw exceptions. I've re-tried it right now, by changing FTS3 to FTS666 - no exceptions, both from from eval and updater.

Status & tagging log

07:21, 20 January 2010 Tim Starling (talk | contribs) changed the status of r58322 [removed: fixme added: resolved]
01:53, 5 January 2010 Tim Starling (talk | contribs) changed the status of r58322 [removed: new added: fixme]