r49218 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r49217‎ \| r49218 \| r49219 >
Date:	17:11, 5 April 2009
Author:	vasilievvv
Status:	ok
Tags:
Comment:	Introduce list (non-associated array) support into abuse filter parser.
Modified paths:	/trunk/extensions/AbuseFilter/AbuseFilter.i18n.php (modified) (history) /trunk/extensions/AbuseFilter/AbuseFilter.parser.php (modified) (history) /trunk/extensions/AbuseFilter/tests/arrays.r (added) (history) /trunk/extensions/AbuseFilter/tests/arrays.t (added) (history) /trunk/extensions/AbuseFilter/tests/wptest1.t (modified) (history) /trunk/extensions/AbuseFilter/tests/wptest2.t (modified) (history) /trunk/extensions/AbuseFilter/tests/wptest3.t (modified) (history)

Diff [purge]

Index: trunk/extensions/AbuseFilter/AbuseFilter.parser.php
—	—	@@ -14,8 +14,13 @@
15	15	* T_STRING - string, in "" or ''
16	16	* T_KEYWORD - keyword
17	17	* T_ID - identifier
	18	+* T_STATEMENT_SEPARATOR - ;
	19	+* T_SQUARE_BRACKETS - [ or ]
18	20
19	21	Levels of parsing:
	22	+* Entry - catches unexpected characters
	23	+* Semicolon - ;
	24	+* Set - :=
20	25	* Conditionls (IF) - if-then-else-end, cond ? a :b
21	26	* BoolOps (BO) - &, \|, ^
22	27	* CompOps (CO) - ==, !=, ===, !==, >, <, >=, <=
—	—	@@ -25,6 +30,7 @@
26	31	* BoolNeg (BN) - ! operation
27	32	* SpecialOperators (SO) - in and like
28	33	* Unarys (U) - plus and minus in cases like -5 or -(2 * +2)
	34	+* ListElement (LE) - list[number]
29	35	* Braces (B) - ( and )
30	36	* Functions (F)
31	37	* Atom (A) - return value
—	—	@@ -40,6 +46,7 @@
41	47	const TFloat = 'T_FLOAT';
42	48	const TOp = 'T_OP';
43	49	const TBrace = 'T_BRACE';
	50	+ const TSquareBracket = 'T_SQUARE_BRACKET';
44	51	const TComma = 'T_COMMA';
45	52	const TStatementSeparator = 'T_STATEMENT_SEPARATOR';
46	53
—	—	@@ -61,6 +68,7 @@
62	69	const DNull = 'null';
63	70	const DBool = 'bool';
64	71	const DFloat = 'float';
	72	+ const DList = 'list';
65	73
66	74	var $type;
67	75	var $data;
—	—	@@ -79,6 +87,12 @@
80	88	return new AFPData( self::DFloat, $var );
81	89	elseif( is_bool( $var ) )
82	90	return new AFPData( self::DBool, $var );
	91	+ elseif( is_array( $var ) ) {
	92	+ $result = array();
	93	+ foreach( $var as $item )
	94	+ $result[] = self::newFromPHPVar( $item );
	95	+ return new AFPData( self::DList, $result );
	96	+ }
83	97	elseif( is_null( $var ) )
84	98	return new AFPData();
85	99	else
—	—	@@ -96,6 +110,24 @@
97	111	if( $target == self::DNull ) {
98	112	return new AFPData();
99	113	}
	114	+
	115	+ if( $orig->type == self::DList ) {
	116	+ if( $target == self::DBool )
	117	+ return new AFPData( self::DBool, (bool)count( $orig->data ) );
	118	+ if( $target == self::DFloat ) {
	119	+ return new AFPData( self::DFloat, doubleval( count( $orig->data ) ) );
	120	+ }
	121	+ if( $target == self::DInt ) {
	122	+ return new AFPData( self::DInt, intval( count( $orig->data ) ) );
	123	+ }
	124	+ if( $target == self::DString ) {
	125	+ $lines = array();
	126	+ foreach( $orig->data as $item )
	127	+ $lines[] = $item->toString();
	128	+ return new AFPData( self::DString, implode( "\n", $lines ) );
	129	+ }
	130	+ }
	131	+
100	132	if( $target == self::DBool ) {
101	133	return new AFPData( self::DBool, (bool)$orig->data );
102	134	}
—	—	@@ -108,6 +140,9 @@
109	141	if( $target == self::DString ) {
110	142	return new AFPData( self::DString, strval( $orig->data ) );
111	143	}
	144	+ if( $target == self::DList ) {
	145	+ return new AFPData( self::DList, array( $orig ) );
	146	+ }
112	147	}
113	148
114	149	public static function boolInvert( $value ) {
—	—	@@ -119,6 +154,8 @@
120	155	}
121	156
122	157	public static function keywordIn( $a, $b ) {
	158	+ if( $b->type == self::DList )
	159	+ return new AFPData( self::DBool, self::listContains( $a, $b ) );
123	160	$a = $a->toString();
124	161	$b = $b->toString();
125	162
—	—	@@ -130,6 +167,8 @@
131	168	}
132	169
133	170	public static function keywordContains( $a, $b ) {
	171	+ if( $a->type == self::DList )
	172	+ return new AFPData( self::DBool, self::listContains( $b, $a ) );
134	173	$a = $a->toString();
135	174	$b = $b->toString();
136	175
—	—	@@ -140,6 +179,20 @@
141	180	return new AFPData( self::DBool, in_string( $b, $a ) );
142	181	}
143	182
	183	+ public static function listContains( $value, $list ) {
	184	+ // Should use built-in PHP function somehow
	185	+ foreach( $list->data as $item ) {
	186	+ if( self::equals( $value, $item ) )
	187	+ return true;
	188	+ }
	189	+ return false;
	190	+ }
	191	+
	192	+ public static function equals( $d1, $d2 ) {
	193	+ return $d1->type != self::DList && $d2->type != self::DList &&
	194	+ $d1->toString() === $d2->toString();
	195	+ }
	196	+
144	197	public static function keywordLike( $str, $pattern ) {
145	198	$str = $str->toString();
146	199	$pattern = $pattern->toString();
—	—	@@ -184,13 +237,13 @@
185	238
186	239	public static function compareOp( $a, $b, $op ) {
187	240	if( $op == '==' \|\| $op == '=' )
188		~~- return new AFPData( self::DBool, $a->toString() === $b->toString() );~~
	241	+ return new AFPData( self::DBool, self::equals( $a, $b ) );
189	242	if( $op == '!=' )
190		~~- return new AFPData( self::DBool, $a->toString() !== $b->toString() );~~
	243	+ return new AFPData( self::DBool, !self::equals( $a, $b ) );
191	244	if( $op == '===' )
192		~~- return new AFPData( self::DBool, $a->data == $b->data && $a->type == $b->type );~~
	245	+ return new AFPData( self::DBool, $a->type == $b->type && self::equals( $a, $b ) );
193	246	if( $op == '!==' )
194		~~- return new AFPData( self::DBool, $a->data !== $b->data \|\| $a->type != $b->type );~~
	247	+ return new AFPData( self::DBool, $a->type != $b->type \|\| !self::equals( $a, $b ) );
195	248	$a = $a->toString();
196	249	$b = $b->toString();
197	250	if( $op == '>' )
—	—	@@ -241,7 +294,9 @@
242	295
243	296	public static function sum( $a, $b ) {
244	297	if( $a->type == self::DString \|\| $b->type == self::DString )
245		~~- return new AFPData( self::DFloat, $a->toString() . $b->toString() );~~
	298	+ return new AFPData( self::DString, $a->toString() . $b->toString() );
	299	+ elseif( $a->type == self::DList && $b->type == self::DList )
	300	+ return new AFPData( self::DList, array_merge( $a->toList(), $b->toList() ) );
246	301	else
247	302	return new AFPData( self::DFloat, $a->toFloat() + $b->toFloat() );
248	303	}
—	—	@@ -266,6 +321,10 @@
267	322	public function toInt() {
268	323	return self::castTypes( $this, self::DInt )->data;
269	324	}
	325	+
	326	+ public function toList() {
	327	+ return self::castTypes( $this, self::DList )->data;
	328	+ }
270	329	}
271	330
272	331	class AFPParserState {
—	—	@@ -497,6 +556,45 @@
498	557	$this->doLevelSet( $result );
499	558	$this->setUserVariable( $varname, $result );
500	559	return;
	560	+ } elseif( $this->mCur->type == AFPToken::TSquareBracket && $this->mCur->value == '[' ) {
	561	+ if( !$this->mVars->varIsSet( $varname ) ) {
	562	+ throw new AFPUserVisibleException( 'unrecognisedvar',
	563	+ $this->mCur->pos,
	564	+ array( $var ) );
	565	+ }
	566	+ $list = $this->mVars->getVar( $varname );
	567	+ if( $list->type != AFPData::DList )
	568	+ throw new AFPUserVisibleException( 'notlist', $this->mCur->pos, array() );
	569	+ $list = $list->toList();
	570	+ $this->move();
	571	+ if( $this->mCur->type == AFPToken::TSquareBracket && $this->mCur->value == ']' ) {
	572	+ $idx = 'new';
	573	+ } else {
	574	+ $this->setState( $prev ); $this->move();
	575	+ $idx = new AFPData();
	576	+ $this->doLevelSemicolon( $idx );
	577	+ $idx = $idx->toInt();
	578	+ if( !($this->mCur->type == AFPToken::TSquareBracket && $this->mCur->value == ']') )
	579	+ throw new AFPUserVisibleException( 'expectednotfound', $this->mCur->pos,
	580	+ array(']', $this->mCur->type, $this->mCur->value ) );
	581	+ if( count( $list ) <= $idx ) {
	582	+ throw new AFPUserVisibleException( 'outofbounds', $this->mCur->pos,
	583	+ array( $idx, count( $result->data ) ) );
	584	+ }
	585	+ }
	586	+ $this->move();
	587	+ if( $this->mCur->type == AFPToken::TOp && $this->mCur->value == ':=' ) {
	588	+ $this->move();
	589	+ $this->doLevelSet( $result );
	590	+ if( $idx == 'new' )
	591	+ $list[] = $result;
	592	+ else
	593	+ $list[$idx] = $result;
	594	+ $this->setUserVariable( $varname, new AFPData( AFPData::DList, $list ) );
	595	+ return;
	596	+ } else {
	597	+ $this->setState( $prev );
	598	+ }
501	599	} else {
502	600	$this->setState( $prev );
503	601	}
—	—	@@ -734,17 +832,40 @@
735	833	$op = $this->mCur->value;
736	834	if( $this->mCur->type == AFPToken::TOp && ( $op == "+" \|\| $op == "-" ) ) {
737	835	$this->move();
738		~~- $this->doLevelBraces( $result );~~
	836	+ $this->doLevelListElements( $result );
739	837	wfProfileIn( __METHOD__ );
740	838	if( $op == '-' ) {
741	839	$result = AFPData::unaryMinus( $result );
742	840	}
743	841	wfProfileOut( __METHOD__ );
744	842	} else {
745		~~- $this->doLevelBraces( $result );~~
	843	+ $this->doLevelListElements( $result );
746	844	}
747	845	}
748		-
	846	+
	847	+ protected function doLevelListElements( &$result ) {
	848	+ $this->doLevelBraces( $result );
	849	+ while( $this->mCur->type == AFPToken::TSquareBracket && $this->mCur->value == '[' ) {
	850	+ $idx = new AFPData();
	851	+ $this->doLevelSemicolon( $idx );
	852	+ if( !($this->mCur->type == AFPToken::TSquareBracket && $this->mCur->value == ']') ) {
	853	+ throw new AFPUserVisibleException( 'expectednotfound', $this->mCur->pos,
	854	+ array(']', $this->mCur->type, $this->mCur->value ) );
	855	+ }
	856	+ $idx = $idx->toInt();
	857	+ if( $result->type == AFPData::DList ) {
	858	+ if( count( $result->data ) <= $idx ) {
	859	+ throw new AFPUserVisibleException( 'outofbounds', $this->mCur->pos,
	860	+ array( $idx, count( $result->data ) ) );
	861	+ }
	862	+ $result = $result->data[$idx];
	863	+ } else {
	864	+ throw new AFPUserVisibleException( 'notlist', $this->mCur->pos, array() );
	865	+ }
	866	+ $this->move();
	867	+ }
	868	+ }
	869	+
749	870	protected function doLevelBraces( &$result ) {
750	871	if( $this->mCur->type == AFPToken::TBrace && $this->mCur->value == '(' ) {
751	872	if( $this->mShortCircuit ) {
—	—	@@ -850,11 +971,31 @@
851	972	$this->mCur->pos,
852	973	array($tok) );
853	974	break;
854		~~- case AFPToken::TBrace:~~
855		~~- if( $this->mCur->value == ')' )~~
856		~~- return; // Handled at the entry level~~
857	975	case AFPToken::TNone:
858	976	return; // Handled at entry level
	977	+ case AFPToken::TBrace:
	978	+ if( $this->mCur->value == ')' )
	979	+ return; // Handled at the entry level
	980	+ case AFPToken::TSquareBracket:
	981	+ if( $this->mCur->value == '[' ) {
	982	+ $list = array();
	983	+ for(;;) {
	984	+ $this->move();
	985	+ if( $this->mCur->type == AFPToken::TSquareBracket && $this->mCur->value == ']' )
	986	+ break;
	987	+ $item = new AFPData();
	988	+ $this->doLevelSet( $item );
	989	+ $list[] = $item;
	990	+ if( $this->mCur->type == AFPToken::TSquareBracket && $this->mCur->value == ']' )
	991	+ break;
	992	+ if( $this->mCur->type != AFPToken::TComma )
	993	+ throw new AFPUserVisibleException( 'expectednotfound',
	994	+ $this->mCur->pos,
	995	+ array(', or ]', $this->mCur->type, $this->mCur->value ) );
	996	+ }
	997	+ $result = new AFPData( AFPData::DList, $list );
	998	+ break;
	999	+ }
859	1000	default:
860	1001	throw new AFPUserVisibleException( 'unexpectedtoken',
861	1002	$this->mCur->pos,
—	—	@@ -930,6 +1071,11 @@
931	1072	return array( $code[$offset], AFPToken::TBrace, $code, $offset + 1 );
932	1073	}
933	1074
	1075	+ // Square brackets
	1076	+ if( $code[$offset] == '[' or $code[$offset] == ']' ) {
	1077	+ return array( $code[$offset], AFPToken::TSquareBracket, $code, $offset + 1 );
	1078	+ }
	1079	+
934	1080	// Semicolons
935	1081	if ($code[$offset] == ';') {
936	1082	return array( ';', AFPToken::TStatementSeparator, $code, $offset + 1 );
—	—	@@ -1112,6 +1258,9 @@
1113	1259	if( count( $args ) < 1 )
1114	1260	throw new AFPUserVisibleException( 'notenoughargs', $this->mCur->pos,
1115	1261	array( 'len', 2, count($args) ) );
	1262	+ if( $args[0]->type == AFPData::DList ) {
	1263	+ return new AFPData( AFPData::DInt, count( $args[0]->data ) );
	1264	+ }
1116	1265	$s = $args[0]->toString();
1117	1266	return new AFPData( AFPData::DInt, mb_strlen( $s, 'utf-8' ) );
1118	1267	}
—	—	@@ -1148,9 +1297,21 @@
1149	1298	if( count( $args ) < 1 )
1150	1299	throw new AFPUserVisibleException( 'notenoughargs', $this->mCur->pos,
1151	1300	array( 'count', 1, count($args) ) );
1152		-
	1301	+
	1302	+ if( $args[0]->type == AFPData::DList && count( $args ) == 1 ) {
	1303	+ return new AFPData( AFPData::DInt, count( $args[0]->data ) );
	1304	+ } elseif( count( $args ) > 1 && $args[1]->type == AFPData::DList ) {
	1305	+ $needle = $args[0];
	1306	+ $haystack = $args[1]->toList();
	1307	+ $count = 0;
	1308	+ foreach( $haystack as $item )
	1309	+ if( AFPData::equals( $needle, $item ))
	1310	+ $count++;
	1311	+ return new AFPData( AFPData::DInt, $count );
	1312	+ }
	1313	+
1153	1314	$offset = -1;
1154		-
	1315	+
1155	1316	if (count($args) == 1) {
1156	1317	$count = count( explode( ",", $args[0]->toString() ) );
1157	1318	} else {
—	—	@@ -1224,7 +1385,15 @@
1225	1386	throw new AFPUserVisibleException( 'notenoughargs', $this->mCur->pos,
1226	1387	array( 'contains_any', 2, count($args) ) );
1227	1388	}
1228		-
	1389	+
	1390	+ if( $args[0]->type == AFPData::DList ) {
	1391	+ $list = array_shift( $args );
	1392	+ foreach( $args as $arg )
	1393	+ if( AFPData::listContains( $arg, $list ) )
	1394	+ return new AFPData( AFPData::DBool, true );
	1395	+ return new AFPData( AFPData::DBool, false );
	1396	+ }
	1397	+
1229	1398	$s = array_shift( $args );
1230	1399	$s = $s->toString();
1231	1400
—	—	@@ -1402,7 +1571,7 @@
1403	1572	throw new AFPUserVisibleException( 'noparams', $this->mCur->pos, array(__METHOD__) );
1404	1573	$val = $args[0];
1405	1574
1406		~~- return new AFPData( AFPData::DString, $val->data );~~
	1575	+ return AFPData::castTypes( $val, AFPData::DString );
1407	1576	}
1408	1577
1409	1578	protected function castInt( $args ) {
—	—	@@ -1410,7 +1579,7 @@
1411	1580	throw new AFPUserVisibleException( 'noparams', $this->mCur->pos, array(__METHOD__) );
1412	1581	$val = $args[0];
1413	1582
1414		~~- return new AFPData( AFPData::DInt, intval($val->data) );~~
	1583	+ return AFPData::castTypes( $val, AFPData::DInt );
1415	1584	}
1416	1585
1417	1586	protected function castFloat( $args ) {
—	—	@@ -1418,7 +1587,7 @@
1419	1588	throw new AFPUserVisibleException( 'noparams', $this->mCur->pos, array(__METHOD__) );
1420	1589	$val = $args[0];
1421	1590
1422		~~- return new AFPData( AFPData::DFloat, doubleval($val->data) );~~
	1591	+ return AFPData::castTypes( $val, AFPData::DFloat );
1423	1592	}
1424	1593
1425	1594	protected function castBool( $args ) {
—	—	@@ -1426,7 +1595,7 @@
1427	1596	throw new AFPUserVisibleException( 'noparams', $this->mCur->pos, array(__METHOD__) );
1428	1597	$val = $args[0];
1429	1598
1430		~~- return new AFPData( AFPData::DBool, (bool)($val->data) );~~
	1599	+ return AFPData::castTypes( $val, AFPData::DBool );
1431	1600	}
1432	1601
1433	1602	public static function regexErrorHandler( $errno, $errstr, $errfile, $errline, $context ) {
Index: trunk/extensions/AbuseFilter/tests/wptest1.t
—	—	@@ -1,5 +1,5 @@
2	2	/* Filter 30 from English Wikipedia (large deletion from article by new editors) */
3		~~-user_groups_test := "*";~~
	3	+user_groups_test := ["*"];
4	4	new_size_test := 100;
5	5	article_namespace_test := 0;
6	6	edit_delta_test := -5000;
Index: trunk/extensions/AbuseFilter/tests/wptest2.t
—	—	@@ -1,5 +1,5 @@
2	2	/* Filter 61 from English Wikipedia (new user removing references) */
3		~~-user_groups_test := "*";~~
	3	+user_groups_test := ["*"];
4	4	new_size_test := 100;
5	5	article_namespace_test := 0;
6	6	edit_delta_test := -22;
Index: trunk/extensions/AbuseFilter/tests/wptest3.t
—	—	@@ -1,5 +1,5 @@
2	2	/* Filter 18 from English Wikipedia (test type edits from clicking on edit bar) */
3		~~-user_groups_test := "*";~~
	3	+user_groups_test := ["*"];
4	4	article_namespace_test := 0;
5	5	added_lines_test := "Hello world! '''Bold text''' [http://www.example.com link title]";
6	6
Index: trunk/extensions/AbuseFilter/tests/arrays.r
—	—	@@ -0,0 +1 @@
	2	+MATCH
Index: trunk/extensions/AbuseFilter/tests/arrays.t
—	—	@@ -0,0 +1,12 @@
	2	+array1 := [ 'a', 'b', 'c', ];
	3	+array2 := [];
	4	+array2[] := 'd';
	5	+array2[] := 'g';
	6	+array2[] := 'f';
	7	+array2[1] := 'e';
	8	+
	9	+array3 := array1 + array2;
	10	+array4 := [ [ 1, 2, 3 ], [ 4, 5, 6 ] ];
	11	+
	12	+(string(array3) == "a\nb\nc\nd\ne\nf" & !('b' in array2) & array1 contains 'c' & [ false, !(1;0), null ][1] & length(array3) == 6 &
	13	+ array4[1][1] == 5 )
\ No newline at end of file
Index: trunk/extensions/AbuseFilter/AbuseFilter.i18n.php
—	—	@@ -338,6 +338,8 @@
339	339	Expected $3 {{PLURAL:$3\|argument\|arguments}}, got $4',
340	340	'abusefilter-exception-regexfailure' => 'Error in regular expression "$3" at character $1: "$2"',
341	341	'abusefilter-exception-overridebuiltin' => 'Illegal overriding of built-in variable "$2" at character $1.',
	342	+ 'abusefilter-exception-outofbounds' => 'Requesting non-existent list item $2 (list size = 3) at character $1.',
	343	+ 'abusefilter-exception-notlist' => 'Requesting array item of non-array at character $1.',
342	344
343	345	// Actions
344	346	'abusefilter-action-throttle' => 'Throttle',

Status & tagging log

16:06, 3 June 2009 Tim Starling (talk | contribs) changed the status of r49218 [removed: new added: ok]