r39086 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r39085‎ | r39086 | r39087 >
Date:20:38, 10 August 2008
Author:river
Status:old
Tags:
Comment:
- using leaf_node_t includes the skip parser in the parsed token, causing expressions like (1 == ONE) to fail. use reduced_node_t instead
- add regression test framework and some basic tests
Modified paths:
  • /trunk/extensions/AbuseFilter/parser_native/expr.cpp (modified) (history)
  • /trunk/extensions/AbuseFilter/parser_native/makefile (modified) (history)
  • /trunk/extensions/AbuseFilter/parser_native/maketest.cpp (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/parser.h (modified) (history)
  • /trunk/extensions/AbuseFilter/parser_native/runtests.sh (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/arith.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/arith.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/eq.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/eq.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/float.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/float.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/in.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/in.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/lcase.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/lcase.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/length.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/length.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/like.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/like.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/norm.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/norm.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/numbers.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/numbers.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/ord.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/ord.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/prec.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/prec.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/regex.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/regex.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/string.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/string.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/tern.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/tern.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/utf8.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/utf8.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/whitespace.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/whitespace.t (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/whitespace1.r (added) (history)
  • /trunk/extensions/AbuseFilter/parser_native/tests/whitespace1.t (added) (history)

Diff [purge]

Index: trunk/extensions/AbuseFilter/parser_native/maketest.cpp
——@@ -0,0 +1,54 @@
 2+/*
 3+ * Copyright (c) 2008 Andrew Garrett.
 4+ * Copyright (c) 2008 River Tarnell <river@wikimedia.org>
 5+ * Derived from public domain code contributed by Victor Vasiliev.
 6+ *
 7+ * Permission is granted to anyone to use this software for any purpose,
 8+ * including commercial applications, and to alter it and redistribute it
 9+ * freely. This software is provided 'as-is', without any express or
 10+ * implied warranty.
 11+ */
 12+
 13+/*
 14+ * Generate a test case for af_parser.
 15+ * Usage: maketest <filter> [VAR=value] [VAR=value...]
 16+ */
 17+
 18+#include <iostream>
 19+
 20+#include <boost/format.hpp>
 21+
 22+int
 23+main(int argc, char **argv)
 24+{
 25+ char zero = 0;
 26+
 27+ if (argc < 2) {
 28+ std::cerr << boost::format("usage: %s <filter> [VAR=value...]") % argv[0];
 29+ return 1;
 30+ }
 31+
 32+ std::cout.write(argv[1], strlen(argv[1]));
 33+ std::cout.write(&zero, 1);
 34+
 35+ argc -= 2;
 36+ argv += 2;
 37+
 38+ while (argc) {
 39+ char *s = strchr(*argv, '=');
 40+ if (s == NULL) {
 41+ std::cerr << "error: variable with no value: " << *argv << '\n';
 42+ return 1;
 43+ }
 44+
 45+ std::cout.write(*argv, s - *argv);
 46+ std::cout.write(&zero, 1);
 47+ *s++ = '\0';
 48+ std::cout.write(s, strlen(s));
 49+ std::cout.write(&zero, 1);
 50+
 51+ argc--;
 52+ argv++;
 53+ }
 54+ std::cout.write(&zero, 1);
 55+}
Index: trunk/extensions/AbuseFilter/parser_native/expr.cpp
——@@ -13,6 +13,7 @@
1414
1515 #include "parser.h"
1616 #include "afstring.h"
 17+#include "affunctions.h"
1718
1819 template<typename charT>
1920 afp::basic_datum<charT>
——@@ -23,13 +24,6 @@
2425
2526 template<typename charT>
2627 afp::basic_datum<charT>
27 -f_norm(std::vector<afp::basic_datum<charT> > const &args)
28 -{
29 - return args[0];
30 -}
31 -
32 -template<typename charT>
33 -afp::basic_datum<charT>
3428 f_length(std::vector<afp::basic_datum<charT> > const &args)
3529 {
3630 return afp::basic_datum<charT>::from_int(args[0].toString().size());
——@@ -49,7 +43,7 @@
5044 e.add_variable(make_u32fray("TWO"), afp::u32datum::from_int(2));
5145 e.add_variable(make_u32fray("THREE"), afp::u32datum::from_int(3));
5246 e.add_function(make_u32fray("add"), f_add<UChar32>);
53 - e.add_function(make_u32fray("norm"), f_norm<UChar32>);
 47+ e.add_function(make_u32fray("norm"), afp::af_norm<UChar32>);
5448 e.add_function(make_u32fray("length"), f_length<UChar32>);
5549
5650 try {
Index: trunk/extensions/AbuseFilter/parser_native/runtests.sh
——@@ -0,0 +1,24 @@
 2+#! /bin/ksh
 3+
 4+npass=0
 5+nfail=0
 6+ntotal=0
 7+
 8+for test in *.t; do
 9+ echo "$test \c"
 10+ filter=$(head -1 $test)
 11+ vars=$(tail +2 $test)
 12+ expect=$(cat ${test%.t}.r)
 13+ result=$(../maketest "$filter" $vars | (cd ..; ./af_parser))
 14+ if [ "$expect" = "$result" ]; then
 15+ echo ...ok
 16+ npass=$((npass + 1))
 17+ else
 18+ echo ...FAIL
 19+ nfail=$((nfail + 1))
 20+ fi
 21+
 22+ ntotal=$((ntotal + 1))
 23+done
 24+
 25+echo "Ran $ntotal tests. PASS: $npass FAIL: $nfail"
Property changes on: trunk/extensions/AbuseFilter/parser_native/runtests.sh
___________________________________________________________________
Added: svn:executable
126 + *
Index: trunk/extensions/AbuseFilter/parser_native/tests/length.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/length.t
——@@ -0,0 +1 @@
 2+length("foobar") == 6
Index: trunk/extensions/AbuseFilter/parser_native/tests/like.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/like.t
——@@ -0,0 +1 @@
 2+"foobér" like "foob?r" & "quux" matches "qu*x"
Index: trunk/extensions/AbuseFilter/parser_native/tests/prec.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/prec.t
——@@ -0,0 +1 @@
 2+(1 + 2 * 3 = 7) & (2 ** 2 * 2 = 8) & (1 - 1 - 1 = -1)
Index: trunk/extensions/AbuseFilter/parser_native/tests/float.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/float.t
——@@ -0,0 +1 @@
 2+(5 / 2 = 2) & (5. / 2 = 2.5) & (5 / 2. = 2.5) & (int(.5) = 0)
Index: trunk/extensions/AbuseFilter/parser_native/tests/whitespace1.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/whitespace1.t
——@@ -0,0 +1 @@
 2+1 == length("a")
Index: trunk/extensions/AbuseFilter/parser_native/tests/ord.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/ord.t
——@@ -0,0 +1 @@
 2+(1 > 0) & (0 < 1) & (2 >= 2) & (2 <= 2)
Index: trunk/extensions/AbuseFilter/parser_native/tests/utf8.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/lcase.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/utf8.t
——@@ -0,0 +1 @@
 2+length("áéý") == 3
Index: trunk/extensions/AbuseFilter/parser_native/tests/lcase.t
——@@ -0,0 +1 @@
 2+lcase("FÁmí") = "fámí"
Index: trunk/extensions/AbuseFilter/parser_native/tests/whitespace.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/whitespace.t
——@@ -0,0 +1,2 @@
 2+1 == FOO
 3+FOO=1
Index: trunk/extensions/AbuseFilter/parser_native/tests/eq.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/string.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/in.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/arith.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/eq.t
——@@ -0,0 +1 @@
 2+(1 == 1) & (1 != 2) & (1 === 1) & (1 == "1") & (1 !== "1")
Index: trunk/extensions/AbuseFilter/parser_native/tests/tern.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/string.t
——@@ -0,0 +1 @@
 2+"\xA3" = "£" & "a\tb" = "a b" & "a\qb" = "aqb"
Index: trunk/extensions/AbuseFilter/parser_native/tests/in.t
——@@ -0,0 +1 @@
 2+"foo" in "foobar" & "quux" contains "ux"
Index: trunk/extensions/AbuseFilter/parser_native/tests/arith.t
——@@ -0,0 +1 @@
 2+(1 + 1 == 2) & (5 - 3 = 2) & (2 * 3 = 6) & (10 / 2 = 5) & (10 % 7 = 3) & (2 ** 4 = 16)
Index: trunk/extensions/AbuseFilter/parser_native/tests/regex.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/tern.t
——@@ -0,0 +1 @@
 2+1 ? 0 ? 3 : 4 : 5 == 4
Index: trunk/extensions/AbuseFilter/parser_native/tests/numbers.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/norm.r
——@@ -0,0 +1 @@
 2+MATCH
Index: trunk/extensions/AbuseFilter/parser_native/tests/regex.t
——@@ -0,0 +1 @@
 2+"foobér" rlike "^[fq]o{2}\\S.r$" & "foo" regex "^f..?.$"
Index: trunk/extensions/AbuseFilter/parser_native/tests/norm.t
——@@ -0,0 +1 @@
 2+norm("foo") = "F0" & norm("anyone") = "ANY0NE"
Index: trunk/extensions/AbuseFilter/parser_native/tests/numbers.t
——@@ -0,0 +1 @@
 2+ax = 10 & Fx = 15 & 10o = 8 & 10b = 2
Index: trunk/extensions/AbuseFilter/parser_native/makefile
——@@ -35,6 +35,7 @@
3636 syntax_check-syntax_check.o
3737
3838 expr_objs = \
 39+ expr-equiv.o \
3940 expr-expr.o
4041
4142 xml_objs = \
——@@ -44,8 +45,11 @@
4546 evaluate-equiv.o \
4647 evaluate-evaluate.o
4748
48 -progs = check af_parser syntax_check af_expr expr evaluate xml
 49+maketest_objs = \
 50+ maketest-maketest.o
4951
 52+progs = check af_parser syntax_check af_expr expr evaluate xml maketest
 53+
5054 all: $(progs)
5155
5256 af_expr: $(af_expr_objs)
——@@ -62,6 +66,8 @@
6367 $(CXX) $(CXXFLAGS) -o $@ $(evaluate_objs) $(LDFLAGS) $(LIBS)
6468 xml: $(xml_objs)
6569 $(CXX) $(CXXFLAGS) -o $@ $(xml_objs) $(LDFLAGS) $(LIBS)
 70+maketest: $(maketest_objs)
 71+ $(CXX) $(CXXFLAGS) -o $@ $(maketest_objs) $(LDFLAGS)
6672
6773 $(af_expr_objs): af_expr-%.o: %.cpp
6874 $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ -c $(@:af_expr-%.o=%.cpp)
——@@ -84,6 +90,12 @@
8591 $(xml_objs): xml-%.o: %.cpp
8692 $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ -c $(@:xml-%.o=%.cpp)
8793
 94+$(maketest_objs): maketest-%.o: %.cpp
 95+ $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ -c $(@:maketest-%.o=%.cpp)
 96+
 97+test: maketest af_parser
 98+ cd tests && ../runtests.sh
 99+
88100 clean:
89101 rm -f *.o $(progs)
90102
Index: trunk/extensions/AbuseFilter/parser_native/parser.h
——@@ -243,7 +243,7 @@
244244 * letters and underscore only) are returned as the
245245 * empty string.
246246 */
247 - variable = leaf_node_d[ +(upper_p | '_') ]
 247+ variable = reduced_node_d[ +(upper_p | '_') ]
248248 ;
249249
250250 /*
——@@ -251,7 +251,7 @@
252252 */
253253 function =
254254 (
255 - root_node_d[ leaf_node_d[
 255+ root_node_d[ reduced_node_d[
256256 +(lower_p | '_')
257257 ] ]
258258 >> inner_node_d[

Status & tagging log