r11168 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r11167‎ | r11168 | r11169 >
Date:11:04, 1 October 2005
Author:vibber
Status:old
Tags:
Comment:
Ongoing SQL work:
* Batch multiple inserts on revision and text in 1.5 output
* Preparing output modes for direct db connection (not done yet)
Modified paths:
  • /trunk/mwdumper/.classpath (modified) (history)
  • /trunk/mwdumper/src/org/mediawiki/importer/SqlFileStream.java (modified) (history)
  • /trunk/mwdumper/src/org/mediawiki/importer/SqlServerStream.java (added) (history)
  • /trunk/mwdumper/src/org/mediawiki/importer/SqlStream.java (added) (history)
  • /trunk/mwdumper/src/org/mediawiki/importer/SqlWriter.java (modified) (history)
  • /trunk/mwdumper/src/org/mediawiki/importer/SqlWriter15.java (modified) (history)

Diff [purge]

Index: trunk/mwdumper/.classpath
——@@ -2,5 +2,6 @@
33 <classpath>
44 <classpathentry kind="src" path="src"/>
55 <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 6+ <classpathentry kind="lib" path="libs/junit.jar"/>
67 <classpathentry kind="output" path="bin"/>
78 </classpath>
Index: trunk/mwdumper/src/org/mediawiki/importer/SqlWriter15.java
——@@ -58,19 +58,21 @@
5959 }
6060
6161 public void writeEndPage() {
62 - if (lastRevision != null)
 62+ if (lastRevision != null) {
 63+ flushInsertBuffers();
6364 updatePage(currentPage, lastRevision);
 65+ }
6466 currentPage = null;
6567 lastRevision = null;
6668 }
6769
6870 public void writeRevision(Revision revision) {
69 - insertRow("text", new Object[][] {
 71+ bufferInsertRow("text", new Object[][] {
7072 {"old_id", new Integer(revision.Id)}, // FIXME
7173 {"old_text", revision.Text},
7274 {"old_flags", "utf-8"}});
7375
74 - insertRow("revision", new Object[][] {
 76+ bufferInsertRow("revision", new Object[][] {
7577 {"rev_id", new Integer(revision.Id)},
7678 {"rev_page", new Integer(currentPage.Id)},
7779 {"rev_text_id", new Integer(revision.Id)}, // FIXME
Index: trunk/mwdumper/src/org/mediawiki/importer/SqlServerStream.java
——@@ -0,0 +1,37 @@
 2+package org.mediawiki.importer;
 3+
 4+import java.io.IOException;
 5+import java.sql.Connection;
 6+import java.sql.SQLException;
 7+import java.sql.Statement;
 8+
 9+public class SqlServerStream implements SqlStream {
 10+ private Connection connection;
 11+
 12+ public SqlServerStream(Connection conn) {
 13+ connection = conn; // TODO
 14+ }
 15+
 16+ public void writeComment(CharSequence sql) {
 17+ // do nothing
 18+ }
 19+
 20+ public void writeStatement(CharSequence sql) throws IOException {
 21+ Statement statement;
 22+ try {
 23+ statement = connection.createStatement();
 24+ statement.execute(sql.toString());
 25+ } catch (SQLException e) {
 26+ throw new IOException(e.toString());
 27+ }
 28+ }
 29+
 30+ public void close() throws IOException {
 31+ try {
 32+ connection.close();
 33+ } catch (SQLException e) {
 34+ throw new IOException(e.toString());
 35+ }
 36+ }
 37+
 38+}
Property changes on: trunk/mwdumper/src/org/mediawiki/importer/SqlServerStream.java
___________________________________________________________________
Added: svn:eol-style
139 + native
Added: svn:keywords
240 + Author Date Id Revision
Index: trunk/mwdumper/src/org/mediawiki/importer/SqlFileStream.java
——@@ -3,7 +3,7 @@
44 import java.io.OutputStream;
55 import java.io.PrintStream;
66
7 -public class SqlFileStream {
 7+public class SqlFileStream implements SqlStream {
88 protected PrintStream stream;
99
1010 public SqlFileStream(OutputStream output) {
Index: trunk/mwdumper/src/org/mediawiki/importer/SqlStream.java
——@@ -0,0 +1,9 @@
 2+package org.mediawiki.importer;
 3+
 4+import java.io.IOException;
 5+
 6+public interface SqlStream {
 7+ public void writeComment(CharSequence sql) throws IOException;
 8+ public void writeStatement(CharSequence sql) throws IOException;
 9+ public void close() throws IOException;
 10+}
Property changes on: trunk/mwdumper/src/org/mediawiki/importer/SqlStream.java
___________________________________________________________________
Added: svn:eol-style
111 + native
Added: svn:keywords
212 + Author Date Id Revision
Index: trunk/mwdumper/src/org/mediawiki/importer/SqlWriter.java
——@@ -28,6 +28,7 @@
2929 import java.text.MessageFormat;
3030 import java.util.Calendar;
3131 import java.util.GregorianCalendar;
 32+import java.util.Hashtable;
3233 import java.util.Iterator;
3334 import java.util.TimeZone;
3435
——@@ -79,11 +80,41 @@
8081 return text;
8182 }
8283
83 - protected Object insertRow(String table, Object[][] row) {
 84+ Hashtable insertBuffers = new Hashtable();
 85+ int blockSize = 1024 * 512; // default 512k inserts
 86+ protected void bufferInsertRow(String table, Object[][] row) {
 87+ StringBuffer sql = (StringBuffer)insertBuffers.get(table);
 88+ if (sql != null) {
 89+ if (sql.length() < blockSize) {
 90+ sql.append(',');
 91+ appendInsertValues(sql, row);
 92+ return;
 93+ } else {
 94+ flushInsertBuffers();
 95+ }
 96+ }
 97+ sql = new StringBuffer();
 98+ appendInsertStatement(sql, table, row);
 99+ insertBuffers.put(table, sql);
 100+ }
 101+
 102+ protected void flushInsertBuffers() {
 103+ Iterator iter = insertBuffers.values().iterator();
 104+ while (iter.hasNext()) {
 105+ stream.writeStatement((CharSequence)iter.next());
 106+ }
 107+ insertBuffers.clear();
 108+ }
 109+
 110+ protected void insertRow(String table, Object[][] row) {
84111 StringBuffer sql = new StringBuffer();
85 -
 112+ appendInsertStatement(sql, table, row);
 113+ stream.writeStatement(sql);
 114+ }
 115+
 116+ private void appendInsertStatement(StringBuffer sql, String table, Object[][] row) {
86117 sql.append("INSERT INTO ");
87 - //sql.append(_tablePrefix);
 118+ //sql.append(tablePrefix);
88119 sql.append(table);
89120 sql.append(" (");
90121
——@@ -93,25 +124,26 @@
94125 sql.append(',');
95126 sql.append(field);
96127 }
97 - sql.append(") VALUES (");
98 -
 128+ sql.append(") VALUES ");
 129+ appendInsertValues(sql, row);
 130+ }
 131+
 132+ private void appendInsertValues(StringBuffer sql, Object[][] row) {
 133+ sql.append('(');
99134 for (int i = 0; i < row.length; i++) {
100135 Object val = row[i][1];
101136 if (i > 0)
102137 sql.append(',');
103138 sql.append(sqlSafe(val));
104139 }
105 - sql.append(")");
106 -
107 - stream.writeStatement(sql);
108 - return null;
 140+ sql.append(')');
109141 }
110142
111143 protected void updateRow(String table, Object[][] row, String keyField, Object keyValue) {
112144 StringBuffer sql = new StringBuffer();
113145
114146 sql.append("UPDATE ");
115 - //sql.append(_tablePrefix);
 147+ //sql.append(tablePrefix);
116148 sql.append(table);
117149 sql.append(" SET ");
118150

Status & tagging log