r46680 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r46679‎ | r46680 | r46681 >
Date:15:04, 1 February 2009
Author:jnatividad
Status:deferred
Tags:
Comment:
now stores generated files in a sharded directory structure as a simple way to do data partitioning for scalability
Modified paths:
  • /trunk/extensions/SemanticResultFormats/Ploticus/SRF_Ploticus.php (modified) (history)
  • /trunk/extensions/SemanticResultFormats/Ploticus/SRF_Ploticus_cleanCache.php (modified) (history)

Diff [purge]

Index: trunk/extensions/SemanticResultFormats/Ploticus/SRF_Ploticus.php
@@ -120,12 +120,18 @@
121121 $sanitized_ploticusparams = preg_replace($searches, $replaces, $this->m_ploticusparams);
122122
123123 // Create the ploticus data directory if it doesn't exist
124 - $ploticusDirectory = $wgUploadDirectory . '/ploticus/';
125 - if (!is_dir($ploticusDirectory))
126 - mkdir($ploticusDirectory, 0777);
 124+ // create sharded directory structure to stay under the file no limits of filesystem
 125+ $ploticusDir = $wgUploadDirectory . '/ploticus/';
 126+ if (!is_dir($ploticusDir)) {
 127+ mkdir($ploticusDir, 0777);
 128+ for ($idx=0; $idx < 10; $idx++)
 129+ mkdir($ploticusDir . $idx);
 130+ foreach(range('a','f') as $idx)
 131+ mkdir($ploticusDir . $idx);
 132+ }
127133
128134 // create result csv file that we pass on to ploticus
129 - $tmpFile = tempnam($ploticusDirectory, 'srf-');
 135+ $tmpFile = tempnam($ploticusDir, 'srf-');
130136 if (($fhandle = fopen($tmpFile, 'w')) === false )
131137 return ("<p><strong>ERROR: Cannot create data file - $tmpFile. Check permissions. </strong></p>");
132138 while ( $row = $res->getNext() ) {
@@ -153,16 +159,20 @@
154160 $hashname .= hash_file('md5',$tmpFile);
155161 }
156162
157 - $dataFile = $ploticusDirectory . $hashname . '.csv';
 163+ $orighash = $hashname;
 164+ // modify hashname so files created with it will be stored in shard dir based on first char of hash
 165+ $hashname = substr($hashname, 0, 1) . '/' . $hashname;
 166+ $dataFile = $ploticusDir . $hashname . '.csv';
158167 @unlink($dataFile);
159168 @rename($tmpFile, $dataFile);
 169+ $dataURL = $wgUploadPath . '/ploticus/' . $hashname . '.csv';
160170
161 - $graphFile = $ploticusDirectory . $hashname . '.' . $this->m_imageformat;
 171+ $graphFile = $ploticusDir . $hashname . '.' . $this->m_imageformat;
162172 $graphURL = $wgUploadPath . '/ploticus/' . $hashname . '.' . $this->m_imageformat;
163 - $errorFile = $ploticusDirectory . $hashname . '.err';
164 - $mapFile = $ploticusDirectory . $hashname . '.map';
 173+ $errorFile = $ploticusDir . $hashname . '.err';
 174+ $mapFile = $ploticusDir . $hashname . '.map';
165175 $mapURL = $wgUploadPath . '/ploticus/' . $hashname . '.map';
166 - $scriptFile = $ploticusDirectory . $hashname . '.plo';
 176+ $scriptFile = $ploticusDir . $hashname . '.plo';
167177 $scriptURL = $wgUploadPath . '/ploticus/' . $hashname . '.plo';
168178
169179 // get time graph was last generated and if liveupdating is on, check to see if the
@@ -196,7 +206,7 @@
197207 // and replace it with actual values. (case-sensitive)
198208 // The special strings currently are: %DATAFILE.CSV%, %WORKINGDIR%
199209 $replaces = array('%DATAFILE.CSV%' => wfEscapeShellArg($dataFile),
200 - '%WORKINGDIR%' => $ploticusDirectory);
 210+ '%WORKINGDIR%' => $ploticusDir);
201211 $literal_ploticusparams = strtr($sanitized_ploticusparams, $replaces);
202212 $fhandle = fopen($scriptFile, 'w');
203213 fputs($fhandle, $literal_ploticusparams);
@@ -215,7 +225,7 @@
216226 ' -' . $this->m_imageformat;
217227
218228 if ($this->m_imageformat == 'drawdump' || $this->m_imageformat == 'drawdumpa' ) {
219 - $commandline .= ' ' . wfEscapeShellArg($ploticusDirectory . '/' . $this->m_drawdumpoutput);
 229+ $commandline .= ' ' . wfEscapeShellArg($ploticusDir . '/' . $this->m_drawdumpoutput);
220230 } else {
221231 $commandline .= ' -o '. wfEscapeShellArg($graphFile);
222232 }
@@ -279,8 +289,8 @@
280290 $mapData = file_get_contents($mapFile);
281291 // we replace + with _ since ploticus uses + to represent spaces which mediawiki does not understand
282292 $mapData = str_replace("+","_",$mapData);
283 - $rtnstr .= '<map name="'. $hashname . '">'. $mapData .
284 - '</map><img src="' . $graphURL . '" border="0" usemap="#' . $hashname . '">';
 293+ $rtnstr .= '<map name="'. $orighash . '">'. $mapData .
 294+ '</map><img src="' . $graphURL . '" border="0" usemap="#' . $orighash . '">';
285295 } else {
286296 $rtnstr .= '<img src="' . $graphURL . '" alt="' . $this->alttext .'">';
287297 }
@@ -300,8 +310,8 @@
301311 // INFOROW - ACTIONS - col 1
302312 // if showcsv or debug is on, add link to data file (CSV)
303313 if ($this->m_showcsv || $this->m_debug) {
304 - $rtnstr .= '<a href="' . $wgUploadPath . '/ploticus/' . $hashname . '.csv" title="CSV file"><img src="'.
305 - $srficonPath.'csv_16.png" alt="CSV file"></a>';
 314+ $rtnstr .= '<a href="' . $dataURL . '" title="CSV file"><img src="'.
 315+ $srficonPath . 'csv_16.png" alt="CSV file"></a>';
306316 } else {
307317 @unlink($dataFile); // otherwise, clean it up
308318 }
@@ -309,13 +319,13 @@
310320 // if showimagelink is on, add link to open image in a new window
311321 if ($this->m_showimagelink ) {
312322 $rtnstr .= ' <a href="' . $graphURL . '" target="_blank" title="Open image in new window"><img src="'.
313 - $srficonPath . 'barchart_16.png" alt="Open image in new window"></a>';
 323+ $srficonPath . 'barchart_16.png" alt="Open image in new window"></a>';
314324 }
315325
316326 // if showrefresh is on, create link to force refresh
317327 if ($this->m_showrefresh) {
318328 $rtnstr .= ' <a href="' . $wgArticlePath . '?action=purge" title="Reload"><img src="'.
319 - $srficonPath . 'reload_16.png" alt="Reload"></a>';
 329+ $srficonPath . 'reload_16.png" alt="Reload"></a>';
320330 }
321331
322332 // INFOROW - col 2
@@ -336,9 +346,11 @@
337347 // add link to script or display ploticus cmdline/script
338348 if ($this->m_debug) {
339349 if ($this->m_ploticusmode == 'script') {
340 - $rtnstr .= '<tr><td align="center" colspan="3"><strong>DEBUG: <a href="' . $scriptURL . '" target="_blank">SCRIPT</a></strong></td></tr>';
 350+ $rtnstr .= '<tr><td align="center" colspan="3"><strong>DEBUG: <a href="' .
 351+ $scriptURL . '" target="_blank">SCRIPT</a></strong></td></tr>';
341352 } else {
342 - $rtnstr .= '<tr><td align="center" colspan="3"><strong>DEBUG: PREFAB</strong></td></tr><tr><td colspan="3">' . $commandline . '</td></tr>';
 353+ $rtnstr .= '<tr><td align="center" colspan="3"><strong>DEBUG: PREFAB</strong></td></tr><tr><td colspan="3">' .
 354+ $commandline . '</td></tr>';
343355 }
344356 }
345357
Index: trunk/extensions/SemanticResultFormats/Ploticus/SRF_Ploticus_cleanCache.php
@@ -47,6 +47,7 @@
4848 $ploticusDirectory = $wgUploadDirectory . '/ploticus';
4949 $deletecount = 0;
5050
 51+// TODO: Modify to be shard dir aware
5152 if( $dirhandle = @opendir($ploticusDirectory) ) {
5253 while( false !== ($filename = readdir($dirhandle)) ) {
5354 if( $filename != '.' && $filename != '..' ) {

Status & tagging log