| Index: trunk/extensions/SemanticMediaWiki/maintenance/SMW_dumpRDF.php |
| — | — | @@ -10,11 +10,14 @@ |
| 11 | 11 | * --properties only do properties |
| 12 | 12 | * --types only do types |
| 13 | 13 | * --individuals only do pages that are no categories, properties, or types |
| | 14 | + * -d <delay> slows down the export in order to stress the server less, |
| | 15 | + * sleeping for <delay> milliseconds every now and then |
| | 16 | + * -e <each> after how many exported entities should the server take a nap? |
| 14 | 17 | * |
| 15 | 18 | * @author Markus Krötzsch |
| 16 | 19 | */ |
| 17 | 20 | |
| 18 | | -$optionsWithArgs = array( 'o' ); // -o <output file> |
| | 21 | +$optionsWithArgs = array( 'o', 'd', 'e' ); |
| 19 | 22 | |
| 20 | 23 | require_once( 'commandLine.inc' ); |
| 21 | 24 | require_once( "$IP/extensions/SemanticMediaWiki/specials/ExportRDF/SMW_SpecialExportRDF.php"); |
| — | — | @@ -24,7 +27,18 @@ |
| 25 | 28 | } else { |
| 26 | 29 | $outfile = false; |
| 27 | 30 | } |
| | 31 | +if ( !empty( $options['d'] ) ) { |
| | 32 | + $delay = intval($options['d']) * 1000; |
| | 33 | +} else { |
| | 34 | + $delay = 0; |
| | 35 | +} |
| | 36 | +if ( !empty( $options['e'] ) ) { |
| | 37 | + $delayeach = intval($options['e']); |
| | 38 | +} else { |
| | 39 | + $delayeach = ( $delay === 0 ) ? 0 : 1; |
| | 40 | +} |
| 28 | 41 | |
| | 42 | + |
| 29 | 43 | if ( array_key_exists( 'categories' , $options ) ) { |
| 30 | 44 | $export_ns = NS_CATEGORY; |
| 31 | 45 | } elseif ( array_key_exists( 'properties' , $options ) ) { |
| — | — | @@ -38,5 +52,5 @@ |
| 39 | 53 | } |
| 40 | 54 | |
| 41 | 55 | $exRDF = new ExportRDF(); |
| 42 | | -$exRDF->printAll($outfile, $export_ns); |
| | 56 | +$exRDF->printAll($outfile, $export_ns, $delay, $delayeach); |
| 43 | 57 | |
| Index: trunk/extensions/SemanticMediaWiki/specials/ExportRDF/SMW_SpecialExportRDF.php |
| — | — | @@ -372,6 +372,9 @@ |
| 373 | 373 | * properties are exported as well. Enables "browsable RDF." |
| 374 | 374 | */ |
| 375 | 375 | public function printPages($pages, $recursion = 1, $backlinks = true) { |
| | 376 | + |
| | 377 | + wfProfileIn("RDF::PrintPages"); |
| | 378 | + |
| 376 | 379 | $this->store = &smwfGetStore(); |
| 377 | 380 | $this->pre_ns_buffer = ''; |
| 378 | 381 | $this->post_ns_buffer = ''; |
| — | — | @@ -381,6 +384,7 @@ |
| 382 | 385 | $this->printHeader(); // also inits global namespaces |
| 383 | 386 | $linkCache =& LinkCache::singleton(); |
| 384 | 387 | |
| | 388 | + wfProfileIn("RDF::PrintPages::PrepareQueue"); |
| 385 | 389 | // transform pages into queued export titles |
| 386 | 390 | $cur_queue = array(); |
| 387 | 391 | foreach ($pages as $page) { |
| — | — | @@ -389,12 +393,15 @@ |
| 390 | 394 | $et = new SMWExportTitle($title, $this); |
| 391 | 395 | $cur_queue[$title->getPrefixedURL() . ' '] = $et; // " " is the modifier separator |
| 392 | 396 | } |
| | 397 | + wfProfileOut("RDF::PrintPages::PrepareQueue"); |
| 393 | 398 | |
| 394 | 399 | while (count($cur_queue) > 0) { |
| 395 | 400 | // first, print all selected pages |
| 396 | 401 | foreach ( $cur_queue as $et) { |
| | 402 | + wfProfileIn("RDF::PrintPages::PrintOne"); |
| 397 | 403 | $this->printTriples($et); |
| 398 | 404 | $this->markAsDone($et); |
| | 405 | + wfProfileOut("RDF::PrintPages::PrintOne"); |
| 399 | 406 | |
| 400 | 407 | // prepare array for next iteration |
| 401 | 408 | $cur_queue = array(); |
| — | — | @@ -404,6 +411,7 @@ |
| 405 | 412 | } |
| 406 | 413 | // possibly add backlinks |
| 407 | 414 | if ($backlinks === true) { |
| | 415 | + wfProfileIn("RDF::PrintPages::GetBacklinks"); |
| 408 | 416 | $inRels = $this->store->getInProperties( $et->value ); |
| 409 | 417 | foreach ($inRels as $inRel) { |
| 410 | 418 | $inSubs = $this->store->getPropertySubjects( $inRel, $et->value ); |
| — | — | @@ -424,12 +432,14 @@ |
| 425 | 433 | } |
| 426 | 434 | } |
| 427 | 435 | if ( 0 == $recursion ) $backlinks = false; // do not recurse through backlinks either |
| | 436 | + wfProfileOut("RDF::PrintPages::GetBacklinks"); |
| 428 | 437 | } |
| 429 | 438 | if ($this->delay_flush > 0) $this->delay_flush--; |
| 430 | 439 | } |
| 431 | 440 | $linkCache->clear(); |
| 432 | 441 | } |
| 433 | 442 | |
| | 443 | + wfProfileIn("RDF::PrintPages::Auxilliary"); |
| 434 | 444 | // if pages are not processed recursively, print mentioned declarations |
| 435 | 445 | if (!empty($this->element_queue)) { |
| 436 | 446 | if ( '' != $this->pre_ns_buffer ) { |
| — | — | @@ -442,16 +452,19 @@ |
| 443 | 453 | $this->printTriples($et,false); |
| 444 | 454 | } |
| 445 | 455 | } |
| | 456 | + wfProfileOut("RDF::PrintPages::Auxilliary"); |
| 446 | 457 | |
| 447 | 458 | $this->printFooter(); |
| 448 | 459 | $this->flushBuffers(true); |
| | 460 | + |
| | 461 | + wfProfileOut("RDF::PrintPages"); |
| 449 | 462 | } |
| 450 | 463 | |
| 451 | 464 | /** |
| 452 | 465 | * This function prints RDF for *all* pages within the wiki, and for all |
| 453 | 466 | * elements that are referred to in the exported RDF. |
| 454 | 467 | */ |
| 455 | | - public function printAll($outfile, $ns_restriction = false) { |
| | 468 | + public function printAll($outfile, $ns_restriction = false, $delay, $delayeach) { |
| 456 | 469 | global $smwgNamespacesWithSemanticLinks; |
| 457 | 470 | $linkCache =& LinkCache::singleton(); |
| 458 | 471 | |
| — | — | @@ -480,6 +493,7 @@ |
| 481 | 494 | |
| 482 | 495 | $a_count = 0; $d_count = 0; //DEBUG |
| 483 | 496 | |
| | 497 | + $delaycount = $delayeach; |
| 484 | 498 | for ($id = $start; $id <= $end; $id++) { |
| 485 | 499 | $title = Title::newFromID($id); |
| 486 | 500 | if ( ($title === NULL) || !smwfIsSemanticsProcessed($title->getNamespace()) ) continue; |
| — | — | @@ -509,6 +523,11 @@ |
| 510 | 524 | // want to export now is a potential memory leak |
| 511 | 525 | } |
| 512 | 526 | } |
| | 527 | + // sleep each $delaycount for $delay ms to be nice to the server |
| | 528 | + if (($delaycount-- < 0) && ($delayeach != 0)) { |
| | 529 | + usleep($delay); |
| | 530 | + $delaycount = $delayeach; |
| | 531 | + } |
| 513 | 532 | } |
| 514 | 533 | if ($outfile !== false) { // flush buffer |
| 515 | 534 | fwrite($file, $this->post_ns_buffer); |