| Index: trunk/extensions/MetavidWiki/maintenance/scrape_and_insert.inc.php |
| — | — | @@ -537,8 +537,8 @@ |
| 538 | 538 | return "[[Mentions Bill:={$matches[0]}]]"; |
| 539 | 539 | } |
| 540 | 540 | } |
| 541 | | - /* converts c-span bill_id to gov_track bill id */ |
| 542 | | - function get_and_proccess_billid( $bill_key, $stream_date = '', $session = '' ) { |
| | 541 | + /* converts c-span bill_id to gov_track bill id */ |
| | 542 | + function get_and_process_billid( $bill_key, $stream_date = '', $session = '' ) { |
| 543 | 543 | global $MvBillTypes; |
| 544 | 544 | // add a space to bill key after $bill_type key |
| 545 | 545 | foreach ( $this->bill_types as $bk => $na ) { |
| — | — | @@ -548,7 +548,7 @@ |
| 549 | 549 | } |
| 550 | 550 | } |
| 551 | 551 | } |
| 552 | | - // first get the year to detrim the house session: |
| | 552 | + // first get the year to determine the house session: |
| 553 | 553 | if ( $session == '' ) { |
| 554 | 554 | $year = date( 'y', $stream_date ); |
| 555 | 555 | if ( $year == '01' || $year == '02' ) { $session = '107'; |
| — | — | @@ -568,7 +568,7 @@ |
| 569 | 569 | } |
| 570 | 570 | } |
| 571 | 571 | if ( trim( $bill_key ) == '' )return false; |
| 572 | | - // attempt to asertain maplight bill id: |
| | 572 | + // attempt to ascertain maplight bill id: |
| 573 | 573 | $mapLightBillId = $this->getMAPLightBillId( $bill_key, $session ); |
| 574 | 574 | |
| 575 | 575 | print "GOT bill id: $govTrackBillId from $bill_key\n"; |
| — | — | @@ -684,25 +684,27 @@ |
| 685 | 685 | } |
| 686 | 686 | } |
| 687 | 687 | /***************************** |
| 688 | | - * Proccess MapLight Info |
| | 688 | + * Process MapLight Info |
| 689 | 689 | *****************************/ |
| 690 | 690 | if ( $mapLightBillId ) { |
| 691 | 691 | $bill_interest = $this->proccMapLightBillIntrests( $mapLightBillId ); |
| 692 | | - $i = 1; |
| 693 | | - foreach ( $bill_interest['support'] as $interest ) { |
| 694 | | - $this->procMapLightInterest( $interest ); |
| 695 | | - $bp .= 'Supporting Interest ' . $i . '=' . $interest['name'] . "|\n"; |
| 696 | | - $i++; |
| 697 | | - //process interest |
| 698 | | - $this->procMapLightInterest( $interest ); |
| | 692 | + if( $bill_interest ) { |
| | 693 | + $i = 1; |
| | 694 | + foreach ( $bill_interest['support'] as $interest ) { |
| | 695 | + $this->procMapLightInterest( $interest ); |
| | 696 | + $bp .= 'Supporting Interest ' . $i . '=' . $interest['name'] . "|\n"; |
| | 697 | + $i++; |
| | 698 | + //process interest |
| | 699 | + $this->procMapLightInterest( $interest ); |
| | 700 | + } |
| | 701 | + $i = 1; |
| | 702 | + foreach ( $bill_interest['oppose'] as $interest ) { |
| | 703 | + $bp .= 'Opposing Interest ' . $i . '=' . $interest['name'] . "|\n"; |
| | 704 | + $i++; |
| | 705 | + //process interest |
| | 706 | + $this->procMapLightInterest( $interest ); |
| | 707 | + } |
| 699 | 708 | } |
| 700 | | - $i = 1; |
| 701 | | - foreach ( $bill_interest['oppose'] as $interest ) { |
| 702 | | - $bp .= 'Opposing Interest ' . $i . '=' . $interest['name'] . "|\n"; |
| 703 | | - $i++; |
| 704 | | - //process interest |
| 705 | | - $this->procMapLightInterest( $interest ); |
| 706 | | - } |
| 707 | 709 | } |
| 708 | 710 | $bp .= "}}\n"; |
| 709 | 711 | |
| — | — | @@ -880,7 +882,7 @@ |
| 881 | 883 | // $sql = "SELECT * FROM `metavid`.`cache_time_url_text` WHERE `url` LIKE '$url'"; |
| 882 | 884 | // select( $table, $vars, $conds='', $fname = 'Database::select', $options = array() ) |
| 883 | 885 | $res = $dbr->select( 'mv_url_cache', '*', array( 'url' => $url ), 'MV_BaseScraper::doRequest' ); |
| 884 | | - // @@todo check date for experation |
| | 886 | + // @@todo check date for expiration |
| 885 | 887 | if ( $res->numRows() == 0 || $get_fresh) { |
| 886 | 888 | echo "do web request: " . $url . "\n"; |
| 887 | 889 | // get the content: |
| Index: trunk/extensions/MetavidWiki/maintenance/video_ocr_thumb_insert.php |
| — | — | @@ -29,6 +29,7 @@ |
| 30 | 30 | OPTIONS: |
| 31 | 31 | --interval @default 5 seconds |
| 32 | 32 | --overwrite_image @default no overwrite; if set will force image output |
| | 33 | + --use_file $file //the full path to the file you want to use to generate thumbnails |
| 33 | 34 | |
| 34 | 35 | DURATION is scraped from ffmpeg |
| 35 | 36 | |
| — | — | @@ -55,9 +56,11 @@ |
| 56 | 57 | $overwrite_image=true; |
| 57 | 58 | } |
| 58 | 59 | $workingdir = '/video/metavid/raw_mpeg2'; |
| 59 | | - |
| 60 | | - |
| 61 | | -$filename = $workingdir .'/'. $stream_name . '.mpeg'; |
| | 60 | +if(isset( $options['use_file'] )){ |
| | 61 | + $filename = $options['use_file']; |
| | 62 | +}else{ |
| | 63 | + $filename = $workingdir .'/'. $stream_name . '.mpeg'; |
| | 64 | +} |
| 62 | 65 | $duration = getDuration($filename); |
| 63 | 66 | |
| 64 | 67 | |
| Index: trunk/extensions/MetavidWiki/maintenance/metavid2mvWiki.php |
| — | — | @@ -96,6 +96,7 @@ |
| 97 | 97 | 'file_check' checks inserted streams file urls/pointers |
| 98 | 98 | 'do_stream_date_check' |
| 99 | 99 | 'do_remove_orphaned_streams' |
| | 100 | + 'mvd_consistancy_check' makes sure all mvd text layers are consistent |
| 100 | 101 | |
| 101 | 102 | EOT; |
| 102 | 103 | exit (); |
| — | — | @@ -147,8 +148,8 @@ |
| 148 | 149 | upTemplates( $force ); |
| 149 | 150 | break; |
| 150 | 151 | // by default treat the argument as a stream name: |
| 151 | | - case 'mvd_error_check': |
| 152 | | - |
| | 152 | + case 'mvd_consistancy_check': |
| | 153 | + mvd_consistancy_check(); |
| 153 | 154 | break; |
| 154 | 155 | case 'rm_congress_persons': |
| 155 | 156 | do_rm_congress_persons(); |
| Index: trunk/extensions/MetavidWiki/maintenance/metavid2mvWiki.inc.php |
| — | — | @@ -695,6 +695,54 @@ |
| 696 | 696 | } |
| 697 | 697 | return $out; |
| 698 | 698 | } |
| | 699 | +function mvd_consistancy_check(){ |
| | 700 | + //get all 2009 streams: |
| | 701 | + $dbr = wfGetDB( DB_READ ); |
| | 702 | + $streams = array(); |
| | 703 | + $result = $dbr->select( 'mv_streams', |
| | 704 | + '*', |
| | 705 | + 'date_start_time >= '. mktime(0, 0, 0, 1, 1, 2009), |
| | 706 | + __METHOD__ |
| | 707 | + ); |
| | 708 | + if ( $dbr->numRows( $result ) == 0 )die("no streams found"."\n". $dbr->lastQuery() ."\n"); |
| | 709 | + while ( $stream = $dbr->fetchObject( $result ) ) { |
| | 710 | + //get all the mvds for this stream |
| | 711 | + $mvd_res = $dbr->select( 'mv_mvd_index', '*', array('stream_id'=>$stream->id)); |
| | 712 | + while ( $mvd = $dbr->fetchObject( $mvd_res ) ) { |
| | 713 | + //make sure the article exists: |
| | 714 | + $mvdTitle = Title::newFromText($mvd->wiki_title, MV_NS_MVD); |
| | 715 | + if($mvdTitle->exists()){ |
| | 716 | + //update the text: |
| | 717 | + $mvdArticle = new Article ($mvdTitle); |
| | 718 | + $text = $mvdArticle->getRawText(); |
| | 719 | + //find the spoken by or speech by text: |
| | 720 | + $sb_pat = '/\[\[Spoken By(\:.)([^\]]*)]]/i'; |
| | 721 | + preg_match($sb_pat, $text, $matches ); |
| | 722 | + if(isset($matches[2])){ |
| | 723 | + $replacement = ($matches[2] == 'Unknown')?'': |
| | 724 | + '[[Spoken By::'. str_replace('_', ' ', $matches[2]).']]'; |
| | 725 | + $text = preg_replace($sb_pat, $replacement, $text); |
| | 726 | + } |
| | 727 | + //do the same for speech by |
| | 728 | + $sb_pat = '/\[\[Speech by(\:.)([^\]]*)]]/i'; |
| | 729 | + preg_match($sb_pat, $text, $matches ); |
| | 730 | + if(isset($matches[2])){ |
| | 731 | + $replacement = ($matches[2] == 'Unknown')?'': |
| | 732 | + '[[Speech by::'. str_replace('_', ' ', $matches[2]).']]'; |
| | 733 | + $text = preg_replace($sb_pat, $replacement, $text); |
| | 734 | + } |
| | 735 | + //trim all double spaces |
| | 736 | + $text = preg_replace('/[\s]+/', ' ', $text); |
| | 737 | + //uc upper words: |
| | 738 | + //$text = preg_replace("/[^A-Z]\.(\s)(\\w)/e", '".$1".strtoupper("$2")', $text); |
| | 739 | + do_update_wiki_page( $mvdTitle, trim($text),'',true); |
| | 740 | + }else{ |
| | 741 | + print "orphaned mvd: {$mvd->wiki_title} (should remove) \n"; |
| | 742 | + } |
| | 743 | + } |
| | 744 | + //die('only update one stream at a time'); |
| | 745 | + } |
| | 746 | +} |
| 699 | 747 | function do_bill_insert( $bill_key ) { |
| 700 | 748 | //grab bill list with categories from govtrack |
| 701 | 749 | } |