| Index: trunk/wikistats/dumps/publish.sh |
| — | — | @@ -1,13 +1,34 @@ |
| 2 | 2 | #!/bin/bash |
| | 3 | +clear |
| | 4 | +now=`date +%s` |
| | 5 | + |
| | 6 | +function echo2 { |
| | 7 | + echo $1 |
| | 8 | + echo $1 >> log_publish_sh.txt |
| | 9 | +} |
| | 10 | + |
| | 11 | +clear |
| | 12 | +echo2 "----------------" |
| | 13 | +echo2 "Start publish.sh $1 $2" |
| | 14 | +date >> log_publish_sh.txt |
| | 15 | + |
| | 16 | +# Validate project code |
| | 17 | +if [ "$1" == "" ] |
| | 18 | +then |
| | 19 | + echo2 "Project code missing! Specify as 1st argument one of wb,wk,wn,wp,wq,ws,wv,wx" |
| | 20 | + exit |
| | 21 | +fi |
| | 22 | + |
| | 23 | +# Abort when 3rd argument specifies a threshold in days, which is not met |
| | 24 | +# This prevents costly processing step when new month has just started and most ocunting still needs to be done |
| 3 | 25 | abort_before=$3 |
| 4 | 26 | day_of_month=$(date +"%d") |
| 5 | 27 | if [ $day_of_month -lt ${abort_before:=0} ] |
| 6 | 28 | then |
| 7 | | - echo publish.sh: day of month $day_of_month lt $abort_before - exit |
| | 29 | + echo2 "publish.sh: day of month $day_of_month lt $abort_before - exit" |
| 8 | 30 | exit |
| 9 | 31 | fi |
| 10 | 32 | |
| 11 | | -clear |
| 12 | 33 | case "$1" in |
| 13 | 34 | wb) project='Wikibooks' ; dir='wikibooks' ;; |
| 14 | 35 | wk) project='Wiktionaries' ; dir='wiktionary' ;; |
| — | — | @@ -19,51 +40,63 @@ |
| 20 | 41 | wx) project='Wikispecial' ; dir='wikispecial' ;; |
| 21 | 42 | *) project='unknown' ; dir='...' ;; |
| 22 | 43 | esac |
| 23 | | -echo "Publish $project" >> WikiCountsLogConcise.txt |
| 24 | | -echo "Publish $project" |
| | 44 | +echo2 "Publish $project" |
| 25 | 45 | |
| 26 | 46 | #case "$2" in |
| 27 | 47 | # en) dir="$dir/EN" ; |
| 28 | 48 | #esac |
| 29 | 49 | |
| 30 | | -echo "Copy csv files" |
| 31 | | -csv=/a/wikistats/csv_$1 |
| 32 | | -htdocs=/mnt/htdocs/$dir/csv |
| 33 | | -echo csv $csv |
| 34 | | -echo htdocs $htdocs |
| | 50 | +htdocs="/mnt/htdocs/$dir/csv" |
| | 51 | +csv="/a/wikistats/csv_$1" |
| | 52 | +archive="/mnt/dumps/xmldatadumps/public/other/pagecounts-ez/wikistats" # odd name, temp location |
| 35 | 53 | |
| 36 | | -chmod 774 $csv/* |
| 37 | | -#cp $csv/csv_$1.zip $csv/csv.zip |
| 38 | | -#rm $csv/csv.zip # temp, obsolete file |
| 39 | | -#rm -f $htdocs/* |
| 40 | | -cp $csv/* $htdocs -rvf |
| | 54 | +rm -f $csv/csv.zip |
| | 55 | +chmod 774 $csv/*.csv |
| 41 | 56 | |
| 42 | | -cp /a/wikistats/zip_all/csv_$1.zip /mnt/htdocs |
| | 57 | +echo2 "" |
| | 58 | +echo2 "Copy new/updated csv files from $csv -> $htdocs" |
| 43 | 59 | |
| 44 | | -publish=\#publish.txt |
| | 60 | +echo2 "" |
| | 61 | +rsync -av $csv/*.csv $htdocs |
| | 62 | + |
| | 63 | +echo2 "" |
| | 64 | +echo2 "Copy csv and html zip files -> public archive $archive" |
| | 65 | +echo2 "" |
| | 66 | +rsync -av /a/wikistats/zip_all/csv*.zip $archive >> log_publish_sh.txt |
| | 67 | +rsync -av /a/wikistats/zip_all/out*.zip $archive >> log_publish_sh.txt |
| 45 | 68 | |
| 46 | | -for lang in EN AST BG BR CA CS DA DE EO ES FR HE HU ID IT JA NL NN PL PT RO RU SK SL SR SV WA ZH |
| 47 | | -#for lang in EN |
| | 69 | +publish="#publish.txt" |
| | 70 | + |
| | 71 | +for lang in AST BG BR CA CS DA DE EO ES FR HE HU ID IT JA NL NN PL PT RO RU SK SL SR SV WA ZH |
| | 72 | +#for lang in EN |
| 48 | 73 | do |
| 49 | | - out=/a/out/out_$1/$lang |
| 50 | | - htdocs=/mnt/htdocs/$dir/$lang |
| 51 | | - echo out $out |
| 52 | | - echo htdocs $htdocs |
| 53 | | - echo dir $dir |
| 54 | | - echo lang $lang |
| | 74 | + out=/a/out/out_$1/$lang/ |
| | 75 | + htdocs=/mnt/htdocs/$dir/$lang/ |
| | 76 | + |
| | 77 | + echo2 "" |
| | 78 | + echo2 "Language $lang" |
| | 79 | + echo2 "" |
| | 80 | + echo2 "Copy new and updated files from $out -> $htdocs" |
| | 81 | + rsync -a $out/* $htdocs/ >> log_publish_sh.txt |
| | 82 | + ls -l $htdocs |
| | 83 | + |
| 55 | 84 | # if test -e $out/WikiReportsPublish.txt |
| 56 | 85 | # then mv $out/WikiReportsPublish.txt "$out/$publish" ; fi ; |
| 57 | 86 | |
| 58 | | - |
| 59 | 87 | #if test -e $out/$publish |
| 60 | 88 | # then |
| 61 | | - echo "New reports generated:\n" |
| 62 | | - cat $out/$publish |
| 63 | | - echo "\n\nCopy $lang\n\n" |
| 64 | | - cp $out/* $htdocs -rvf |
| 65 | | - rm $out/$publish |
| | 89 | +# echo2 "New reports generated" |
| | 90 | +# cat $out/$publish |
| | 91 | +# echo2 "Copy $lang" |
| | 92 | +# cp $out/* $htdocs -rvf |
| | 93 | +# rm $out/$publish |
| 66 | 94 | # else echo "No new files, do not publish" |
| 67 | 95 | # fi |
| 68 | 96 | done |
| 69 | 97 | |
| | 98 | +echo2 "" |
| | 99 | +echo2 "Ready" |
| | 100 | +date >> log_publish_sh.txt |
| | 101 | +echo2 "-----" |
| 70 | 102 | |
| | 103 | + |