infra/codestorage/hg-repo-sync
2015-08-17
Child:cc08c8ed2128
infra/codestorage/hg-repo-sync/run.sh
First commit. This contains the basic implementation of hg-repo-sync, which is only useful when you want to do a constant two-way sync every few minutes to Google Cloud Storage. It also hardcodes more than it should. But it's running on code.secondbit.org without issue, so it should probably be preserved.
| paddy@0 | 1 #!/bin/bash |
| paddy@0 | 2 today=$(date +%F) |
| paddy@0 | 3 |
| paddy@0 | 4 # added trying to be clever |
| paddy@0 | 5 yesterday=$(date --date yesterday +%F) |
| paddy@0 | 6 |
| paddy@0 | 7 echo "Fixing permissions..." |
| paddy@0 | 8 chgrp -R 2000 /current |
| paddy@0 | 9 echo "Cleaning up..." |
| paddy@0 | 10 rm -rf /scratch/* |
| paddy@0 | 11 echo "Downloading bundles from Google Cloud Storage..." |
| paddy@0 | 12 output=$(gsutil cp -R gs://backups.code.secondbit.org/$today/\* /scratch 2>&1) |
| paddy@0 | 13 echo $output |
| paddy@0 | 14 |
| paddy@0 | 15 bundles=$(find /scratch -name '*.bundle') |
| paddy@0 | 16 |
| paddy@0 | 17 # added trying to be clever |
| paddy@0 | 18 # basically, if we get an error downloading the bundles, try for yesterday's |
| paddy@0 | 19 # this could happen if the pod restarts between the last upload of day A and the first upload of day B |
| paddy@0 | 20 if [ $output == *"No URLs matched:"* ] |
| paddy@0 | 21 echo "Downloading yesterday's bundles from Google Cloud Storage..." |
| paddy@0 | 22 gsutil cp -R gs://backups.code.secondbit.org/$yesterday/\* /scratch |
| paddy@0 | 23 mv /scratch/$yesterday /scratch/$today |
| paddy@0 | 24 bundles=$(find /scratch -name '*.bundle' | wc -l) |
| paddy@0 | 25 fi |
| paddy@0 | 26 |
| paddy@0 | 27 for bundle in $bundles |
| paddy@0 | 28 do |
| paddy@0 | 29 dir=$(dirname $bundle) |
| paddy@0 | 30 echo "dir=$dir" |
| paddy@0 | 31 stripped=${dir#.} |
| paddy@0 | 32 echo "stripped=$stripped" |
| paddy@0 | 33 stripped=${stripped#/scratch} |
| paddy@0 | 34 echo "stripped=$stripped" |
| paddy@0 | 35 target=${bundle#/scratch} |
| paddy@0 | 36 echo "target=$target" |
| paddy@0 | 37 target=${target%.bundle} |
| paddy@0 | 38 echo "target=$target" |
| paddy@0 | 39 if [ -d "/current${target}" ] |
| paddy@0 | 40 then |
| paddy@0 | 41 echo "Pulling changes from $bundle to /current$target" |
| paddy@0 | 42 hg --cwd /current${target} pull $bundle |
| paddy@0 | 43 else |
| paddy@0 | 44 echo "Creating /current$target repo from $bundle" |
| paddy@0 | 45 hg clone $bundle /current${target} |
| paddy@0 | 46 chgrp -R 2000 /current${target} |
| paddy@0 | 47 fi |
| paddy@0 | 48 done |
| paddy@0 | 49 echo "Cleaning up..." |
| paddy@0 | 50 rm -rf /scratch/* |
| paddy@0 | 51 |
| paddy@0 | 52 repos=$(find /current -name .hg -type d) |
| paddy@0 | 53 for repo in $repos |
| paddy@0 | 54 do |
| paddy@0 | 55 dir=$(dirname $repo) |
| paddy@0 | 56 echo "dir=$dir" |
| paddy@0 | 57 stripped=${dir#.} |
| paddy@0 | 58 echo "stripped=$stripped" |
| paddy@0 | 59 stripped=${stripped#/current} |
| paddy@0 | 60 echo "stripped=$stripped" |
| paddy@0 | 61 target=/scratch/$today$stripped.bundle |
| paddy@0 | 62 echo "target=$target" |
| paddy@0 | 63 mkdir -p $(dirname $target) |
| paddy@0 | 64 echo "Bundling $dir to $target" |
| paddy@0 | 65 hg --cwd $dir bundle --all $target |
| paddy@0 | 66 done |
| paddy@0 | 67 echo "Pushing bundles to Google Cloud Storage..." |
| paddy@0 | 68 gsutil cp -R /scratch/* gs://backups.code.secondbit.org/ |
| paddy@0 | 69 echo "Cleaning up..." |
| paddy@0 | 70 rm -rf /scratch/* |
| paddy@0 | 71 # sleep between 5 and 10 minutes |
| paddy@0 | 72 # randomized to prevent all our servers running this at the same time |
| paddy@0 | 73 sleepfor="$[($RANDOM % 5) + 5]m" |
| paddy@0 | 74 echo "Sleeping for $sleepfor..." |
| paddy@0 | 75 sleep $sleepfor |