infra/codestorage/hg-repo-sync
2015-08-17
Child:cc08c8ed2128
infra/codestorage/hg-repo-sync/run.sh
First commit. This contains the basic implementation of hg-repo-sync, which is only useful when you want to do a constant two-way sync every few minutes to Google Cloud Storage. It also hardcodes more than it should. But it's running on code.secondbit.org without issue, so it should probably be preserved.
1 #!/bin/bash
2 today=$(date +%F)
4 # added trying to be clever
5 yesterday=$(date --date yesterday +%F)
7 echo "Fixing permissions..."
8 chgrp -R 2000 /current
9 echo "Cleaning up..."
10 rm -rf /scratch/*
11 echo "Downloading bundles from Google Cloud Storage..."
12 output=$(gsutil cp -R gs://backups.code.secondbit.org/$today/\* /scratch 2>&1)
13 echo $output
15 bundles=$(find /scratch -name '*.bundle')
17 # added trying to be clever
18 # basically, if we get an error downloading the bundles, try for yesterday's
19 # this could happen if the pod restarts between the last upload of day A and the first upload of day B
20 if [ $output == *"No URLs matched:"* ]
21 echo "Downloading yesterday's bundles from Google Cloud Storage..."
22 gsutil cp -R gs://backups.code.secondbit.org/$yesterday/\* /scratch
23 mv /scratch/$yesterday /scratch/$today
24 bundles=$(find /scratch -name '*.bundle' | wc -l)
25 fi
27 for bundle in $bundles
28 do
29 dir=$(dirname $bundle)
30 echo "dir=$dir"
31 stripped=${dir#.}
32 echo "stripped=$stripped"
33 stripped=${stripped#/scratch}
34 echo "stripped=$stripped"
35 target=${bundle#/scratch}
36 echo "target=$target"
37 target=${target%.bundle}
38 echo "target=$target"
39 if [ -d "/current${target}" ]
40 then
41 echo "Pulling changes from $bundle to /current$target"
42 hg --cwd /current${target} pull $bundle
43 else
44 echo "Creating /current$target repo from $bundle"
45 hg clone $bundle /current${target}
46 chgrp -R 2000 /current${target}
47 fi
48 done
49 echo "Cleaning up..."
50 rm -rf /scratch/*
52 repos=$(find /current -name .hg -type d)
53 for repo in $repos
54 do
55 dir=$(dirname $repo)
56 echo "dir=$dir"
57 stripped=${dir#.}
58 echo "stripped=$stripped"
59 stripped=${stripped#/current}
60 echo "stripped=$stripped"
61 target=/scratch/$today$stripped.bundle
62 echo "target=$target"
63 mkdir -p $(dirname $target)
64 echo "Bundling $dir to $target"
65 hg --cwd $dir bundle --all $target
66 done
67 echo "Pushing bundles to Google Cloud Storage..."
68 gsutil cp -R /scratch/* gs://backups.code.secondbit.org/
69 echo "Cleaning up..."
70 rm -rf /scratch/*
71 # sleep between 5 and 10 minutes
72 # randomized to prevent all our servers running this at the same time
73 sleepfor="$[($RANDOM % 5) + 5]m"
74 echo "Sleeping for $sleepfor..."
75 sleep $sleepfor