infra/codestorage/hg-repo-sync
infra/codestorage/hg-repo-sync/pull.sh
Use pip to install Mercurial, pin to 3.5.2 We want to use a relatively recent (but consistent) version of Mercurial, so I had to switch to using pip to install Mercurial, so I could pin it to version 3.5.2. apt-get didn't have version 3 available.
| paddy@1 | 1 #!/bin/bash |
| paddy@1 | 2 today=$(date +%F) |
| paddy@1 | 3 |
| paddy@1 | 4 yesterday=$(date --date yesterday +%F) |
| paddy@1 | 5 |
| paddy@1 | 6 DOMAIN=${DOMAIN:-code.secondbit.org} |
| paddy@1 | 7 BACKUP_BUCKET=${BACKUP_BUCKET:-backups.$DOMAIN} |
| paddy@1 | 8 |
| paddy@1 | 9 echo "Fixing permissions..." |
| paddy@1 | 10 chgrp -R 2000 /mounted/repos |
| paddy@1 | 11 |
| paddy@1 | 12 echo "Cleaning up..." |
| paddy@1 | 13 rm -rf /tmp/repos/* |
| paddy@1 | 14 |
| paddy@1 | 15 echo "Downloading bundles from gs://${BACKUP_BUCKET}/${today}" |
| paddy@1 | 16 |
| paddy@1 | 17 output=$(gsutil cp -R gs://$BACKUP_BUCKET/$today/\* /tmp/repos 2>&1) |
| paddy@1 | 18 echo $output |
| paddy@1 | 19 |
| paddy@1 | 20 bundles=$(find /tmp/repos -name '*.bundle') |
| paddy@1 | 21 |
| paddy@1 | 22 # basically, if we get an error downloading the bundles, try for yesterday's |
| paddy@1 | 23 # this could happen if the pod restarts between the last upload of day A and the first upload of day B |
| paddy@1 | 24 if [[ $output == *"No URLs matched:"* ]] |
| paddy@1 | 25 then |
| paddy@1 | 26 echo "Downloading yesterday's bundles from gs://${BACKUP_BUCKET}/${yesterday}" |
| paddy@1 | 27 gsutil cp -R gs://$BACKUP_BUCKET/$yesterday/\* /tmp/repos |
| paddy@1 | 28 mv /tmp/repos/$yesterday /tmp/repos/$today |
| paddy@1 | 29 bundles=$(find /tmp/repos -name '*.bundle') |
| paddy@1 | 30 fi |
| paddy@1 | 31 |
| paddy@1 | 32 for bundle in $bundles |
| paddy@1 | 33 do |
| paddy@1 | 34 dir=$(dirname $bundle) |
| paddy@1 | 35 stripped=${dir#.} |
| paddy@1 | 36 stripped=${stripped#/tmp/repos} |
| paddy@1 | 37 target=${bundle#/tmp/repos} |
| paddy@1 | 38 target=${target%.bundle} |
| paddy@1 | 39 if [ -d "/mounted/repos${target}" ] |
| paddy@1 | 40 then |
| paddy@1 | 41 echo "Pulling changes from $bundle to /mounted/repos$target" |
| paddy@1 | 42 hg --cwd /mounted/repos${target} pull $bundle |
| paddy@1 | 43 else |
| paddy@1 | 44 echo "Creating /mounted/repos$target repo from $bundle" |
| paddy@1 | 45 hg clone $bundle /mounted/repos${target} |
| paddy@1 | 46 chgrp -R 2000 /mounted/repos${target} |
| paddy@1 | 47 fi |
| paddy@1 | 48 done |
| paddy@1 | 49 |
| paddy@1 | 50 echo "Cleaning up..." |
| paddy@1 | 51 rm -rf /tmp/repos/* |
| paddy@1 | 52 |
| paddy@1 | 53 echo "Pull complete." |