infra/codestorage/hg-repo-sync
infra/codestorage/hg-repo-sync/pull.sh
Update to be more modular. We now have a more modular approach to running our mercurial servers, and the approach to the individual containers is a bit more modular as well. The hg-repo-sync container now has a few responsibilities: 1. Install Mercurial 2. Create the group that’s going to be writing all our files 3. Create the folders we’ll store files in, and make the new group the owner 4. Define a script that pulls repos from GCS to local disk 5. Define a script that pushes repos to GCS from local disk 6. Define a script that causes a delay between runs This mainly involved breaking the previous run script into a few separate scripts, so they could be invoked individually. This docker container _can_ be used as a standalone image that will keep the local disk in sync with GCS, but it’s more useful as a base image for our other containers to work off of to achieve specialised functionality.
| paddy@1 | 1 #!/bin/bash |
| paddy@1 | 2 today=$(date +%F) |
| paddy@1 | 3 |
| paddy@1 | 4 yesterday=$(date --date yesterday +%F) |
| paddy@1 | 5 |
| paddy@1 | 6 DOMAIN=${DOMAIN:-code.secondbit.org} |
| paddy@1 | 7 BACKUP_BUCKET=${BACKUP_BUCKET:-backups.$DOMAIN} |
| paddy@1 | 8 |
| paddy@1 | 9 echo "Fixing permissions..." |
| paddy@1 | 10 chgrp -R 2000 /mounted/repos |
| paddy@1 | 11 |
| paddy@1 | 12 echo "Cleaning up..." |
| paddy@1 | 13 rm -rf /tmp/repos/* |
| paddy@1 | 14 |
| paddy@1 | 15 echo "Downloading bundles from gs://${BACKUP_BUCKET}/${today}" |
| paddy@1 | 16 |
| paddy@1 | 17 output=$(gsutil cp -R gs://$BACKUP_BUCKET/$today/\* /tmp/repos 2>&1) |
| paddy@1 | 18 echo $output |
| paddy@1 | 19 |
| paddy@1 | 20 bundles=$(find /tmp/repos -name '*.bundle') |
| paddy@1 | 21 |
| paddy@1 | 22 # basically, if we get an error downloading the bundles, try for yesterday's |
| paddy@1 | 23 # this could happen if the pod restarts between the last upload of day A and the first upload of day B |
| paddy@1 | 24 if [[ $output == *"No URLs matched:"* ]] |
| paddy@1 | 25 then |
| paddy@1 | 26 echo "Downloading yesterday's bundles from gs://${BACKUP_BUCKET}/${yesterday}" |
| paddy@1 | 27 gsutil cp -R gs://$BACKUP_BUCKET/$yesterday/\* /tmp/repos |
| paddy@1 | 28 mv /tmp/repos/$yesterday /tmp/repos/$today |
| paddy@1 | 29 bundles=$(find /tmp/repos -name '*.bundle') |
| paddy@1 | 30 fi |
| paddy@1 | 31 |
| paddy@1 | 32 for bundle in $bundles |
| paddy@1 | 33 do |
| paddy@1 | 34 dir=$(dirname $bundle) |
| paddy@1 | 35 stripped=${dir#.} |
| paddy@1 | 36 stripped=${stripped#/tmp/repos} |
| paddy@1 | 37 target=${bundle#/tmp/repos} |
| paddy@1 | 38 target=${target%.bundle} |
| paddy@1 | 39 if [ -d "/mounted/repos${target}" ] |
| paddy@1 | 40 then |
| paddy@1 | 41 echo "Pulling changes from $bundle to /mounted/repos$target" |
| paddy@1 | 42 hg --cwd /mounted/repos${target} pull $bundle |
| paddy@1 | 43 else |
| paddy@1 | 44 echo "Creating /mounted/repos$target repo from $bundle" |
| paddy@1 | 45 hg clone $bundle /mounted/repos${target} |
| paddy@1 | 46 chgrp -R 2000 /mounted/repos${target} |
| paddy@1 | 47 fi |
| paddy@1 | 48 done |
| paddy@1 | 49 |
| paddy@1 | 50 echo "Cleaning up..." |
| paddy@1 | 51 rm -rf /tmp/repos/* |
| paddy@1 | 52 |
| paddy@1 | 53 echo "Pull complete." |