infra/codestorage/hg-repo-sync

     1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/run.sh	Mon Aug 17 18:56:19 2015 -0400
     1.3 @@ -0,0 +1,75 @@
     1.4 +#!/bin/bash
     1.5 +today=$(date +%F)
     1.6 +
     1.7 +# added trying to be clever
     1.8 +yesterday=$(date --date yesterday +%F)
     1.9 +
    1.10 +echo "Fixing permissions..."
    1.11 +chgrp -R 2000 /current
    1.12 +echo "Cleaning up..."
    1.13 +rm -rf /scratch/*
    1.14 +echo "Downloading bundles from Google Cloud Storage..."
    1.15 +output=$(gsutil cp -R gs://backups.code.secondbit.org/$today/\* /scratch 2>&1)
    1.16 +echo $output
    1.17 +
    1.18 +bundles=$(find /scratch -name '*.bundle')
    1.19 +
    1.20 +# added trying to be clever
    1.21 +# basically, if we get an error downloading the bundles, try for yesterday's
    1.22 +# this could happen if the pod restarts between the last upload of day A and the first upload of day B
    1.23 +if [ $output == *"No URLs matched:"* ]
    1.24 +	echo "Downloading yesterday's bundles from Google Cloud Storage..."
    1.25 +	gsutil cp -R gs://backups.code.secondbit.org/$yesterday/\* /scratch
    1.26 +	mv /scratch/$yesterday /scratch/$today
    1.27 +	bundles=$(find /scratch -name '*.bundle' | wc -l)
    1.28 +fi
    1.29 +
    1.30 +for bundle in $bundles
    1.31 +do
    1.32 +	dir=$(dirname $bundle)
    1.33 +	echo "dir=$dir"
    1.34 +	stripped=${dir#.}
    1.35 +	echo "stripped=$stripped"
    1.36 +	stripped=${stripped#/scratch}
    1.37 +	echo "stripped=$stripped"
    1.38 +	target=${bundle#/scratch}
    1.39 +	echo "target=$target"
    1.40 +	target=${target%.bundle}
    1.41 +	echo "target=$target"
    1.42 +	if [ -d "/current${target}" ]
    1.43 +	then
    1.44 +		echo "Pulling changes from $bundle to /current$target"
    1.45 +		hg --cwd /current${target} pull $bundle
    1.46 +	else
    1.47 +		echo "Creating /current$target repo from $bundle"
    1.48 +		hg clone $bundle /current${target}
    1.49 +		chgrp -R 2000 /current${target}
    1.50 +	fi
    1.51 +done
    1.52 +echo "Cleaning up..."
    1.53 +rm -rf /scratch/*
    1.54 +
    1.55 +repos=$(find /current -name .hg -type d)
    1.56 +for repo in $repos
    1.57 +do
    1.58 +	dir=$(dirname $repo)
    1.59 +	echo "dir=$dir"
    1.60 +	stripped=${dir#.}
    1.61 +	echo "stripped=$stripped"
    1.62 +	stripped=${stripped#/current}
    1.63 +	echo "stripped=$stripped"
    1.64 +	target=/scratch/$today$stripped.bundle
    1.65 +	echo "target=$target"
    1.66 +	mkdir -p $(dirname $target)
    1.67 +	echo "Bundling $dir to $target"
    1.68 +	hg --cwd $dir bundle --all $target
    1.69 +done
    1.70 +echo "Pushing bundles to Google Cloud Storage..."
    1.71 +gsutil cp -R /scratch/* gs://backups.code.secondbit.org/
    1.72 +echo "Cleaning up..."
    1.73 +rm -rf /scratch/*
    1.74 +# sleep between 5 and 10 minutes
    1.75 +# randomized to prevent all our servers running this at the same time
    1.76 +sleepfor="$[($RANDOM % 5) + 5]m"
    1.77 +echo "Sleeping for $sleepfor..."
    1.78 +sleep $sleepfor