gifs/api

Paddy 2014-08-27 Child:1bbbe113f599

0:08ec88016e2f Go to Latest

gifs/api/storage.go

Spike out functionality and tests. Create our interfaces around storing data and retrieving it. Create an in-memory implementation of our interfaces, for testing and rapid dev purposes. Begin sketching out what our unit tests look like. Create our Google Cloud Storage datastore implementation. Sketch out an idea for a usage collection process to keep track of which users are actually using stuff.

Download raw file
View source Diff to previous Annotate
History
paddy@0 1 package api
paddy@0 2
paddy@0 3 import (
paddy@0 4 "errors"
paddy@0 5 "io"
paddy@0 6 "io/ioutil"
paddy@0 7 "log"
paddy@0 8 "net/http"
paddy@0 9
paddy@0 10 "code.google.com/p/google-api-go-client/googleapi"
paddy@0 11 "code.google.com/p/google-api-go-client/storage/v1beta2"
paddy@0 12 )
paddy@0 13
paddy@0 14 var (
paddy@0 15 BucketNotFoundError = errors.New("bucket not found")
paddy@0 16 BlobNotFoundError = errors.New("blob not found")
paddy@0 17 )
paddy@0 18
paddy@0 19 type Storage interface {
paddy@0 20 Upload(bucket, tmp string, r io.Reader, errs chan error, done chan struct{})
paddy@0 21 Delete(bucket, tmp string) error
paddy@0 22 Move(srcBucket, src, dstBucket, dst string) error
paddy@0 23 Download(bucket, id string, w io.Writer) (int64, error)
paddy@0 24 }
paddy@0 25
paddy@0 26 type GoogleCloudStorage struct {
paddy@0 27 *storage.Service
paddy@0 28 }
paddy@0 29
paddy@0 30 func (gcs *GoogleCloudStorage) Upload(bucket, tmp string, r io.Reader, errs chan error, done chan struct{}) {
paddy@0 31 if errs != nil {
paddy@0 32 defer close(errs)
paddy@0 33 }
paddy@0 34 if done != nil {
paddy@0 35 defer close(done)
paddy@0 36 }
paddy@0 37 object := &storage.Object{Name: tmp}
paddy@0 38 _, err := gcs.Objects.Insert(bucket, object).Media(r).Do()
paddy@0 39 if err != nil && errs != nil {
paddy@0 40 errs <- err
paddy@0 41 }
paddy@0 42 }
paddy@0 43
paddy@0 44 func (gcs *GoogleCloudStorage) Delete(bucket, tmp string) error {
paddy@0 45 return gcs.Objects.Delete(bucket, tmp).Do()
paddy@0 46 }
paddy@0 47
paddy@0 48 func (gcs *GoogleCloudStorage) Move(srcBucket, src, dstBucket, dst string) error {
paddy@0 49 _, err := gcs.Objects.Get(dstBucket, dst).Do()
paddy@0 50 if err == nil {
paddy@0 51 go gcs.del(srcBucket, src)
paddy@0 52 return nil
paddy@0 53 }
paddy@0 54 if e, ok := err.(*googleapi.Error); !ok || e.Code != 404 {
paddy@0 55 return err
paddy@0 56 }
paddy@0 57 _, err = gcs.Objects.Copy(srcBucket, src, dstBucket, dst, nil).Do()
paddy@0 58 if err != nil {
paddy@0 59 return err
paddy@0 60 }
paddy@0 61 objectAcl := &storage.ObjectAccessControl{
paddy@0 62 Bucket: dstBucket, Entity: "allUsers", Object: dst, Role: "READER",
paddy@0 63 }
paddy@0 64 _, err = gcs.ObjectAccessControls.Insert(dstBucket, dst, objectAcl).Do()
paddy@0 65 if err != nil {
paddy@0 66 return err
paddy@0 67 }
paddy@0 68 go gcs.del(srcBucket, src)
paddy@0 69 return nil
paddy@0 70 }
paddy@0 71
paddy@0 72 func (gcs *GoogleCloudStorage) del(bucket, tmp string) {
paddy@0 73 err := gcs.Delete(bucket, tmp)
paddy@0 74 if err != nil {
paddy@0 75 log.Printf("Error deleting temporary upload %s in %s: %s\n", tmp, bucket, err)
paddy@0 76 }
paddy@0 77 }
paddy@0 78
paddy@0 79 func (gcs *GoogleCloudStorage) Download(bucket, id string, w io.Writer) (int64, error) {
paddy@0 80 res, err := gcs.Objects.Get(bucket, id).Do()
paddy@0 81 if err != nil {
paddy@0 82 return 0, err
paddy@0 83 }
paddy@0 84 resp, err := http.Get(res.MediaLink)
paddy@0 85 if err != nil {
paddy@0 86 return 0, err
paddy@0 87 }
paddy@0 88 defer resp.Body.Close()
paddy@0 89 return io.Copy(w, resp.Body)
paddy@0 90 }
paddy@0 91
paddy@0 92 type Memstorage map[string]Bucket
paddy@0 93
paddy@0 94 type Bucket map[string][]byte
paddy@0 95
paddy@0 96 func (m Memstorage) Upload(bucket, tmp string, r io.Reader, errs chan error, done chan struct{}) {
paddy@0 97 if errs != nil {
paddy@0 98 defer close(errs)
paddy@0 99 }
paddy@0 100 if done != nil {
paddy@0 101 defer close(done)
paddy@0 102 }
paddy@0 103 if _, ok := m[bucket]; !ok {
paddy@0 104 m[bucket] = make(Bucket)
paddy@0 105 }
paddy@0 106 bytes, err := ioutil.ReadAll(r)
paddy@0 107 if err != nil {
paddy@0 108 errs <- err
paddy@0 109 return
paddy@0 110 }
paddy@0 111 m[bucket][tmp] = bytes
paddy@0 112 }
paddy@0 113
paddy@0 114 func (m Memstorage) Delete(bucket, tmp string) error {
paddy@0 115 delete(m[bucket], tmp)
paddy@0 116 return nil
paddy@0 117 }
paddy@0 118
paddy@0 119 func (m Memstorage) Move(srcBucket, src, dstBucket, dst string) error {
paddy@0 120 if _, ok := m[srcBucket]; !ok {
paddy@0 121 return BucketNotFoundError
paddy@0 122 }
paddy@0 123 if _, ok := m[srcBucket][src]; !ok {
paddy@0 124 return BlobNotFoundError
paddy@0 125 }
paddy@0 126 if _, ok := m[dstBucket]; !ok {
paddy@0 127 m[dstBucket] = make(Bucket)
paddy@0 128 }
paddy@0 129 m[dstBucket][dst] = m[srcBucket][src]
paddy@0 130 return m.Delete(srcBucket, src)
paddy@0 131 }
paddy@0 132
paddy@0 133 func (m Memstorage) Download(bucket, id string, w io.Writer) (int64, error) {
paddy@0 134 if _, ok := m[bucket]; !ok {
paddy@0 135 return 0, BucketNotFoundError
paddy@0 136 }
paddy@0 137 if _, ok := m[bucket][id]; !ok {
paddy@0 138 return 0, BlobNotFoundError
paddy@0 139 }
paddy@0 140 n, err := w.Write(m[bucket][id])
paddy@0 141 return int64(n), err
paddy@0 142 }