gifs/api

Paddy 2014-10-17 Parent:1bbbe113f599

5:b5d88d57d587 Go to Latest

gifs/api/storage.go

Simplify upload. Simplify the upload code by not running the hashing async, which requires fewer copy operations and less channel synchronization. Also, take advantage of the fact that PipeWriters and PipeReaders will return an error to the PipeReaders/PipeWriters (respectively) when read/write is called (respectively) to avoid passing back errors through channels.

Download raw file
View source Diff to previous Annotate
History
paddy@0 1 package api
paddy@0 2
paddy@0 3 import (
paddy@0 4 "errors"
paddy@0 5 "io"
paddy@0 6 "io/ioutil"
paddy@0 7 "net/http"
paddy@4 8 "sync"
paddy@0 9
paddy@0 10 "code.google.com/p/google-api-go-client/googleapi"
paddy@4 11 "code.google.com/p/google-api-go-client/storage/v1"
paddy@0 12 )
paddy@0 13
paddy@0 14 var (
paddy@4 15 ErrBucketNotFound = errors.New("bucket not found")
paddy@4 16 ErrBlobNotFound = errors.New("blob not found")
paddy@0 17 )
paddy@0 18
paddy@0 19 type Storage interface {
paddy@4 20 Upload(bucket, tmp string, r io.Reader) error
paddy@0 21 Delete(bucket, tmp string) error
paddy@0 22 Move(srcBucket, src, dstBucket, dst string) error
paddy@0 23 Download(bucket, id string, w io.Writer) (int64, error)
paddy@0 24 }
paddy@0 25
paddy@4 26 type memstorage struct {
paddy@4 27 buckets map[string]membucket
paddy@4 28 sync.RWMutex
paddy@4 29 }
paddy@4 30
paddy@4 31 type membucket map[string][]byte
paddy@4 32
paddy@4 33 func NewMemStorage() *memstorage {
paddy@4 34 return &memstorage{
paddy@4 35 buckets: map[string]membucket{},
paddy@4 36 }
paddy@4 37 }
paddy@4 38
paddy@4 39 func (m *memstorage) Upload(bucket, tmp string, r io.Reader) error {
paddy@4 40 m.Lock()
paddy@4 41 defer m.Unlock()
paddy@4 42
paddy@4 43 if _, ok := m.buckets[bucket]; !ok {
paddy@4 44 m.buckets[bucket] = membucket{}
paddy@4 45 }
paddy@4 46 bytes, err := ioutil.ReadAll(r)
paddy@4 47 if err != nil {
paddy@4 48 return err
paddy@4 49 }
paddy@4 50 m.buckets[bucket][tmp] = bytes
paddy@4 51 return nil
paddy@4 52 }
paddy@4 53
paddy@4 54 func (m *memstorage) Delete(bucket, tmp string) error {
paddy@4 55 m.Lock()
paddy@4 56 defer m.Unlock()
paddy@4 57
paddy@4 58 delete(m.buckets[bucket], tmp)
paddy@4 59 return nil
paddy@4 60 }
paddy@4 61
paddy@4 62 func (m *memstorage) Move(srcBucket, src, dstBucket, dst string) error {
paddy@4 63 m.Lock()
paddy@4 64 defer m.Unlock()
paddy@4 65
paddy@4 66 if _, ok := m.buckets[srcBucket]; !ok {
paddy@4 67 return ErrBucketNotFound
paddy@4 68 }
paddy@4 69 if _, ok := m.buckets[srcBucket][src]; !ok {
paddy@4 70 return ErrBlobNotFound
paddy@4 71 }
paddy@4 72 if _, ok := m.buckets[dstBucket]; !ok {
paddy@4 73 m.buckets[dstBucket] = membucket{}
paddy@4 74 }
paddy@4 75 m.buckets[dstBucket][dst] = m.buckets[srcBucket][src]
paddy@4 76 return m.Delete(srcBucket, src)
paddy@4 77 }
paddy@4 78
paddy@4 79 func (m *memstorage) Download(bucket, id string, w io.Writer) (int64, error) {
paddy@4 80 m.RLock()
paddy@4 81 defer m.RUnlock()
paddy@4 82
paddy@4 83 if _, ok := m.buckets[bucket]; !ok {
paddy@4 84 return 0, ErrBucketNotFound
paddy@4 85 }
paddy@4 86 if _, ok := m.buckets[bucket][id]; !ok {
paddy@4 87 return 0, ErrBlobNotFound
paddy@4 88 }
paddy@4 89 n, err := w.Write(m.buckets[bucket][id])
paddy@4 90 return int64(n), err
paddy@4 91 }
paddy@4 92
paddy@0 93 type GoogleCloudStorage struct {
paddy@0 94 *storage.Service
paddy@0 95 }
paddy@0 96
paddy@4 97 func (gcs *GoogleCloudStorage) Upload(bucket, tmp string, r io.Reader) error {
paddy@0 98 object := &storage.Object{Name: tmp}
paddy@0 99 _, err := gcs.Objects.Insert(bucket, object).Media(r).Do()
paddy@4 100 if err != nil {
paddy@4 101 // TODO: abstract out this error
paddy@4 102 return err
paddy@0 103 }
paddy@4 104 return nil
paddy@0 105 }
paddy@0 106
paddy@0 107 func (gcs *GoogleCloudStorage) Delete(bucket, tmp string) error {
paddy@4 108 // TODO: abstract out this error
paddy@0 109 return gcs.Objects.Delete(bucket, tmp).Do()
paddy@0 110 }
paddy@0 111
paddy@0 112 func (gcs *GoogleCloudStorage) Move(srcBucket, src, dstBucket, dst string) error {
paddy@0 113 _, err := gcs.Objects.Get(dstBucket, dst).Do()
paddy@0 114 if err == nil {
paddy@0 115 return nil
paddy@0 116 }
paddy@4 117 // ignore 404 errors on the destination; we don't expect it to exist, after all
paddy@0 118 if e, ok := err.(*googleapi.Error); !ok || e.Code != 404 {
paddy@4 119 // TODO: abstract out this error
paddy@0 120 return err
paddy@0 121 }
paddy@0 122 _, err = gcs.Objects.Copy(srcBucket, src, dstBucket, dst, nil).Do()
paddy@0 123 if err != nil {
paddy@4 124 // TODO: abstract out this error
paddy@0 125 return err
paddy@0 126 }
paddy@0 127 objectAcl := &storage.ObjectAccessControl{
paddy@0 128 Bucket: dstBucket, Entity: "allUsers", Object: dst, Role: "READER",
paddy@0 129 }
paddy@0 130 _, err = gcs.ObjectAccessControls.Insert(dstBucket, dst, objectAcl).Do()
paddy@0 131 if err != nil {
paddy@4 132 // TODO: abstract out this error
paddy@0 133 return err
paddy@0 134 }
paddy@0 135 return nil
paddy@0 136 }
paddy@0 137
paddy@0 138 func (gcs *GoogleCloudStorage) Download(bucket, id string, w io.Writer) (int64, error) {
paddy@0 139 res, err := gcs.Objects.Get(bucket, id).Do()
paddy@0 140 if err != nil {
paddy@4 141 // TODO: abstract out this error
paddy@0 142 return 0, err
paddy@0 143 }
paddy@0 144 resp, err := http.Get(res.MediaLink)
paddy@0 145 if err != nil {
paddy@4 146 // TODO: abstract out this error
paddy@0 147 return 0, err
paddy@0 148 }
paddy@0 149 defer resp.Body.Close()
paddy@4 150 // TODO: abstract out this error
paddy@0 151 return io.Copy(w, resp.Body)
paddy@0 152 }