gifs/api

Paddy 2014-10-17 Parent:08ec88016e2f

4:1bbbe113f599 Go to Latest

gifs/api/storage.go

Upload is no longer async, memstorage is parallel-safe. Upload no longer needs to be run async (it can be run inside a goroutine), so it now returns stuff instead of taking a channel as an argument. This will make it easier to implement, as all the async stuff is an abstraction above, and therefore doesn't need to be worried about for each reimplementation. The memstorage type is no longer exported and can now be safely used by multiple goroutines, thanks to the sync package.

Download raw file
View source Diff to previous Annotate
History
paddy@0 1 package api
paddy@0 2
paddy@0 3 import (
paddy@0 4 "errors"
paddy@0 5 "io"
paddy@0 6 "io/ioutil"
paddy@0 7 "net/http"
paddy@4 8 "sync"
paddy@0 9
paddy@0 10 "code.google.com/p/google-api-go-client/googleapi"
paddy@4 11 "code.google.com/p/google-api-go-client/storage/v1"
paddy@0 12 )
paddy@0 13
paddy@0 14 var (
paddy@4 15 ErrBucketNotFound = errors.New("bucket not found")
paddy@4 16 ErrBlobNotFound = errors.New("blob not found")
paddy@0 17 )
paddy@0 18
paddy@0 19 type Storage interface {
paddy@4 20 Upload(bucket, tmp string, r io.Reader) error
paddy@0 21 Delete(bucket, tmp string) error
paddy@0 22 Move(srcBucket, src, dstBucket, dst string) error
paddy@0 23 Download(bucket, id string, w io.Writer) (int64, error)
paddy@0 24 }
paddy@0 25
paddy@4 26 type memstorage struct {
paddy@4 27 buckets map[string]membucket
paddy@4 28 sync.RWMutex
paddy@4 29 }
paddy@4 30
paddy@4 31 type membucket map[string][]byte
paddy@4 32
paddy@4 33 func NewMemStorage() *memstorage {
paddy@4 34 return &memstorage{
paddy@4 35 buckets: map[string]membucket{},
paddy@4 36 }
paddy@4 37 }
paddy@4 38
paddy@4 39 func (m *memstorage) Upload(bucket, tmp string, r io.Reader) error {
paddy@4 40 m.Lock()
paddy@4 41 defer m.Unlock()
paddy@4 42
paddy@4 43 if _, ok := m.buckets[bucket]; !ok {
paddy@4 44 m.buckets[bucket] = membucket{}
paddy@4 45 }
paddy@4 46 bytes, err := ioutil.ReadAll(r)
paddy@4 47 if err != nil {
paddy@4 48 return err
paddy@4 49 }
paddy@4 50 m.buckets[bucket][tmp] = bytes
paddy@4 51 return nil
paddy@4 52 }
paddy@4 53
paddy@4 54 func (m *memstorage) Delete(bucket, tmp string) error {
paddy@4 55 m.Lock()
paddy@4 56 defer m.Unlock()
paddy@4 57
paddy@4 58 delete(m.buckets[bucket], tmp)
paddy@4 59 return nil
paddy@4 60 }
paddy@4 61
paddy@4 62 func (m *memstorage) Move(srcBucket, src, dstBucket, dst string) error {
paddy@4 63 m.Lock()
paddy@4 64 defer m.Unlock()
paddy@4 65
paddy@4 66 if _, ok := m.buckets[srcBucket]; !ok {
paddy@4 67 return ErrBucketNotFound
paddy@4 68 }
paddy@4 69 if _, ok := m.buckets[srcBucket][src]; !ok {
paddy@4 70 return ErrBlobNotFound
paddy@4 71 }
paddy@4 72 if _, ok := m.buckets[dstBucket]; !ok {
paddy@4 73 m.buckets[dstBucket] = membucket{}
paddy@4 74 }
paddy@4 75 m.buckets[dstBucket][dst] = m.buckets[srcBucket][src]
paddy@4 76 return m.Delete(srcBucket, src)
paddy@4 77 }
paddy@4 78
paddy@4 79 func (m *memstorage) Download(bucket, id string, w io.Writer) (int64, error) {
paddy@4 80 m.RLock()
paddy@4 81 defer m.RUnlock()
paddy@4 82
paddy@4 83 if _, ok := m.buckets[bucket]; !ok {
paddy@4 84 return 0, ErrBucketNotFound
paddy@4 85 }
paddy@4 86 if _, ok := m.buckets[bucket][id]; !ok {
paddy@4 87 return 0, ErrBlobNotFound
paddy@4 88 }
paddy@4 89 n, err := w.Write(m.buckets[bucket][id])
paddy@4 90 return int64(n), err
paddy@4 91 }
paddy@4 92
paddy@0 93 type GoogleCloudStorage struct {
paddy@0 94 *storage.Service
paddy@0 95 }
paddy@0 96
paddy@4 97 func (gcs *GoogleCloudStorage) Upload(bucket, tmp string, r io.Reader) error {
paddy@0 98 object := &storage.Object{Name: tmp}
paddy@0 99 _, err := gcs.Objects.Insert(bucket, object).Media(r).Do()
paddy@4 100 if err != nil {
paddy@4 101 // TODO: abstract out this error
paddy@4 102 return err
paddy@0 103 }
paddy@4 104 return nil
paddy@0 105 }
paddy@0 106
paddy@0 107 func (gcs *GoogleCloudStorage) Delete(bucket, tmp string) error {
paddy@4 108 // TODO: abstract out this error
paddy@0 109 return gcs.Objects.Delete(bucket, tmp).Do()
paddy@0 110 }
paddy@0 111
paddy@0 112 func (gcs *GoogleCloudStorage) Move(srcBucket, src, dstBucket, dst string) error {
paddy@0 113 _, err := gcs.Objects.Get(dstBucket, dst).Do()
paddy@0 114 if err == nil {
paddy@0 115 return nil
paddy@0 116 }
paddy@4 117 // ignore 404 errors on the destination; we don't expect it to exist, after all
paddy@0 118 if e, ok := err.(*googleapi.Error); !ok || e.Code != 404 {
paddy@4 119 // TODO: abstract out this error
paddy@0 120 return err
paddy@0 121 }
paddy@0 122 _, err = gcs.Objects.Copy(srcBucket, src, dstBucket, dst, nil).Do()
paddy@0 123 if err != nil {
paddy@4 124 // TODO: abstract out this error
paddy@0 125 return err
paddy@0 126 }
paddy@0 127 objectAcl := &storage.ObjectAccessControl{
paddy@0 128 Bucket: dstBucket, Entity: "allUsers", Object: dst, Role: "READER",
paddy@0 129 }
paddy@0 130 _, err = gcs.ObjectAccessControls.Insert(dstBucket, dst, objectAcl).Do()
paddy@0 131 if err != nil {
paddy@4 132 // TODO: abstract out this error
paddy@0 133 return err
paddy@0 134 }
paddy@0 135 return nil
paddy@0 136 }
paddy@0 137
paddy@0 138 func (gcs *GoogleCloudStorage) Download(bucket, id string, w io.Writer) (int64, error) {
paddy@0 139 res, err := gcs.Objects.Get(bucket, id).Do()
paddy@0 140 if err != nil {
paddy@4 141 // TODO: abstract out this error
paddy@0 142 return 0, err
paddy@0 143 }
paddy@0 144 resp, err := http.Get(res.MediaLink)
paddy@0 145 if err != nil {
paddy@4 146 // TODO: abstract out this error
paddy@0 147 return 0, err
paddy@0 148 }
paddy@0 149 defer resp.Body.Close()
paddy@4 150 // TODO: abstract out this error
paddy@0 151 return io.Copy(w, resp.Body)
paddy@0 152 }