mirrors
/
gotosocial
mirror of
1
Fork 0
Code Issues Releases Activity

[chore] use kv.KVStore also for S3 storage (#1113) 

* replace s3 storage implementation to also use kv.KVStore

Signed-off-by: kim <grufwub@gmail.com>

* pull in latest `go-store` fix

Signed-off-by: kim <grufwub@gmail.com>

* pull-in go-store v2.0.9 fixes, update s3 put chunk size to 5MiB

Signed-off-by: kim <grufwub@gmail.com>

Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2022-11-22 13:28:55 +00:00 committed by GitHub
parent 6d43319fe1
commit a898160b0c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 85 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -12,7 +12,7 @@ require (
codeberg.org/gruf/go-logger/v2 v2.2.1
codeberg.org/gruf/go-mutexes v1.1.4
codeberg.org/gruf/go-runners v1.3.1
codeberg.org/gruf/go-store/v2 v2.0.7
codeberg.org/gruf/go-store/v2 v2.0.9
github.com/buckket/go-blurhash v1.1.0
github.com/coreos/go-oidc/v3 v3.4.0
github.com/cornelk/hashmap v1.0.8

4
go.sum
View File

@ -99,8 +99,8 @@ codeberg.org/gruf/go-runners v1.3.1 h1:d/OQMMMiA6yPaDSbSr0/Jc+lucWmm7AiAZjWffpNK
codeberg.org/gruf/go-runners v1.3.1/go.mod h1:rl0EdZNozkRMb21DAtOL5L4oTfmslYQdZgq2RMMc/H4=
codeberg.org/gruf/go-sched v1.1.1 h1:YtLSQhpypzuD3HTup5oF7LLWB79gTL4nqW06kH4Vwks=
codeberg.org/gruf/go-sched v1.1.1/go.mod h1:SRcdP/5qim+EBT3n3r4aUra1C30yPqV4OJOXuqvgdQM=
codeberg.org/gruf/go-store/v2 v2.0.7 h1:P+0d8jnXdgzxfHLqKjHMV+MAxVJmq056PvzaHRyR8jE=
codeberg.org/gruf/go-store/v2 v2.0.7/go.mod h1:D4r5PV0BXDhxQyATw/03JkwvziZDkVMgzTpElZyWTXI=
codeberg.org/gruf/go-store/v2 v2.0.9 h1:nFurtZCBdZac79aXo3PFPVn2N744B06dHLygVrHf+SA=
codeberg.org/gruf/go-store/v2 v2.0.9/go.mod h1:D4r5PV0BXDhxQyATw/03JkwvziZDkVMgzTpElZyWTXI=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=

68
internal/storage/s3.go
View File

@ -19,80 +19,32 @@
package storage
import (
"bytes"
"context"
"fmt"
"io"
"mime"
"net/url"
"path"
"time"
"github.com/minio/minio-go/v7"
"codeberg.org/gruf/go-store/v2/kv"
"codeberg.org/gruf/go-store/v2/storage"
)
type S3 struct {
mc *minio.Client
bucket string
proxy bool
}
func NewS3(mc *minio.Client, bucket string, proxy bool) *S3 {
return &S3{
mc: mc,
bucket: bucket,
proxy: proxy,
}
}
func (s *S3) Get(ctx context.Context, key string) ([]byte, error) {
r, err := s.GetStream(ctx, key)
if err != nil {
return nil, err
}
defer r.Close()
b, err := io.ReadAll(r)
if err != nil {
return nil, fmt.Errorf("reading data from s3: %w", err)
}
return b, nil
}
func (s *S3) GetStream(ctx context.Context, key string) (io.ReadCloser, error) {
o, err := s.mc.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
if err != nil {
err = fmt.Errorf("retrieving object from s3: %w", err)
}
return o, err
}
func (s *S3) PutStream(ctx context.Context, key string, r io.Reader) error {
if _, err := s.mc.PutObject(ctx, s.bucket, key, r, -1, minio.PutObjectOptions{}); err != nil {
return fmt.Errorf("uploading data stream: %w", err)
}
return nil
}
func (s *S3) Put(ctx context.Context, key string, value []byte) error {
if _, err := s.mc.PutObject(ctx, s.bucket, key, bytes.NewBuffer(value), -1, minio.PutObjectOptions{}); err != nil {
return fmt.Errorf("uploading data slice: %w", err)
}
return nil
}
func (s *S3) Delete(ctx context.Context, key string) error {
return s.mc.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
Proxy bool
Bucket string
Storage *storage.S3Storage
*kv.KVStore
}
func (s *S3) URL(ctx context.Context, key string) *url.URL {
if s.proxy {
if s.Proxy {
return nil
}
// it's safe to ignore the error here, as we just fall back to fetching the
// file if the url request fails
url, _ := s.mc.PresignedGetObject(ctx, s.bucket, key, time.Hour, url.Values{
// it's safe to ignore the error here, as we just fall back to fetching the file if URL request fails
url, _ := s.Storage.Client().PresignedGetObject(ctx, s.Bucket, key, time.Hour, url.Values{
"response-content-type": []string{mime.TypeByExtension(path.Ext(key))},
})
return url
}

39
internal/storage/storage.go
View File

@ -52,20 +52,38 @@ type Driver interface {
func AutoConfig() (Driver, error) {
switch config.GetStorageBackend() {
case "s3":
mc, err := minio.New(config.GetStorageS3Endpoint(), &minio.Options{
Creds: credentials.NewStaticV4(config.GetStorageS3AccessKey(), config.GetStorageS3SecretKey(), ""),
Secure: config.GetStorageS3UseSSL(),
endpoint := config.GetStorageS3Endpoint()
access := config.GetStorageS3AccessKey()
secret := config.GetStorageS3SecretKey()
secure := config.GetStorageS3UseSSL()
bucket := config.GetStorageS3BucketName()
proxy := config.GetStorageS3Proxy()
s3, err := storage.OpenS3(endpoint, bucket, &storage.S3Config{
CoreOpts: minio.Options{
Creds: credentials.NewStaticV4(access, secret, ""),
Secure: secure,
},
GetOpts: minio.GetObjectOptions{},
PutOpts: minio.PutObjectOptions{},
PutChunkSize: 5 * 1024 * 1024, // 5MiB
StatOpts: minio.StatObjectOptions{},
RemoveOpts: minio.RemoveObjectOptions{},
ListSize: 200,
})
if err != nil {
return nil, fmt.Errorf("creating minio client: %w", err)
return nil, fmt.Errorf("error opening s3 storage: %w", err)
}
return NewS3(
mc,
config.GetStorageS3BucketName(),
config.GetStorageS3Proxy(),
), nil
return &S3{
Proxy: proxy,
Bucket: bucket,
Storage: s3,
KVStore: kv.New(s3),
}, nil
case "local":
basePath := config.GetStorageLocalBasePath()
disk, err := storage.OpenDisk(basePath, &storage.DiskConfig{
// Put the store lockfile in the storage dir itself.
// Normally this would not be safe, since we could end up
@ -75,8 +93,9 @@ func AutoConfig() (Driver, error) {
LockFile: path.Join(basePath, "store.lock"),
})
if err != nil {
return nil, fmt.Errorf("error openingdisk storage: %v", err)
return nil, fmt.Errorf("error opening disk storage: %w", err)
}
return &Local{kv.New(disk)}, nil
}
return nil, fmt.Errorf("invalid storage backend %s", config.GetStorageBackend())

32
testrig/storage.go
View File

@ -28,6 +28,7 @@ import (
"codeberg.org/gruf/go-store/v2/storage"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/superseriousbusiness/gotosocial/internal/config"
gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage"
)
@ -41,14 +42,35 @@ func NewInMemoryStorage() *gtsstorage.Local {
}
func NewS3Storage() gtsstorage.Driver {
mc, err := minio.New(os.Getenv("GTS_STORAGE_S3_ENDPOINT"), &minio.Options{
Creds: credentials.NewStaticV4(os.Getenv("GTS_STORAGE_S3_ACCESS_KEY"), os.Getenv("GTS_STORAGE_S3_SECRET_KEY"), ""),
Secure: false,
endpoint := config.GetStorageS3Endpoint()
access := config.GetStorageS3AccessKey()
secret := config.GetStorageS3SecretKey()
secure := config.GetStorageS3UseSSL()
bucket := config.GetStorageS3BucketName()
proxy := config.GetStorageS3Proxy()
s3, err := storage.OpenS3(endpoint, bucket, &storage.S3Config{
CoreOpts: minio.Options{
Creds: credentials.NewStaticV4(access, secret, ""),
Secure: secure,
},
GetOpts: minio.GetObjectOptions{},
PutOpts: minio.PutObjectOptions{},
PutChunkSize: 5 * 1024 * 1024, // 2MiB
StatOpts: minio.StatObjectOptions{},
RemoveOpts: minio.RemoveObjectOptions{},
ListSize: 200,
})
if err != nil {
panic(err)
panic(fmt.Errorf("error opening s3 storage: %w", err))
}
return &gtsstorage.S3{
Proxy: proxy,
Bucket: bucket,
Storage: s3,
KVStore: kv.New(s3),
}
return gtsstorage.NewS3(mc, os.Getenv("GTS_STORAGE_S3_BUCKET"), false)
}
// StandardStorageSetup populates the storage with standard test entries from the given directory.

15
vendor/codeberg.org/gruf/go-store/v2/storage/s3.go generated vendored
View File

@ -50,14 +50,17 @@ type S3Config struct {
// getS3Config returns a valid S3Config for supplied ptr.
func getS3Config(cfg *S3Config) S3Config {
const minChunkSz = 5 * 1024 * 1024
// If nil, use default
if cfg == nil {
cfg = DefaultS3Config
}
// Assume 0 chunk size == use default
if cfg.PutChunkSize <= 0 {
cfg.PutChunkSize = 4 * 1024 * 1024
// Ensure a minimum compatible chunk size
if cfg.PutChunkSize <= minChunkSz {
// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
cfg.PutChunkSize = minChunkSz
}
// Assume 0 list size == use default
@ -70,6 +73,8 @@ func getS3Config(cfg *S3Config) S3Config {
CoreOpts: cfg.CoreOpts,
GetOpts: cfg.GetOpts,
PutOpts: cfg.PutOpts,
PutChunkSize: cfg.PutChunkSize,
ListSize: cfg.ListSize,
StatOpts: cfg.StatOpts,
RemoveOpts: cfg.RemoveOpts,
}
@ -198,7 +203,7 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) e
}
var (
count int
count = 1
parts []minio.CompletePart
chunk = make([]byte, st.config.PutChunkSize)
rdr = bytes.NewReader(nil)
@ -243,7 +248,7 @@ loop:
uploadID,
count,
rdr,
st.config.PutChunkSize,
int64(n),
"",
"",
nil,

2
vendor/modules.txt vendored
View File

@ -58,7 +58,7 @@ codeberg.org/gruf/go-runners
# codeberg.org/gruf/go-sched v1.1.1
## explicit; go 1.19
codeberg.org/gruf/go-sched
# codeberg.org/gruf/go-store/v2 v2.0.7
# codeberg.org/gruf/go-store/v2 v2.0.9
## explicit; go 1.19
codeberg.org/gruf/go-store/v2/kv
codeberg.org/gruf/go-store/v2/storage