[chore] use kv.KVStore also for S3 storage (#1113)

* replace s3 storage implementation to also use kv.KVStore

Signed-off-by: kim <grufwub@gmail.com>

* pull in latest `go-store` fix

Signed-off-by: kim <grufwub@gmail.com>

* pull-in go-store v2.0.9 fixes, update s3 put chunk size to 5MiB

Signed-off-by: kim <grufwub@gmail.com>

Signed-off-by: kim <grufwub@gmail.com>

go.mod

@@ -12,7 +12,7 @@ require (
 	codeberg.org/gruf/go-logger/v2 v2.2.1
 	codeberg.org/gruf/go-mutexes v1.1.4
 	codeberg.org/gruf/go-runners v1.3.1
-	codeberg.org/gruf/go-store/v2 v2.0.7
+	codeberg.org/gruf/go-store/v2 v2.0.9
 	github.com/buckket/go-blurhash v1.1.0
 	github.com/coreos/go-oidc/v3 v3.4.0
 	github.com/cornelk/hashmap v1.0.8

go.sum

@@ -99,8 +99,8 @@ codeberg.org/gruf/go-runners v1.3.1 h1:d/OQMMMiA6yPaDSbSr0/Jc+lucWmm7AiAZjWffpNK
 codeberg.org/gruf/go-runners v1.3.1/go.mod h1:rl0EdZNozkRMb21DAtOL5L4oTfmslYQdZgq2RMMc/H4=
 codeberg.org/gruf/go-sched v1.1.1 h1:YtLSQhpypzuD3HTup5oF7LLWB79gTL4nqW06kH4Vwks=
 codeberg.org/gruf/go-sched v1.1.1/go.mod h1:SRcdP/5qim+EBT3n3r4aUra1C30yPqV4OJOXuqvgdQM=
-codeberg.org/gruf/go-store/v2 v2.0.7 h1:P+0d8jnXdgzxfHLqKjHMV+MAxVJmq056PvzaHRyR8jE=
+codeberg.org/gruf/go-store/v2 v2.0.9 h1:nFurtZCBdZac79aXo3PFPVn2N744B06dHLygVrHf+SA=
-codeberg.org/gruf/go-store/v2 v2.0.7/go.mod h1:D4r5PV0BXDhxQyATw/03JkwvziZDkVMgzTpElZyWTXI=
+codeberg.org/gruf/go-store/v2 v2.0.9/go.mod h1:D4r5PV0BXDhxQyATw/03JkwvziZDkVMgzTpElZyWTXI=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=

internal/storage/s3.go

@@ -19,80 +19,32 @@
 package storage
 
 import (
-	"bytes"
 	"context"
-	"fmt"
-	"io"
 	"mime"
 	"net/url"
 	"path"
 	"time"
 
-	"github.com/minio/minio-go/v7"
+	"codeberg.org/gruf/go-store/v2/kv"
+	"codeberg.org/gruf/go-store/v2/storage"
 )
 
 type S3 struct {
-	mc     *minio.Client
+	Proxy   bool
-	bucket string
+	Bucket  string
-	proxy  bool
+	Storage *storage.S3Storage
-}
+	*kv.KVStore
-
-func NewS3(mc *minio.Client, bucket string, proxy bool) *S3 {
-	return &S3{
-		mc:     mc,
-		bucket: bucket,
-		proxy:  proxy,
-	}
-}
-
-func (s *S3) Get(ctx context.Context, key string) ([]byte, error) {
-	r, err := s.GetStream(ctx, key)
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-	b, err := io.ReadAll(r)
-	if err != nil {
-		return nil, fmt.Errorf("reading data from s3: %w", err)
-	}
-	return b, nil
-}
-
-func (s *S3) GetStream(ctx context.Context, key string) (io.ReadCloser, error) {
-	o, err := s.mc.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
-	if err != nil {
-		err = fmt.Errorf("retrieving object from s3: %w", err)
-	}
-	return o, err
-}
-
-func (s *S3) PutStream(ctx context.Context, key string, r io.Reader) error {
-	if _, err := s.mc.PutObject(ctx, s.bucket, key, r, -1, minio.PutObjectOptions{}); err != nil {
-		return fmt.Errorf("uploading data stream: %w", err)
-	}
-	return nil
-}
-
-func (s *S3) Put(ctx context.Context, key string, value []byte) error {
-	if _, err := s.mc.PutObject(ctx, s.bucket, key, bytes.NewBuffer(value), -1, minio.PutObjectOptions{}); err != nil {
-		return fmt.Errorf("uploading data slice: %w", err)
-	}
-	return nil
-}
-
-func (s *S3) Delete(ctx context.Context, key string) error {
-	return s.mc.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
 }
 
 func (s *S3) URL(ctx context.Context, key string) *url.URL {
-	if s.proxy {
+	if s.Proxy {
 		return nil
 	}
 
-	// it's safe to ignore the error here, as we just fall back to fetching the
+	// it's safe to ignore the error here, as we just fall back to fetching the file if URL request fails
-	// file if the url request fails
+	url, _ := s.Storage.Client().PresignedGetObject(ctx, s.Bucket, key, time.Hour, url.Values{
-	url, _ := s.mc.PresignedGetObject(ctx, s.bucket, key, time.Hour, url.Values{
 		"response-content-type": []string{mime.TypeByExtension(path.Ext(key))},
 	})
+
 	return url
 }

internal/storage/storage.go

@@ -52,20 +52,38 @@ type Driver interface {
 func AutoConfig() (Driver, error) {
 	switch config.GetStorageBackend() {
 	case "s3":
-		mc, err := minio.New(config.GetStorageS3Endpoint(), &minio.Options{
+		endpoint := config.GetStorageS3Endpoint()
-			Creds:  credentials.NewStaticV4(config.GetStorageS3AccessKey(), config.GetStorageS3SecretKey(), ""),
+		access := config.GetStorageS3AccessKey()
-			Secure: config.GetStorageS3UseSSL(),
+		secret := config.GetStorageS3SecretKey()
+		secure := config.GetStorageS3UseSSL()
+		bucket := config.GetStorageS3BucketName()
+		proxy := config.GetStorageS3Proxy()
+
+		s3, err := storage.OpenS3(endpoint, bucket, &storage.S3Config{
+			CoreOpts: minio.Options{
+				Creds:  credentials.NewStaticV4(access, secret, ""),
+				Secure: secure,
+			},
+			GetOpts:      minio.GetObjectOptions{},
+			PutOpts:      minio.PutObjectOptions{},
+			PutChunkSize: 5 * 1024 * 1024, // 5MiB
+			StatOpts:     minio.StatObjectOptions{},
+			RemoveOpts:   minio.RemoveObjectOptions{},
+			ListSize:     200,
 		})
 		if err != nil {
-			return nil, fmt.Errorf("creating minio client: %w", err)
+			return nil, fmt.Errorf("error opening s3 storage: %w", err)
 		}
-		return NewS3(
+
-			mc,
+		return &S3{
-			config.GetStorageS3BucketName(),
+			Proxy:   proxy,
-			config.GetStorageS3Proxy(),
+			Bucket:  bucket,
-		), nil
+			Storage: s3,
+			KVStore: kv.New(s3),
+		}, nil
 	case "local":
 		basePath := config.GetStorageLocalBasePath()
+
 		disk, err := storage.OpenDisk(basePath, &storage.DiskConfig{
 			// Put the store lockfile in the storage dir itself.
 			// Normally this would not be safe, since we could end up
@@ -75,8 +93,9 @@ func AutoConfig() (Driver, error) {
 			LockFile: path.Join(basePath, "store.lock"),
 		})
 		if err != nil {
-			return nil, fmt.Errorf("error openingdisk storage: %v", err)
+			return nil, fmt.Errorf("error opening disk storage: %w", err)
 		}
+
 		return &Local{kv.New(disk)}, nil
 	}
 	return nil, fmt.Errorf("invalid storage backend %s", config.GetStorageBackend())

testrig/storage.go

@@ -28,6 +28,7 @@ import (
 	"codeberg.org/gruf/go-store/v2/storage"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
 	gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage"
 )
 
@@ -41,14 +42,35 @@ func NewInMemoryStorage() *gtsstorage.Local {
 }
 
 func NewS3Storage() gtsstorage.Driver {
-	mc, err := minio.New(os.Getenv("GTS_STORAGE_S3_ENDPOINT"), &minio.Options{
+	endpoint := config.GetStorageS3Endpoint()
-		Creds:  credentials.NewStaticV4(os.Getenv("GTS_STORAGE_S3_ACCESS_KEY"), os.Getenv("GTS_STORAGE_S3_SECRET_KEY"), ""),
+	access := config.GetStorageS3AccessKey()
-		Secure: false,
+	secret := config.GetStorageS3SecretKey()
+	secure := config.GetStorageS3UseSSL()
+	bucket := config.GetStorageS3BucketName()
+	proxy := config.GetStorageS3Proxy()
+
+	s3, err := storage.OpenS3(endpoint, bucket, &storage.S3Config{
+		CoreOpts: minio.Options{
+			Creds:  credentials.NewStaticV4(access, secret, ""),
+			Secure: secure,
+		},
+		GetOpts:      minio.GetObjectOptions{},
+		PutOpts:      minio.PutObjectOptions{},
+		PutChunkSize: 5 * 1024 * 1024, // 2MiB
+		StatOpts:     minio.StatObjectOptions{},
+		RemoveOpts:   minio.RemoveObjectOptions{},
+		ListSize:     200,
 	})
 	if err != nil {
-		panic(err)
+		panic(fmt.Errorf("error opening s3 storage: %w", err))
+	}
+
+	return &gtsstorage.S3{
+		Proxy:   proxy,
+		Bucket:  bucket,
+		Storage: s3,
+		KVStore: kv.New(s3),
 	}
-	return gtsstorage.NewS3(mc, os.Getenv("GTS_STORAGE_S3_BUCKET"), false)
 }
 
 // StandardStorageSetup populates the storage with standard test entries from the given directory.

vendor/codeberg.org/gruf/go-store/v2/storage/s3.go

@@ -50,14 +50,17 @@ type S3Config struct {
 
 // getS3Config returns a valid S3Config for supplied ptr.
 func getS3Config(cfg *S3Config) S3Config {
+	const minChunkSz = 5 * 1024 * 1024
+
 	// If nil, use default
 	if cfg == nil {
 		cfg = DefaultS3Config
 	}
 
-	// Assume 0 chunk size == use default
+	// Ensure a minimum compatible chunk size
-	if cfg.PutChunkSize <= 0 {
+	if cfg.PutChunkSize <= minChunkSz {
-		cfg.PutChunkSize = 4 * 1024 * 1024
+		// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
+		cfg.PutChunkSize = minChunkSz
 	}
 
 	// Assume 0 list size == use default
@@ -67,11 +70,13 @@ func getS3Config(cfg *S3Config) S3Config {
 
 	// Return owned config copy
 	return S3Config{
-		CoreOpts:   cfg.CoreOpts,
+		CoreOpts:     cfg.CoreOpts,
-		GetOpts:    cfg.GetOpts,
+		GetOpts:      cfg.GetOpts,
-		PutOpts:    cfg.PutOpts,
+		PutOpts:      cfg.PutOpts,
-		StatOpts:   cfg.StatOpts,
+		PutChunkSize: cfg.PutChunkSize,
-		RemoveOpts: cfg.RemoveOpts,
+		ListSize:     cfg.ListSize,
+		StatOpts:     cfg.StatOpts,
+		RemoveOpts:   cfg.RemoveOpts,
 	}
 }
 
@@ -198,7 +203,7 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) e
 	}
 
 	var (
-		count int
+		count = 1
 		parts []minio.CompletePart
 		chunk = make([]byte, st.config.PutChunkSize)
 		rdr   = bytes.NewReader(nil)
@@ -243,7 +248,7 @@ loop:
 			uploadID,
 			count,
 			rdr,
-			st.config.PutChunkSize,
+			int64(n),
 			"",
 			"",
 			nil,

vendor/modules.txt

@@ -58,7 +58,7 @@ codeberg.org/gruf/go-runners
 # codeberg.org/gruf/go-sched v1.1.1
 ## explicit; go 1.19
 codeberg.org/gruf/go-sched
-# codeberg.org/gruf/go-store/v2 v2.0.7
+# codeberg.org/gruf/go-store/v2 v2.0.9
 ## explicit; go 1.19
 codeberg.org/gruf/go-store/v2/kv
 codeberg.org/gruf/go-store/v2/storage