From 43519324b39de697e3403691fb286de03bf0d4d1 Mon Sep 17 00:00:00 2001 From: CDN Date: Wed, 31 Jul 2024 20:44:18 +0800 Subject: [PATCH] [feature] Object store custom URL (S3) (#3046) * tweaks * boobs * fix variable name + typo --------- Co-authored-by: tobi --- docs/configuration/storage.md | 43 ++++++++++++++++++++++++++++++++-- example/config.yaml | 33 +++++++++++++++++++++++++- internal/config/config.go | 1 + internal/config/defaults.go | 1 + internal/config/helpers.gen.go | 25 ++++++++++++++++++++ internal/config/validate.go | 24 +++++++++++++++++++ internal/storage/storage.go | 38 +++++++++++++++++++++++++----- test/envparsing.sh | 2 ++ 8 files changed, 158 insertions(+), 9 deletions(-) diff --git a/docs/configuration/storage.md b/docs/configuration/storage.md index d76a9bed4..539898e11 100644 --- a/docs/configuration/storage.md +++ b/docs/configuration/storage.md @@ -30,11 +30,42 @@ storage-local-base-path: "/gotosocial/storage" # Default: "" storage-s3-endpoint: "" -# Bool. If data stored in S3 should be proxied through GoToSocial instead of redirecting to a presigned URL. +# Bool. Set this to true if data stored in S3 should be proxied through +# GoToSocial instead of forwarding the request to a presigned URL. +# +# In most cases you won't need to touch this setting, but it might be useful +# if it's not possible for your bucket provider to generate presigned URLs, +# or if your bucket is not able to exposed to the wider internet. # # Default: false storage-s3-proxy: false +# String. URL to use a base for redirecting incoming media requests to. +# +# Must start with "http://" or "https://" and end without a trailing slash. +# +# DON'T SET THIS VALUE UNLESS YOU HAVE GOOD REASON TO! It's not necessary for +# "normal" s3 usage, and most admins can happily just ignore this setting. +# +# If set, then media fileserver requests to your instance will be redirected +# to this URL instead of your bucket URL, preserving relevant path parts. +# +# This is useful if you are using a CDN proxy in front of your S3 bucket, and you +# want to serve media from the CDN rather than serving from your S3 bucket directly. +# +# For example, if you have your storage-s3-endpoint value set to "s3.my-storage.example.org", +# and you have a CDN set up to proxy your bucket, serving from "cdn.some-fancy-host.org", +# then you should set storage-s3-redirect-url to "https://cdn.some-fancy-host.org". +# +# This will allow your GoToSocial instance to *upload* data to "s3.my-storage.example.org", +# but direct callers to *download* that data from "https://cdn.some-fancy-host.org". +# +# This value is ignored if storage-backend is not s3, or if storage-s3-proxy is true. +# +# Examples: ["https://cdn.some-fancy-host.org"] +# Default: "" +storage-s3-redirect-url: "" + # Bool. Use SSL for S3 connections. # # Only set this to 'false' when testing locally. @@ -76,7 +107,7 @@ storage-s3-bucket: "" GoToSocial by default creates signed URL's which means we don't need to change anything major on the policies of the bucket. 1. Login to AWS -> select S3 as service. -2. click Create Bucket +2. Click Create Bucket 3. Provide a unique name and avoid adding "." in the name 4. Do not change the public access settings (Let them be on "block public access" mode) @@ -110,6 +141,14 @@ GoToSocial by default creates signed URL's which means we don't need to change a * `storage-s3-secret-key` -> Secret key you obtained for the user created above * `storage-s3-bucket` -> The `` that you created just now +### `storage-s3-redirect-url` + +If you are using a CDN in front of your S3 bucket, and you want to serve media from the CDN rather than serving from your S3 bucket directly, you should set the `storage-s3-redirect-url` to the CDN URL. + +For example, if you have your `storage-s3-endpoint` value set to "s3.my-storage.example.org", and you have a CDN set up to proxy your bucket, serving from "cdn.some-fancy-host.org", then you should set `storage-s3-redirect-url` to "https://cdn.some-fancy-host.org". + +This will allow your GoToSocial instance to *upload* data to "s3.my-storage.example.org", but direct callers to *download* that data from "https://cdn.some-fancy-host.org". + ## Storage migration Migration between backends is freely possible. To do so, you only have to move the directories (and their contents) between the different implementations. diff --git a/example/config.yaml b/example/config.yaml index 75d0587cf..60fdd88cc 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -551,11 +551,42 @@ storage-local-base-path: "/gotosocial/storage" # Default: "" storage-s3-endpoint: "" -# Bool. If data stored in S3 should be proxied through GoToSocial instead of redirecting to a presigned URL. +# Bool. Set this to true if data stored in S3 should be proxied through +# GoToSocial instead of forwarding the request to a presigned URL. +# +# In most cases you won't need to touch this setting, but it might be useful +# if it's not possible for your bucket provider to generate presigned URLs, +# or if your bucket is not able to exposed to the wider internet. # # Default: false storage-s3-proxy: false +# String. URL to use a base for redirecting incoming media requests to. +# +# Must start with "http://" or "https://" and end without a trailing slash. +# +# DON'T SET THIS VALUE UNLESS YOU HAVE GOOD REASON TO! It's not necessary for +# "normal" s3 usage, and most admins can happily just ignore this setting. +# +# If set, then media fileserver requests to your instance will be redirected +# to this URL instead of your bucket URL, preserving relevant path parts. +# +# This is useful if you are using a CDN proxy in front of your S3 bucket, and you +# want to serve media from the CDN rather than serving from your S3 bucket directly. +# +# For example, if you have your storage-s3-endpoint value set to "s3.my-storage.example.org", +# and you have a CDN set up to proxy your bucket, serving from "cdn.some-fancy-host.org", +# then you should set storage-s3-redirect-url to "https://cdn.some-fancy-host.org". +# +# This will allow your GoToSocial instance to *upload* data to "s3.my-storage.example.org", +# but direct callers to *download* that data from "https://cdn.some-fancy-host.org". +# +# This value is ignored if storage-backend is not s3, or if storage-s3-proxy is true. +# +# Examples: ["https://cdn.some-fancy-host.org"] +# Default: "" +storage-s3-redirect-url: "" + # Bool. Use SSL for S3 connections. # # Only set this to 'false' when testing locally. diff --git a/internal/config/config.go b/internal/config/config.go index a6499d822..bba284d56 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -110,6 +110,7 @@ type Configuration struct { StorageS3UseSSL bool `name:"storage-s3-use-ssl" usage:"Use SSL for S3 connections. Only set this to 'false' when testing locally"` StorageS3BucketName string `name:"storage-s3-bucket" usage:"Place blobs in this bucket"` StorageS3Proxy bool `name:"storage-s3-proxy" usage:"Proxy S3 contents through GoToSocial instead of redirecting to a presigned URL"` + StorageS3RedirectURL string `name:"storage-s3-redirect-url" usage:"Custom URL to use for redirecting S3 media links. If set, this will be used instead of the S3 bucket URL."` StatusesMaxChars int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses, including content warning"` StatusesPollMaxOptions int `name:"statuses-poll-max-options" usage:"Max amount of options permitted on a poll"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 835841c84..d16df6802 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -85,6 +85,7 @@ var Defaults = Configuration{ StorageLocalBasePath: "/gotosocial/storage", StorageS3UseSSL: true, StorageS3Proxy: false, + StorageS3RedirectURL: "", StatusesMaxChars: 5000, StatusesPollMaxOptions: 6, diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 587fba364..7523f17ad 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -1500,6 +1500,31 @@ func GetStorageS3Proxy() bool { return global.GetStorageS3Proxy() } // SetStorageS3Proxy safely sets the value for global configuration 'StorageS3Proxy' field func SetStorageS3Proxy(v bool) { global.SetStorageS3Proxy(v) } +// GetStorageS3RedirectURL safely fetches the Configuration value for state's 'StorageS3RedirectURL' field +func (st *ConfigState) GetStorageS3RedirectURL() (v string) { + st.mutex.RLock() + v = st.config.StorageS3RedirectURL + st.mutex.RUnlock() + return +} + +// SetStorageS3RedirectURL safely sets the Configuration value for state's 'StorageS3RedirectURL' field +func (st *ConfigState) SetStorageS3RedirectURL(v string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.StorageS3RedirectURL = v + st.reloadToViper() +} + +// StorageS3RedirectURLFlag returns the flag name for the 'StorageS3RedirectURL' field +func StorageS3RedirectURLFlag() string { return "storage-s3-redirect-url" } + +// GetStorageS3RedirectURL safely fetches the value for global configuration 'StorageS3RedirectURL' field +func GetStorageS3RedirectURL() string { return global.GetStorageS3RedirectURL() } + +// SetStorageS3RedirectURL safely sets the value for global configuration 'StorageS3RedirectURL' field +func SetStorageS3RedirectURL(v string) { global.SetStorageS3RedirectURL(v) } + // GetStatusesMaxChars safely fetches the Configuration value for state's 'StatusesMaxChars' field func (st *ConfigState) GetStatusesMaxChars() (v int) { st.mutex.RLock() diff --git a/internal/config/validate.go b/internal/config/validate.go index d79d83b9d..723d5c931 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -19,6 +19,8 @@ package config import ( "fmt" + "net/url" + "strings" "github.com/miekg/dns" "github.com/superseriousbusiness/gotosocial/internal/gtserror" @@ -118,6 +120,28 @@ func Validate() error { errf("%s must be set", WebAssetBaseDirFlag()) } + // `storage-s3-redirect-url` + if s3RedirectURL := GetStorageS3RedirectURL(); s3RedirectURL != "" { + if strings.HasSuffix(s3RedirectURL, "/") { + errf( + "%s must not end with a trailing slash", + StorageS3RedirectURLFlag(), + ) + } + + if url, err := url.Parse(s3RedirectURL); err != nil { + errf( + "%s invalid: %w", + StorageS3RedirectURLFlag(), err, + ) + } else if url.Scheme != "https" && url.Scheme != "http" { + errf( + "%s scheme must be https or http", + StorageS3RedirectURLFlag(), + ) + } + } + // Custom / LE TLS settings. // // Only one of custom certs or LE can be set, diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 5d5baf283..f3cb814f1 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -79,6 +79,7 @@ type Driver struct { Proxy bool Bucket string PresignedCache *ttl.Cache[string, PresignedURL] + RedirectURL string } // Get returns the byte value for key in storage. @@ -163,12 +164,27 @@ func (d *Driver) URL(ctx context.Context, key string) *PresignedURL { return &e.Value } - u, err := s3.Client().PresignedGetObject(ctx, d.Bucket, key, urlCacheTTL, url.Values{ - "response-content-type": []string{mime.TypeByExtension(path.Ext(key))}, - }) - if err != nil { - // If URL request fails, fallback is to fetch the file. So ignore the error here - return nil + var ( + u *url.URL + err error + ) + + if d.RedirectURL != "" { + u, err = url.Parse(d.RedirectURL + "/" + key) + if err != nil { + // If URL parsing fails, fallback is to + // fetch the file. So ignore the error here + return nil + } + } else { + u, err = s3.Client().PresignedGetObject(ctx, d.Bucket, key, urlCacheTTL, url.Values{ + "response-content-type": []string{mime.TypeByExtension(path.Ext(key))}, + }) + if err != nil { + // If URL request fails, fallback is to + // fetch the file. So ignore the error here + return nil + } } psu := PresignedURL{ @@ -204,6 +220,14 @@ func (d *Driver) ProbeCSPUri(ctx context.Context) (string, error) { return "", nil } + // If an S3 redirect URL is set, just + // return this URL without probing; we + // likely don't have write access on it + // anyway since it's probs a CDN bucket. + if d.RedirectURL != "" { + return d.RedirectURL + "/", nil + } + const cspKey = "gotosocial-csp-probe" // Create an empty file in S3 storage. @@ -273,6 +297,7 @@ func NewS3Storage() (*Driver, error) { secret := config.GetStorageS3SecretKey() secure := config.GetStorageS3UseSSL() bucket := config.GetStorageS3BucketName() + redirectURL := config.GetStorageS3RedirectURL() // Open the s3 storage implementation s3, err := s3.Open(endpoint, bucket, &s3.Config{ @@ -300,5 +325,6 @@ func NewS3Storage() (*Driver, error) { Bucket: config.GetStorageS3BucketName(), Storage: s3, PresignedCache: presignedCache, + RedirectURL: redirectURL, }, nil } diff --git a/test/envparsing.sh b/test/envparsing.sh index 281bf7405..3855c372f 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -173,6 +173,7 @@ EXPECT=$(cat << "EOF" "storage-s3-bucket": "gts", "storage-s3-endpoint": "localhost:9000", "storage-s3-proxy": true, + "storage-s3-redirect-url": "", "storage-s3-secret-key": "miniostorage", "storage-s3-use-ssl": false, "syslog-address": "127.0.0.1:6969", @@ -253,6 +254,7 @@ GTS_STORAGE_S3_SECRET_KEY='miniostorage' \ GTS_STORAGE_S3_ENDPOINT='localhost:9000' \ GTS_STORAGE_S3_USE_SSL='false' \ GTS_STORAGE_S3_PROXY='true' \ +GTS_STORAGE_S3_REDIRECT_URL='' \ GTS_STORAGE_S3_BUCKET='gts' \ GTS_STATUSES_MAX_CHARS=69 \ GTS_STATUSES_CW_MAX_CHARS=420 \