From fa59c3713c14bd1463b923d57a57a914622626d7 Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Sat, 3 Aug 2024 16:40:26 +0200 Subject: [PATCH] [chore] Add `media-ffmpeg-pool-size` config var (#3164) --- cmd/gotosocial/action/server/server.go | 16 ++++++++++++---- docs/configuration/media.md | 18 ++++++++++++++++++ example/config.yaml | 18 ++++++++++++++++++ internal/config/config.go | 1 + internal/config/defaults.go | 1 + internal/config/helpers.gen.go | 25 +++++++++++++++++++++++++ test/envparsing.sh | 2 ++ 7 files changed, 77 insertions(+), 4 deletions(-) diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index 68b039d0c..c2c5e25cd 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -487,16 +487,24 @@ func setLimits(ctx context.Context) { } func precompileWASM(ctx context.Context) error { - // TODO: make max number instances configurable - maxprocs := runtime.GOMAXPROCS(0) if err := sqlite3.Initialize(); err != nil { return gtserror.Newf("error compiling sqlite3: %w", err) } - if err := ffmpeg.InitFfmpeg(ctx, maxprocs); err != nil { + + // Use admin-set ffmpeg pool size, and fall + // back to GOMAXPROCS if number 0 or less. + ffPoolSize := config.GetMediaFfmpegPoolSize() + if ffPoolSize <= 0 { + ffPoolSize = runtime.GOMAXPROCS(0) + } + + if err := ffmpeg.InitFfmpeg(ctx, ffPoolSize); err != nil { return gtserror.Newf("error compiling ffmpeg: %w", err) } - if err := ffmpeg.InitFfprobe(ctx, maxprocs); err != nil { + + if err := ffmpeg.InitFfprobe(ctx, ffPoolSize); err != nil { return gtserror.Newf("error compiling ffprobe: %w", err) } + return nil } diff --git a/docs/configuration/media.md b/docs/configuration/media.md index e07463320..4e222c6c7 100644 --- a/docs/configuration/media.md +++ b/docs/configuration/media.md @@ -56,6 +56,24 @@ media-emoji-local-max-size: 50KiB # Default: 100KiB (102400 bytes) media-emoji-remote-max-size: 100KiB +# Int. Number of instances of ffmpeg+ffprobe to add to the media processing pool. +# +# Increasing this number will lead to faster concurrent media processing, +# but at the cost of up to about 250MB of (spiking) memory usage per increment. +# +# You'll want to increase this number if you have RAM to spare, and/or if you're +# hosting an instance for more than 50 or so people who post/view lots of media, +# but you should leave it at 1 for single-user instances or when running GoToSocial +# in a constrained (low-memory) environment. +# +# If you set this number to 0 or less, then instead of a fixed number of instances, +# it will scale with GOMAXPROCS x 1, yielding (usually) one ffmpeg instance and one +# ffprobe instance per CPU core on the host machine. +# +# Examples: [1, 2, -1, 8] +# Default: 1 +media-ffmpeg-pool-size: 1 + # The below media cleanup settings allow admins to customize when and # how often media cleanup + prune jobs run, while being set to a fairly # sensible default (every night @ midnight). For more information on exactly diff --git a/example/config.yaml b/example/config.yaml index 60fdd88cc..1fca954dc 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -493,6 +493,24 @@ media-emoji-local-max-size: 50KiB # Default: 100KiB (102400 bytes) media-emoji-remote-max-size: 100KiB +# Int. Number of instances of ffmpeg+ffprobe to add to the media processing pool. +# +# Increasing this number will lead to faster concurrent media processing, +# but at the cost of up to about 250MB of (spiking) memory usage per increment. +# +# You'll want to increase this number if you have RAM to spare, and/or if you're +# hosting an instance for more than 50 or so people who post/view lots of media, +# but you should leave it at 1 for single-user instances or when running GoToSocial +# in a constrained (low-memory) environment. +# +# If you set this number to 0 or less, then instead of a fixed number of instances, +# it will scale with GOMAXPROCS x 1, yielding (usually) one ffmpeg instance and one +# ffprobe instance per CPU core on the host machine. +# +# Examples: [1, 2, -1, 8] +# Default: 1 +media-ffmpeg-pool-size: 1 + # The below media cleanup settings allow admins to customize when and # how often media cleanup + prune jobs run, while being set to a fairly # sensible default (every night @ midnight). For more information on exactly diff --git a/internal/config/config.go b/internal/config/config.go index bba284d56..79f74cf71 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -101,6 +101,7 @@ type Configuration struct { MediaRemoteMaxSize bytesize.Size `name:"media-remote-max-size" usage:"Max size in bytes of media to download from other instances"` MediaCleanupFrom string `name:"media-cleanup-from" usage:"Time of day from which to start running media cleanup/prune jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."` MediaCleanupEvery time.Duration `name:"media-cleanup-every" usage:"Period to elapse between cleanups, starting from media-cleanup-at."` + MediaFfmpegPoolSize int `name:"media-ffmpeg-pool-size" usage:"Number of instances of the embedded ffmpeg WASM binary to add to the media processing pool. 0 or less uses GOMAXPROCS."` StorageBackend string `name:"storage-backend" usage:"Storage backend to use for media attachments"` StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index d16df6802..7728b4a7f 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -80,6 +80,7 @@ var Defaults = Configuration{ MediaEmojiRemoteMaxSize: 100 * bytesize.KiB, MediaCleanupFrom: "00:00", // Midnight. MediaCleanupEvery: 24 * time.Hour, // 1/day. + MediaFfmpegPoolSize: 1, StorageBackend: "local", StorageLocalBasePath: "/gotosocial/storage", diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 7523f17ad..d75956963 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -1300,6 +1300,31 @@ func GetMediaCleanupEvery() time.Duration { return global.GetMediaCleanupEvery() // SetMediaCleanupEvery safely sets the value for global configuration 'MediaCleanupEvery' field func SetMediaCleanupEvery(v time.Duration) { global.SetMediaCleanupEvery(v) } +// GetMediaFfmpegPoolSize safely fetches the Configuration value for state's 'MediaFfmpegPoolSize' field +func (st *ConfigState) GetMediaFfmpegPoolSize() (v int) { + st.mutex.RLock() + v = st.config.MediaFfmpegPoolSize + st.mutex.RUnlock() + return +} + +// SetMediaFfmpegPoolSize safely sets the Configuration value for state's 'MediaFfmpegPoolSize' field +func (st *ConfigState) SetMediaFfmpegPoolSize(v int) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.MediaFfmpegPoolSize = v + st.reloadToViper() +} + +// MediaFfmpegPoolSizeFlag returns the flag name for the 'MediaFfmpegPoolSize' field +func MediaFfmpegPoolSizeFlag() string { return "media-ffmpeg-pool-size" } + +// GetMediaFfmpegPoolSize safely fetches the value for global configuration 'MediaFfmpegPoolSize' field +func GetMediaFfmpegPoolSize() int { return global.GetMediaFfmpegPoolSize() } + +// SetMediaFfmpegPoolSize safely sets the value for global configuration 'MediaFfmpegPoolSize' field +func SetMediaFfmpegPoolSize(v int) { global.SetMediaFfmpegPoolSize(v) } + // GetStorageBackend safely fetches the Configuration value for state's 'StorageBackend' field func (st *ConfigState) GetStorageBackend() (v string) { st.mutex.RLock() diff --git a/test/envparsing.sh b/test/envparsing.sh index 3855c372f..f88b6a031 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -126,6 +126,7 @@ EXPECT=$(cat << "EOF" "media-description-min-chars": 69, "media-emoji-local-max-size": 420, "media-emoji-remote-max-size": 420, + "media-ffmpeg-pool-size": 8, "media-local-max-size": 420, "media-remote-cache-days": 30, "media-remote-max-size": 420, @@ -245,6 +246,7 @@ GTS_MEDIA_REMOTE_MAX_SIZE=420 \ GTS_MEDIA_REMOTE_CACHE_DAYS=30 \ GTS_MEDIA_EMOJI_LOCAL_MAX_SIZE=420 \ GTS_MEDIA_EMOJI_REMOTE_MAX_SIZE=420 \ +GTS_MEDIA_FFMPEG_POOL_SIZE=8 \ GTS_METRICS_AUTH_ENABLED=false \ GTS_METRICS_ENABLED=false \ GTS_STORAGE_BACKEND='local' \