[chore] Recompile wasm every 500 goes
This commit is contained in:
tobi
parent
1f3dfbf10c
commit
d970347c58
|
|
@ -19,22 +19,121 @@ package ffmpeg
|
|||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
ffmpeglib "codeberg.org/gruf/go-ffmpreg/embed/ffmpeg"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/log"
|
||||
"github.com/tetratelabs/wazero"
|
||||
"github.com/tetratelabs/wazero/sys"
|
||||
)
|
||||
|
||||
// ffmpegRunner limits the number of
|
||||
// ffmpeg WebAssembly instances that
|
||||
// may be concurrently running, in
|
||||
// order to reduce memory usage.
|
||||
var ffmpegRunner runner
|
||||
var (
|
||||
// ffmpegRunner limits the number of
|
||||
// ffmpeg WebAssembly instances that
|
||||
// may be concurrently running, in
|
||||
// order to reduce memory usage.
|
||||
ffmpegRunner runner
|
||||
|
||||
// ffmpeg compiled WASM.
|
||||
ffmpeg wazero.CompiledModule
|
||||
|
||||
// Number of times ffmpeg
|
||||
// compiled WASM has run.
|
||||
ffmpegRunCount int
|
||||
|
||||
// Sync for updating run count
|
||||
// and recompiling ffmpeg.
|
||||
ffmpegM sync.Mutex
|
||||
)
|
||||
|
||||
// InitFfmpeg precompiles the ffmpeg WebAssembly source into memory and
|
||||
// prepares the runner to only allow max given concurrent running instances.
|
||||
func InitFfmpeg(ctx context.Context, max int) error {
|
||||
|
||||
// Ensure runner initialized.
|
||||
ffmpegRunner.Init(max)
|
||||
|
||||
// Ensure runtime initialized.
|
||||
if err := initRuntime(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Ensure ffmpeg compiled.
|
||||
if ffmpeg == nil {
|
||||
return compileFfmpeg(ctx)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// compileFfmpeg ensures the ffmpeg WebAssembly
|
||||
// module has been pre-compiled into memory.
|
||||
func compileFfmpeg(ctx context.Context) error {
|
||||
var err error
|
||||
ffmpeg, err = runtime.CompileModule(ctx, ffmpeglib.B)
|
||||
return err
|
||||
}
|
||||
|
||||
// Ffmpeg runs the given arguments with an instance of ffmpeg.
|
||||
func Ffmpeg(ctx context.Context, args Args) (uint32, error) {
|
||||
return ffmpegRunner.Run(ctx, ffmpeg, args)
|
||||
return ffmpegRunner.Run(ctx, func() (uint32, error) {
|
||||
|
||||
// Update run count + check if we
|
||||
// need to recompile the module.
|
||||
ffmpegM.Lock()
|
||||
{
|
||||
ffmpegRunCount++
|
||||
if ffmpegRunCount > 500 {
|
||||
// Over our threshold of runs, close
|
||||
// current compiled module and recompile.
|
||||
if err := ffmpeg.Close(ctx); err != nil {
|
||||
ffmpegM.Unlock()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := compileFfmpeg(ctx); err != nil {
|
||||
ffmpegM.Unlock()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
ffmpegRunCount = 0
|
||||
}
|
||||
}
|
||||
ffmpegM.Unlock()
|
||||
|
||||
// Prefix module name as argv0 to args.
|
||||
cargs := make([]string, len(args.Args)+1)
|
||||
copy(cargs[1:], args.Args)
|
||||
cargs[0] = "ffmpeg"
|
||||
|
||||
// Create base module config.
|
||||
modcfg := wazero.NewModuleConfig()
|
||||
modcfg = modcfg.WithArgs(cargs...)
|
||||
modcfg = modcfg.WithStdin(args.Stdin)
|
||||
modcfg = modcfg.WithStdout(args.Stdout)
|
||||
modcfg = modcfg.WithStderr(args.Stderr)
|
||||
|
||||
if args.Config != nil {
|
||||
// Pass through config fn.
|
||||
modcfg = args.Config(modcfg)
|
||||
}
|
||||
|
||||
// Instantiate the module from precompiled wasm module data.
|
||||
mod, err := runtime.InstantiateModule(ctx, ffmpeg, modcfg)
|
||||
|
||||
if mod != nil {
|
||||
// Ensure closed.
|
||||
if err := mod.Close(ctx); err != nil {
|
||||
log.Errorf(ctx, "error closing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Try extract exit code.
|
||||
switch err := err.(type) {
|
||||
case *sys.ExitError:
|
||||
return err.ExitCode(), nil
|
||||
default:
|
||||
return 0, err
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,22 +19,121 @@ package ffmpeg
|
|||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
ffprobelib "codeberg.org/gruf/go-ffmpreg/embed/ffprobe"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/log"
|
||||
"github.com/tetratelabs/wazero"
|
||||
"github.com/tetratelabs/wazero/sys"
|
||||
)
|
||||
|
||||
// ffprobeRunner limits the number of
|
||||
// ffprobe WebAssembly instances that
|
||||
// may be concurrently running, in
|
||||
// order to reduce memory usage.
|
||||
var ffprobeRunner runner
|
||||
var (
|
||||
// ffprobeRunner limits the number of
|
||||
// ffprobe WebAssembly instances that
|
||||
// may be concurrently running, in
|
||||
// order to reduce memory usage.
|
||||
ffprobeRunner runner
|
||||
|
||||
// ffprobe compiled WASM.
|
||||
ffprobe wazero.CompiledModule
|
||||
|
||||
// Number of times ffprobe
|
||||
// compiled WASM has run.
|
||||
ffprobeRunCount int
|
||||
|
||||
// Sync for updating run count
|
||||
// and recompiling ffprobe.
|
||||
ffprobeM sync.Mutex
|
||||
)
|
||||
|
||||
// InitFfprobe precompiles the ffprobe WebAssembly source into memory and
|
||||
// prepares the runner to only allow max given concurrent running instances.
|
||||
func InitFfprobe(ctx context.Context, max int) error {
|
||||
|
||||
// Ensure runner initialized.
|
||||
ffprobeRunner.Init(max)
|
||||
|
||||
// Ensure runtime initialized.
|
||||
if err := initRuntime(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Ensure ffprobe compiled.
|
||||
if ffprobe == nil {
|
||||
return compileFfprobe(ctx)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// compileFfprobe ensures the ffprobe WebAssembly
|
||||
// module has been pre-compiled into memory.
|
||||
func compileFfprobe(ctx context.Context) error {
|
||||
var err error
|
||||
ffprobe, err = runtime.CompileModule(ctx, ffprobelib.B)
|
||||
return err
|
||||
}
|
||||
|
||||
// Ffprobe runs the given arguments with an instance of ffprobe.
|
||||
func Ffprobe(ctx context.Context, args Args) (uint32, error) {
|
||||
return ffprobeRunner.Run(ctx, ffprobe, args)
|
||||
return ffprobeRunner.Run(ctx, func() (uint32, error) {
|
||||
|
||||
// Update run count + check if we
|
||||
// need to recompile the module.
|
||||
ffprobeM.Lock()
|
||||
{
|
||||
ffprobeRunCount++
|
||||
if ffprobeRunCount > 500 {
|
||||
// Over our threshold of runs, close
|
||||
// current compiled module and recompile.
|
||||
if err := ffprobe.Close(ctx); err != nil {
|
||||
ffprobeM.Unlock()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := compileFfprobe(ctx); err != nil {
|
||||
ffprobeM.Unlock()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
ffprobeRunCount = 0
|
||||
}
|
||||
}
|
||||
ffprobeM.Unlock()
|
||||
|
||||
// Prefix module name as argv0 to args.
|
||||
cargs := make([]string, len(args.Args)+1)
|
||||
copy(cargs[1:], args.Args)
|
||||
cargs[0] = "ffprobe"
|
||||
|
||||
// Create base module config.
|
||||
modcfg := wazero.NewModuleConfig()
|
||||
modcfg = modcfg.WithArgs(cargs...)
|
||||
modcfg = modcfg.WithStdin(args.Stdin)
|
||||
modcfg = modcfg.WithStdout(args.Stdout)
|
||||
modcfg = modcfg.WithStderr(args.Stderr)
|
||||
|
||||
if args.Config != nil {
|
||||
// Pass through config fn.
|
||||
modcfg = args.Config(modcfg)
|
||||
}
|
||||
|
||||
// Instantiate the module from precompiled wasm module data.
|
||||
mod, err := runtime.InstantiateModule(ctx, ffprobe, modcfg)
|
||||
|
||||
if mod != nil {
|
||||
// Ensure closed.
|
||||
if err := mod.Close(ctx); err != nil {
|
||||
log.Errorf(ctx, "error closing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Try extract exit code.
|
||||
switch err := err.(type) {
|
||||
case *sys.ExitError:
|
||||
return err.ExitCode(), nil
|
||||
default:
|
||||
return 0, err
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,8 +19,6 @@ package ffmpeg
|
|||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/tetratelabs/wazero"
|
||||
)
|
||||
|
||||
// runner simply abstracts away the complexities
|
||||
|
|
@ -50,9 +48,12 @@ func (r *runner) Init(n int) {
|
|||
}
|
||||
}
|
||||
|
||||
// Run will attempt to pass the given compiled WebAssembly module with args to run(), waiting on
|
||||
// the receiving runner until a free slot is available to run an instance, (if a limit is enabled).
|
||||
func (r *runner) Run(ctx context.Context, cmod wazero.CompiledModule, args Args) (uint32, error) {
|
||||
// Run will instantiate (run) the given
|
||||
// function once a free slot is available.
|
||||
func (r *runner) Run(
|
||||
ctx context.Context,
|
||||
run func() (uint32, error),
|
||||
) (uint32, error) {
|
||||
select {
|
||||
// Context canceled.
|
||||
case <-ctx.Done():
|
||||
|
|
@ -65,6 +66,6 @@ func (r *runner) Run(ctx context.Context, cmod wazero.CompiledModule, args Args)
|
|||
// Release slot back to pool on end.
|
||||
defer func() { r.pool <- struct{}{} }()
|
||||
|
||||
// Pass to main module runner.
|
||||
return run(ctx, cmod, args)
|
||||
// Run provided function.
|
||||
return run()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ import (
|
|||
ffprobelib "codeberg.org/gruf/go-ffmpreg/embed/ffprobe"
|
||||
"github.com/tetratelabs/wazero"
|
||||
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
|
||||
"github.com/tetratelabs/wazero/sys"
|
||||
)
|
||||
|
||||
// Use all core features required by ffmpeg / ffprobe
|
||||
|
|
@ -34,15 +33,6 @@ import (
|
|||
const corefeatures = ffprobelib.CoreFeatures |
|
||||
ffmpeglib.CoreFeatures
|
||||
|
||||
var (
|
||||
// shared WASM runtime instance.
|
||||
runtime wazero.Runtime
|
||||
|
||||
// ffmpeg / ffprobe compiled WASM.
|
||||
ffmpeg wazero.CompiledModule
|
||||
ffprobe wazero.CompiledModule
|
||||
)
|
||||
|
||||
// Args encapsulates the passing of common
|
||||
// configuration options to run an instance
|
||||
// of a compiled WebAssembly module that is
|
||||
|
|
@ -62,96 +52,8 @@ type Args struct {
|
|||
Args []string
|
||||
}
|
||||
|
||||
// run will run the given compiled
|
||||
// WebAssembly module using given args,
|
||||
// using the global wazero runtime.
|
||||
func run(
|
||||
ctx context.Context,
|
||||
cmod wazero.CompiledModule,
|
||||
args Args,
|
||||
) (
|
||||
uint32, // exit code
|
||||
error,
|
||||
) {
|
||||
// Prefix module name as argv0 to args.
|
||||
cargs := make([]string, len(args.Args)+1)
|
||||
copy(cargs[1:], args.Args)
|
||||
cargs[0] = cmod.Name()
|
||||
|
||||
// Create base module config.
|
||||
modcfg := wazero.NewModuleConfig()
|
||||
modcfg = modcfg.WithArgs(cargs...)
|
||||
modcfg = modcfg.WithStdin(args.Stdin)
|
||||
modcfg = modcfg.WithStdout(args.Stdout)
|
||||
modcfg = modcfg.WithStderr(args.Stderr)
|
||||
|
||||
if args.Config != nil {
|
||||
// Pass through config fn.
|
||||
modcfg = args.Config(modcfg)
|
||||
}
|
||||
|
||||
// Instantiate the module from precompiled wasm module data.
|
||||
mod, err := runtime.InstantiateModule(ctx, cmod, modcfg)
|
||||
|
||||
if mod != nil {
|
||||
// Ensure closed.
|
||||
_ = mod.Close(ctx)
|
||||
}
|
||||
|
||||
// Try extract exit code.
|
||||
switch err := err.(type) {
|
||||
case *sys.ExitError:
|
||||
return err.ExitCode(), nil
|
||||
default:
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
// compileFfmpeg ensures the ffmpeg WebAssembly has been
|
||||
// pre-compiled into memory. If already compiled is a no-op.
|
||||
func compileFfmpeg(ctx context.Context) error {
|
||||
if ffmpeg != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ensure runtime already initialized.
|
||||
if err := initRuntime(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Compile the ffmpeg WebAssembly module into memory.
|
||||
cmod, err := runtime.CompileModule(ctx, ffmpeglib.B)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set module.
|
||||
ffmpeg = cmod
|
||||
return nil
|
||||
}
|
||||
|
||||
// compileFfprobe ensures the ffprobe WebAssembly has been
|
||||
// pre-compiled into memory. If already compiled is a no-op.
|
||||
func compileFfprobe(ctx context.Context) error {
|
||||
if ffprobe != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ensure runtime already initialized.
|
||||
if err := initRuntime(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Compile the ffprobe WebAssembly module into memory.
|
||||
cmod, err := runtime.CompileModule(ctx, ffprobelib.B)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set module.
|
||||
ffprobe = cmod
|
||||
return nil
|
||||
}
|
||||
// shared WASM runtime instance.
|
||||
var runtime wazero.Runtime
|
||||
|
||||
// initRuntime initializes the global wazero.Runtime,
|
||||
// if already initialized this function is a no-op.
|
||||
Loading…
Reference in New Issue