From 137ef5a9ff8f06f9167a1aca9bafa0e8ab2e21e6 Mon Sep 17 00:00:00 2001 From: Daenney Date: Sat, 29 Jun 2024 09:35:57 +0200 Subject: [PATCH] [feature] Default to WASM-based SQLite driver (#3053) * [feature] Default to WASM-based SQLite driver With 0.16 out this switches our default SQLite driver to the WASM-based solution instead. So far the driver seems to perform just as well. Switching our default should result in it getting a bit more testing during the 0.17 development cycle. * add the ol' john hancock --------- Co-authored-by: tobi --- .drone.yml | 4 +- .goreleaser.yml | 2 +- internal/db/sqlite/driver.go | 70 +++++++++++-------- ...asmsqlite3.go => driver_moderncsqlite3.go} | 70 ++++++++----------- internal/db/sqlite/errors.go | 30 ++++---- ...asmsqlite3.go => errors_moderncsqlite3.go} | 30 ++++---- scripts/build.sh | 16 ++--- 7 files changed, 111 insertions(+), 111 deletions(-) rename internal/db/sqlite/{driver_wasmsqlite3.go => driver_moderncsqlite3.go} (80%) rename internal/db/sqlite/{errors_wasmsqlite3.go => errors_moderncsqlite3.go} (71%) diff --git a/.drone.yml b/.drone.yml index 144e21078..944f0544f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -42,7 +42,7 @@ steps: go test -failfast -timeout=20m - -tags "wasmsqlite3 netgo osusergo static_build kvformat timetzdata" + -tags "netgo osusergo static_build kvformat timetzdata" ./... - ./test/envparsing.sh - ./test/swagger.sh @@ -204,6 +204,6 @@ steps: --- kind: signature -hmac: 2e74313f4192b3e6daf6d1d00a7c3796019d93da7ce7e0a77208ccc3c37089b0 +hmac: 86ebddcd630792cac43aa92fa7f45118943c51b5157491d05eb480ac21762329 ... diff --git a/.goreleaser.yml b/.goreleaser.yml index aa3b5bf60..6a7fccfd0 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -30,7 +30,7 @@ builds: - >- {{ if and (index .Env "DEBUG") (.Env.DEBUG) }}debugenv{{ end }} - >- - {{ if and (index .Env "WASMSQLITE3") (.Env.WASMSQLITE3) }}wasmsqlite3{{ end }} + {{ if and (index .Env "MODERNCSQLITE3") (.Env.MODERNCSQLITE3) }}moderncsqlite3{{ end }} env: - CGO_ENABLED=0 goos: diff --git a/internal/db/sqlite/driver.go b/internal/db/sqlite/driver.go index 11cb6b27d..cea976d94 100644 --- a/internal/db/sqlite/driver.go +++ b/internal/db/sqlite/driver.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -//go:build !wasmsqlite3 +//go:build !moderncsqlite3 package sqlite @@ -23,19 +23,22 @@ import ( "context" "database/sql/driver" - "modernc.org/sqlite" - "github.com/superseriousbusiness/gotosocial/internal/db" + + "github.com/ncruces/go-sqlite3" + sqlite3driver "github.com/ncruces/go-sqlite3/driver" + _ "github.com/ncruces/go-sqlite3/embed" // embed wasm binary + _ "github.com/ncruces/go-sqlite3/vfs/memdb" // include memdb vfs ) // Driver is our own wrapper around the -// sqlite.Driver{} type in order to wrap +// driver.SQLite{} type in order to wrap // further SQL types with our own // functionality, e.g. err processing. -type Driver struct{ sqlite.Driver } +type Driver struct{ sqlite3driver.SQLite } func (d *Driver) Open(name string) (driver.Conn, error) { - conn, err := d.Driver.Open(name) + conn, err := d.SQLite.Open(name) if err != nil { err = processSQLiteError(err) return nil, err @@ -43,6 +46,30 @@ func (d *Driver) Open(name string) (driver.Conn, error) { return &sqliteConn{conn.(connIface)}, nil } +func (d *Driver) OpenConnector(name string) (driver.Connector, error) { + cc, err := d.SQLite.OpenConnector(name) + if err != nil { + return nil, err + } + return &sqliteConnector{driver: d, Connector: cc}, nil +} + +type sqliteConnector struct { + driver *Driver + driver.Connector +} + +func (c *sqliteConnector) Driver() driver.Driver { return c.driver } + +func (c *sqliteConnector) Connect(ctx context.Context) (driver.Conn, error) { + conn, err := c.Connector.Connect(ctx) + err = processSQLiteError(err) + if err != nil { + return nil, err + } + return &sqliteConn{conn.(connIface)}, nil +} + type sqliteConn struct{ connIface } func (c *sqliteConn) Begin() (driver.Tx, error) { @@ -81,26 +108,16 @@ func (c *sqliteConn) ExecContext(ctx context.Context, query string, args []drive return } -func (c *sqliteConn) Query(query string, args []driver.Value) (driver.Rows, error) { - return c.QueryContext(context.Background(), query, db.ToNamedValues(args)) -} - -func (c *sqliteConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (rows driver.Rows, err error) { - rows, err = c.connIface.QueryContext(ctx, query, args) - err = processSQLiteError(err) - if err != nil { - return nil, err - } - return &sqliteRows{rows.(rowsIface)}, nil -} - func (c *sqliteConn) Close() (err error) { + // Get acces the underlying raw sqlite3 conn. + raw := c.connIface.(sqlite3.DriverConn).Raw() + // see: https://www.sqlite.org/pragma.html#pragma_optimize const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;" - _, _ = c.connIface.ExecContext(context.Background(), onClose, nil) + _ = raw.Exec(onClose) - // Finally, close the conn. - err = c.connIface.Close() + // Finally, close. + err = raw.Close() return } @@ -164,7 +181,7 @@ func (r *sqliteRows) Close() (err error) { } // connIface is the driver.Conn interface -// types (and the like) that modernc.org/sqlite.conn +// types (and the like) that go-sqlite3/driver.conn // conforms to. Useful so you don't need // to repeatedly perform checks yourself. type connIface interface { @@ -172,11 +189,10 @@ type connIface interface { driver.ConnBeginTx driver.ConnPrepareContext driver.ExecerContext - driver.QueryerContext } // StmtIface is the driver.Stmt interface -// types (and the like) that modernc.org/sqlite.stmt +// types (and the like) that go-sqlite3/driver.stmt // conforms to. Useful so you don't need // to repeatedly perform checks yourself. type stmtIface interface { @@ -186,12 +202,10 @@ type stmtIface interface { } // RowsIface is the driver.Rows interface -// types (and the like) that modernc.org/sqlite.rows +// types (and the like) that go-sqlite3/driver.rows // conforms to. Useful so you don't need // to repeatedly perform checks yourself. type rowsIface interface { driver.Rows driver.RowsColumnTypeDatabaseTypeName - driver.RowsColumnTypeLength - driver.RowsColumnTypeScanType } diff --git a/internal/db/sqlite/driver_wasmsqlite3.go b/internal/db/sqlite/driver_moderncsqlite3.go similarity index 80% rename from internal/db/sqlite/driver_wasmsqlite3.go rename to internal/db/sqlite/driver_moderncsqlite3.go index afe499a98..7cb31efea 100644 --- a/internal/db/sqlite/driver_wasmsqlite3.go +++ b/internal/db/sqlite/driver_moderncsqlite3.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -//go:build wasmsqlite3 +//go:build moderncsqlite3 package sqlite @@ -23,22 +23,19 @@ import ( "context" "database/sql/driver" - "github.com/superseriousbusiness/gotosocial/internal/db" + "modernc.org/sqlite" - "github.com/ncruces/go-sqlite3" - sqlite3driver "github.com/ncruces/go-sqlite3/driver" - _ "github.com/ncruces/go-sqlite3/embed" // embed wasm binary - _ "github.com/ncruces/go-sqlite3/vfs/memdb" // include memdb vfs + "github.com/superseriousbusiness/gotosocial/internal/db" ) // Driver is our own wrapper around the -// driver.SQLite{} type in order to wrap +// sqlite.Driver{} type in order to wrap // further SQL types with our own // functionality, e.g. err processing. -type Driver struct{ sqlite3driver.SQLite } +type Driver struct{ sqlite.Driver } func (d *Driver) Open(name string) (driver.Conn, error) { - conn, err := d.SQLite.Open(name) + conn, err := d.Driver.Open(name) if err != nil { err = processSQLiteError(err) return nil, err @@ -46,30 +43,6 @@ func (d *Driver) Open(name string) (driver.Conn, error) { return &sqliteConn{conn.(connIface)}, nil } -func (d *Driver) OpenConnector(name string) (driver.Connector, error) { - cc, err := d.SQLite.OpenConnector(name) - if err != nil { - return nil, err - } - return &sqliteConnector{driver: d, Connector: cc}, nil -} - -type sqliteConnector struct { - driver *Driver - driver.Connector -} - -func (c *sqliteConnector) Driver() driver.Driver { return c.driver } - -func (c *sqliteConnector) Connect(ctx context.Context) (driver.Conn, error) { - conn, err := c.Connector.Connect(ctx) - err = processSQLiteError(err) - if err != nil { - return nil, err - } - return &sqliteConn{conn.(connIface)}, nil -} - type sqliteConn struct{ connIface } func (c *sqliteConn) Begin() (driver.Tx, error) { @@ -108,16 +81,26 @@ func (c *sqliteConn) ExecContext(ctx context.Context, query string, args []drive return } -func (c *sqliteConn) Close() (err error) { - // Get acces the underlying raw sqlite3 conn. - raw := c.connIface.(sqlite3.DriverConn).Raw() +func (c *sqliteConn) Query(query string, args []driver.Value) (driver.Rows, error) { + return c.QueryContext(context.Background(), query, db.ToNamedValues(args)) +} +func (c *sqliteConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (rows driver.Rows, err error) { + rows, err = c.connIface.QueryContext(ctx, query, args) + err = processSQLiteError(err) + if err != nil { + return nil, err + } + return &sqliteRows{rows.(rowsIface)}, nil +} + +func (c *sqliteConn) Close() (err error) { // see: https://www.sqlite.org/pragma.html#pragma_optimize const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;" - _ = raw.Exec(onClose) + _, _ = c.connIface.ExecContext(context.Background(), onClose, nil) - // Finally, close. - err = raw.Close() + // Finally, close the conn. + err = c.connIface.Close() return } @@ -181,7 +164,7 @@ func (r *sqliteRows) Close() (err error) { } // connIface is the driver.Conn interface -// types (and the like) that go-sqlite3/driver.conn +// types (and the like) that modernc.org/sqlite.conn // conforms to. Useful so you don't need // to repeatedly perform checks yourself. type connIface interface { @@ -189,10 +172,11 @@ type connIface interface { driver.ConnBeginTx driver.ConnPrepareContext driver.ExecerContext + driver.QueryerContext } // StmtIface is the driver.Stmt interface -// types (and the like) that go-sqlite3/driver.stmt +// types (and the like) that modernc.org/sqlite.stmt // conforms to. Useful so you don't need // to repeatedly perform checks yourself. type stmtIface interface { @@ -202,10 +186,12 @@ type stmtIface interface { } // RowsIface is the driver.Rows interface -// types (and the like) that go-sqlite3/driver.rows +// types (and the like) that modernc.org/sqlite.rows // conforms to. Useful so you don't need // to repeatedly perform checks yourself. type rowsIface interface { driver.Rows driver.RowsColumnTypeDatabaseTypeName + driver.RowsColumnTypeLength + driver.RowsColumnTypeScanType } diff --git a/internal/db/sqlite/errors.go b/internal/db/sqlite/errors.go index b07b026de..f814fa8a4 100644 --- a/internal/db/sqlite/errors.go +++ b/internal/db/sqlite/errors.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -//go:build !wasmsqlite3 +//go:build !moderncsqlite3 package sqlite @@ -23,9 +23,7 @@ import ( "database/sql/driver" "fmt" - "modernc.org/sqlite" - sqlite3 "modernc.org/sqlite/lib" - + "github.com/ncruces/go-sqlite3" "github.com/superseriousbusiness/gotosocial/internal/db" ) @@ -33,30 +31,30 @@ import ( // handle conversion to any of our common db types. func processSQLiteError(err error) error { // Attempt to cast as sqlite error. - sqliteErr, ok := err.(*sqlite.Error) + sqliteErr, ok := err.(*sqlite3.Error) if !ok { return err } // Handle supplied error code: - switch sqliteErr.Code() { - case sqlite3.SQLITE_CONSTRAINT_UNIQUE, - sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: + switch sqliteErr.ExtendedCode() { + case sqlite3.CONSTRAINT_UNIQUE, + sqlite3.CONSTRAINT_PRIMARYKEY: return db.ErrAlreadyExists - // Busy should be very rare, but - // on busy tell the database to close - // the connection, re-open and re-attempt - // which should give a necessary timeout. - case sqlite3.SQLITE_BUSY, - sqlite3.SQLITE_BUSY_RECOVERY, - sqlite3.SQLITE_BUSY_SNAPSHOT: + // Busy should be very rare, but on + // busy tell the database to close the + // connection, re-open and re-attempt + // which should give necessary timeout. + case sqlite3.BUSY_RECOVERY, + sqlite3.BUSY_SNAPSHOT: return driver.ErrBadConn } // Wrap the returned error with the code and // extended code for easier debugging later. - return fmt.Errorf("%w (code=%d)", err, + return fmt.Errorf("%w (code=%d extended=%d)", err, sqliteErr.Code(), + sqliteErr.ExtendedCode(), ) } diff --git a/internal/db/sqlite/errors_wasmsqlite3.go b/internal/db/sqlite/errors_moderncsqlite3.go similarity index 71% rename from internal/db/sqlite/errors_wasmsqlite3.go rename to internal/db/sqlite/errors_moderncsqlite3.go index 26668a898..b17cebefb 100644 --- a/internal/db/sqlite/errors_wasmsqlite3.go +++ b/internal/db/sqlite/errors_moderncsqlite3.go @@ -15,7 +15,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -//go:build wasmsqlite3 +//go:build moderncsqlite3 package sqlite @@ -23,7 +23,9 @@ import ( "database/sql/driver" "fmt" - "github.com/ncruces/go-sqlite3" + "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" + "github.com/superseriousbusiness/gotosocial/internal/db" ) @@ -31,30 +33,30 @@ import ( // handle conversion to any of our common db types. func processSQLiteError(err error) error { // Attempt to cast as sqlite error. - sqliteErr, ok := err.(*sqlite3.Error) + sqliteErr, ok := err.(*sqlite.Error) if !ok { return err } // Handle supplied error code: - switch sqliteErr.ExtendedCode() { - case sqlite3.CONSTRAINT_UNIQUE, - sqlite3.CONSTRAINT_PRIMARYKEY: + switch sqliteErr.Code() { + case sqlite3.SQLITE_CONSTRAINT_UNIQUE, + sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: return db.ErrAlreadyExists - // Busy should be very rare, but on - // busy tell the database to close the - // connection, re-open and re-attempt - // which should give necessary timeout. - case sqlite3.BUSY_RECOVERY, - sqlite3.BUSY_SNAPSHOT: + // Busy should be very rare, but + // on busy tell the database to close + // the connection, re-open and re-attempt + // which should give a necessary timeout. + case sqlite3.SQLITE_BUSY, + sqlite3.SQLITE_BUSY_RECOVERY, + sqlite3.SQLITE_BUSY_SNAPSHOT: return driver.ErrBadConn } // Wrap the returned error with the code and // extended code for easier debugging later. - return fmt.Errorf("%w (code=%d extended=%d)", err, + return fmt.Errorf("%w (code=%d)", err, sqliteErr.Code(), - sqliteErr.ExtendedCode(), ) } diff --git a/scripts/build.sh b/scripts/build.sh index f65e5b9b9..5b10a5493 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -15,14 +15,14 @@ GO_GCFLAGS=${GO_GCFLAGS-} GO_BUILDTAGS="${GO_BUILDTAGS} debugenv" # Available Go build tags, with explanation, followed by benefits of enabling it: -# - kvformat: enables prettier output of log fields (slightly better performance) -# - timetzdata: embed timezone database inside binary (allow setting local time inside Docker containers, at cost of 450KB) -# - notracing: disables compiling-in otel tracing support (reduced binary size, better performance) -# - nometrics: disables compiling-in otel metrics support (reduced binary size, better performance) -# - noerrcaller: disables caller function prefix in errors (slightly better performance, at cost of err readability) -# - debug: enables /debug/pprof endpoint (adds debug, at performance cost) -# - debugenv: enables /debug/pprof endpoint if DEBUG=1 env during runtime (adds debug, at performance cost) -# - wasmsqlite3: uses SQLite through WASM instead of the C-to-Go transpilation (experimental) +# - kvformat: enables prettier output of log fields (slightly better performance) +# - timetzdata: embed timezone database inside binary (allow setting local time inside Docker containers, at cost of 450KB) +# - notracing: disables compiling-in otel tracing support (reduced binary size, better performance) +# - nometrics: disables compiling-in otel metrics support (reduced binary size, better performance) +# - noerrcaller: disables caller function prefix in errors (slightly better performance, at cost of err readability) +# - debug: enables /debug/pprof endpoint (adds debug, at performance cost) +# - debugenv: enables /debug/pprof endpoint if DEBUG=1 env during runtime (adds debug, at performance cost) +# - moderncsqlite3: reverts to using the C-to-Go transpiled SQLite driver (disables the WASM-based SQLite driver) log_exec env CGO_ENABLED=0 go build -trimpath -v \ -tags "${GO_BUILDTAGS}" \ -ldflags="${GO_LDFLAGS}" \