mirrors
/
gotosocial
mirror of
1
Fork 0
Code Issues Releases Activity

[feature] Default to WASM-based SQLite driver (#3053) 

* [feature] Default to WASM-based SQLite driver

With 0.16 out this switches our default SQLite driver to the WASM-based
solution instead. So far the driver seems to perform just as well.
Switching our default should result in it getting a bit more testing
during the 0.17 development cycle.

* add the ol' john hancock

---------

Co-authored-by: tobi <tobi.smethurst@protonmail.com>
This commit is contained in:
Daenney 2024-06-29 09:35:57 +02:00 committed by GitHub
parent 86786ae5b3
commit 137ef5a9ff
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 111 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.drone.yml
View File

@ -42,7 +42,7 @@ steps:
go test
-failfast
-timeout=20m
-tags "wasmsqlite3 netgo osusergo static_build kvformat timetzdata"
-tags "netgo osusergo static_build kvformat timetzdata"
./...
- ./test/envparsing.sh
- ./test/swagger.sh
@ -204,6 +204,6 @@ steps:
---
kind: signature
hmac: 2e74313f4192b3e6daf6d1d00a7c3796019d93da7ce7e0a77208ccc3c37089b0
hmac: 86ebddcd630792cac43aa92fa7f45118943c51b5157491d05eb480ac21762329
...

2
.goreleaser.yml
View File

@ -30,7 +30,7 @@ builds:
- >-
{{ if and (index .Env "DEBUG") (.Env.DEBUG) }}debugenv{{ end }}
- >-
{{ if and (index .Env "WASMSQLITE3") (.Env.WASMSQLITE3) }}wasmsqlite3{{ end }}
{{ if and (index .Env "MODERNCSQLITE3") (.Env.MODERNCSQLITE3) }}moderncsqlite3{{ end }}
env:
- CGO_ENABLED=0
goos:

70
internal/db/sqlite/driver.go
View File

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build !wasmsqlite3
//go:build !moderncsqlite3
package sqlite
@ -23,19 +23,22 @@ import (
"context"
"database/sql/driver"
"modernc.org/sqlite"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/ncruces/go-sqlite3"
sqlite3driver "github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed" // embed wasm binary
_ "github.com/ncruces/go-sqlite3/vfs/memdb" // include memdb vfs
)
// Driver is our own wrapper around the
// sqlite.Driver{} type in order to wrap
// driver.SQLite{} type in order to wrap
// further SQL types with our own
// functionality, e.g. err processing.
type Driver struct{ sqlite.Driver }
type Driver struct{ sqlite3driver.SQLite }
func (d *Driver) Open(name string) (driver.Conn, error) {
conn, err := d.Driver.Open(name)
conn, err := d.SQLite.Open(name)
if err != nil {
err = processSQLiteError(err)
return nil, err
@ -43,6 +46,30 @@ func (d *Driver) Open(name string) (driver.Conn, error) {
return &sqliteConn{conn.(connIface)}, nil
}
func (d *Driver) OpenConnector(name string) (driver.Connector, error) {
cc, err := d.SQLite.OpenConnector(name)
if err != nil {
return nil, err
}
return &sqliteConnector{driver: d, Connector: cc}, nil
}
type sqliteConnector struct {
driver *Driver
driver.Connector
}
func (c *sqliteConnector) Driver() driver.Driver { return c.driver }
func (c *sqliteConnector) Connect(ctx context.Context) (driver.Conn, error) {
conn, err := c.Connector.Connect(ctx)
err = processSQLiteError(err)
if err != nil {
return nil, err
}
return &sqliteConn{conn.(connIface)}, nil
}
type sqliteConn struct{ connIface }
func (c *sqliteConn) Begin() (driver.Tx, error) {
@ -81,26 +108,16 @@ func (c *sqliteConn) ExecContext(ctx context.Context, query string, args []drive
return
}
func (c *sqliteConn) Query(query string, args []driver.Value) (driver.Rows, error) {
return c.QueryContext(context.Background(), query, db.ToNamedValues(args))
}
func (c *sqliteConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (rows driver.Rows, err error) {
rows, err = c.connIface.QueryContext(ctx, query, args)
err = processSQLiteError(err)
if err != nil {
return nil, err
}
return &sqliteRows{rows.(rowsIface)}, nil
}
func (c *sqliteConn) Close() (err error) {
// Get acces the underlying raw sqlite3 conn.
raw := c.connIface.(sqlite3.DriverConn).Raw()
// see: https://www.sqlite.org/pragma.html#pragma_optimize
const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;"
_, _ = c.connIface.ExecContext(context.Background(), onClose, nil)
_ = raw.Exec(onClose)
// Finally, close the conn.
err = c.connIface.Close()
// Finally, close.
err = raw.Close()
return
}
@ -164,7 +181,7 @@ func (r *sqliteRows) Close() (err error) {
}
// connIface is the driver.Conn interface
// types (and the like) that modernc.org/sqlite.conn
// types (and the like) that go-sqlite3/driver.conn
// conforms to. Useful so you don't need
// to repeatedly perform checks yourself.
type connIface interface {
@ -172,11 +189,10 @@ type connIface interface {
driver.ConnBeginTx
driver.ConnPrepareContext
driver.ExecerContext
driver.QueryerContext
}
// StmtIface is the driver.Stmt interface
// types (and the like) that modernc.org/sqlite.stmt
// types (and the like) that go-sqlite3/driver.stmt
// conforms to. Useful so you don't need
// to repeatedly perform checks yourself.
type stmtIface interface {
@ -186,12 +202,10 @@ type stmtIface interface {
}
// RowsIface is the driver.Rows interface
// types (and the like) that modernc.org/sqlite.rows
// types (and the like) that go-sqlite3/driver.rows
// conforms to. Useful so you don't need
// to repeatedly perform checks yourself.
type rowsIface interface {
driver.Rows
driver.RowsColumnTypeDatabaseTypeName
driver.RowsColumnTypeLength
driver.RowsColumnTypeScanType
}

70
internal/db/sqlite/driver_wasmsqlite3.go → internal/db/sqlite/driver_moderncsqlite3.go
View File

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build wasmsqlite3
//go:build moderncsqlite3
package sqlite
@ -23,22 +23,19 @@ import (
"context"
"database/sql/driver"
"github.com/superseriousbusiness/gotosocial/internal/db"
"modernc.org/sqlite"
"github.com/ncruces/go-sqlite3"
sqlite3driver "github.com/ncruces/go-sqlite3/driver"
_ "github.com/ncruces/go-sqlite3/embed" // embed wasm binary
_ "github.com/ncruces/go-sqlite3/vfs/memdb" // include memdb vfs
"github.com/superseriousbusiness/gotosocial/internal/db"
)
// Driver is our own wrapper around the
// driver.SQLite{} type in order to wrap
// sqlite.Driver{} type in order to wrap
// further SQL types with our own
// functionality, e.g. err processing.
type Driver struct{ sqlite3driver.SQLite }
type Driver struct{ sqlite.Driver }
func (d *Driver) Open(name string) (driver.Conn, error) {
conn, err := d.SQLite.Open(name)
conn, err := d.Driver.Open(name)
if err != nil {
err = processSQLiteError(err)
return nil, err
@ -46,30 +43,6 @@ func (d *Driver) Open(name string) (driver.Conn, error) {
return &sqliteConn{conn.(connIface)}, nil
}
func (d *Driver) OpenConnector(name string) (driver.Connector, error) {
cc, err := d.SQLite.OpenConnector(name)
if err != nil {
return nil, err
}
return &sqliteConnector{driver: d, Connector: cc}, nil
}
type sqliteConnector struct {
driver *Driver
driver.Connector
}
func (c *sqliteConnector) Driver() driver.Driver { return c.driver }
func (c *sqliteConnector) Connect(ctx context.Context) (driver.Conn, error) {
conn, err := c.Connector.Connect(ctx)
err = processSQLiteError(err)
if err != nil {
return nil, err
}
return &sqliteConn{conn.(connIface)}, nil
}
type sqliteConn struct{ connIface }
func (c *sqliteConn) Begin() (driver.Tx, error) {
@ -108,16 +81,26 @@ func (c *sqliteConn) ExecContext(ctx context.Context, query string, args []drive
return
}
func (c *sqliteConn) Close() (err error) {
// Get acces the underlying raw sqlite3 conn.
raw := c.connIface.(sqlite3.DriverConn).Raw()
func (c *sqliteConn) Query(query string, args []driver.Value) (driver.Rows, error) {
return c.QueryContext(context.Background(), query, db.ToNamedValues(args))
}
func (c *sqliteConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (rows driver.Rows, err error) {
rows, err = c.connIface.QueryContext(ctx, query, args)
err = processSQLiteError(err)
if err != nil {
return nil, err
}
return &sqliteRows{rows.(rowsIface)}, nil
}
func (c *sqliteConn) Close() (err error) {
// see: https://www.sqlite.org/pragma.html#pragma_optimize
const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;"
_ = raw.Exec(onClose)
_, _ = c.connIface.ExecContext(context.Background(), onClose, nil)
// Finally, close.
err = raw.Close()
// Finally, close the conn.
err = c.connIface.Close()
return
}
@ -181,7 +164,7 @@ func (r *sqliteRows) Close() (err error) {
}
// connIface is the driver.Conn interface
// types (and the like) that go-sqlite3/driver.conn
// types (and the like) that modernc.org/sqlite.conn
// conforms to. Useful so you don't need
// to repeatedly perform checks yourself.
type connIface interface {
@ -189,10 +172,11 @@ type connIface interface {
driver.ConnBeginTx
driver.ConnPrepareContext
driver.ExecerContext
driver.QueryerContext
}
// StmtIface is the driver.Stmt interface
// types (and the like) that go-sqlite3/driver.stmt
// types (and the like) that modernc.org/sqlite.stmt
// conforms to. Useful so you don't need
// to repeatedly perform checks yourself.
type stmtIface interface {
@ -202,10 +186,12 @@ type stmtIface interface {
}
// RowsIface is the driver.Rows interface
// types (and the like) that go-sqlite3/driver.rows
// types (and the like) that modernc.org/sqlite.rows
// conforms to. Useful so you don't need
// to repeatedly perform checks yourself.
type rowsIface interface {
driver.Rows
driver.RowsColumnTypeDatabaseTypeName
driver.RowsColumnTypeLength
driver.RowsColumnTypeScanType
}

30
internal/db/sqlite/errors.go
View File

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build !wasmsqlite3
//go:build !moderncsqlite3
package sqlite
@ -23,9 +23,7 @@ import (
"database/sql/driver"
"fmt"
"modernc.org/sqlite"
sqlite3 "modernc.org/sqlite/lib"
"github.com/ncruces/go-sqlite3"
"github.com/superseriousbusiness/gotosocial/internal/db"
)
@ -33,30 +31,30 @@ import (
// handle conversion to any of our common db types.
func processSQLiteError(err error) error {
// Attempt to cast as sqlite error.
sqliteErr, ok := err.(*sqlite.Error)
sqliteErr, ok := err.(*sqlite3.Error)
if !ok {
return err
}
// Handle supplied error code:
switch sqliteErr.Code() {
case sqlite3.SQLITE_CONSTRAINT_UNIQUE,
sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY:
switch sqliteErr.ExtendedCode() {
case sqlite3.CONSTRAINT_UNIQUE,
sqlite3.CONSTRAINT_PRIMARYKEY:
return db.ErrAlreadyExists
// Busy should be very rare, but
// on busy tell the database to close
// the connection, re-open and re-attempt
// which should give a necessary timeout.
case sqlite3.SQLITE_BUSY,
sqlite3.SQLITE_BUSY_RECOVERY,
sqlite3.SQLITE_BUSY_SNAPSHOT:
// Busy should be very rare, but on
// busy tell the database to close the
// connection, re-open and re-attempt
// which should give necessary timeout.
case sqlite3.BUSY_RECOVERY,
sqlite3.BUSY_SNAPSHOT:
return driver.ErrBadConn
}
// Wrap the returned error with the code and
// extended code for easier debugging later.
return fmt.Errorf("%w (code=%d)", err,
return fmt.Errorf("%w (code=%d extended=%d)", err,
sqliteErr.Code(),
sqliteErr.ExtendedCode(),
)
}

30
internal/db/sqlite/errors_wasmsqlite3.go → internal/db/sqlite/errors_moderncsqlite3.go
View File

@ -15,7 +15,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//go:build wasmsqlite3
//go:build moderncsqlite3
package sqlite
@ -23,7 +23,9 @@ import (
"database/sql/driver"
"fmt"
"github.com/ncruces/go-sqlite3"
"modernc.org/sqlite"
sqlite3 "modernc.org/sqlite/lib"
"github.com/superseriousbusiness/gotosocial/internal/db"
)
@ -31,30 +33,30 @@ import (
// handle conversion to any of our common db types.
func processSQLiteError(err error) error {
// Attempt to cast as sqlite error.
sqliteErr, ok := err.(*sqlite3.Error)
sqliteErr, ok := err.(*sqlite.Error)
if !ok {
return err
}
// Handle supplied error code:
switch sqliteErr.ExtendedCode() {
case sqlite3.CONSTRAINT_UNIQUE,
sqlite3.CONSTRAINT_PRIMARYKEY:
switch sqliteErr.Code() {
case sqlite3.SQLITE_CONSTRAINT_UNIQUE,
sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY:
return db.ErrAlreadyExists
// Busy should be very rare, but on
// busy tell the database to close the
// connection, re-open and re-attempt
// which should give necessary timeout.
case sqlite3.BUSY_RECOVERY,
sqlite3.BUSY_SNAPSHOT:
// Busy should be very rare, but
// on busy tell the database to close
// the connection, re-open and re-attempt
// which should give a necessary timeout.
case sqlite3.SQLITE_BUSY,
sqlite3.SQLITE_BUSY_RECOVERY,
sqlite3.SQLITE_BUSY_SNAPSHOT:
return driver.ErrBadConn
}
// Wrap the returned error with the code and
// extended code for easier debugging later.
return fmt.Errorf("%w (code=%d extended=%d)", err,
return fmt.Errorf("%w (code=%d)", err,
sqliteErr.Code(),
sqliteErr.ExtendedCode(),
)
}

16
scripts/build.sh
View File

@ -15,14 +15,14 @@ GO_GCFLAGS=${GO_GCFLAGS-}
GO_BUILDTAGS="${GO_BUILDTAGS} debugenv"
# Available Go build tags, with explanation, followed by benefits of enabling it:
# - kvformat: enables prettier output of log fields (slightly better performance)
# - timetzdata: embed timezone database inside binary (allow setting local time inside Docker containers, at cost of 450KB)
# - notracing: disables compiling-in otel tracing support (reduced binary size, better performance)
# - nometrics: disables compiling-in otel metrics support (reduced binary size, better performance)
# - noerrcaller: disables caller function prefix in errors (slightly better performance, at cost of err readability)
# - debug: enables /debug/pprof endpoint (adds debug, at performance cost)
# - debugenv: enables /debug/pprof endpoint if DEBUG=1 env during runtime (adds debug, at performance cost)
# - wasmsqlite3: uses SQLite through WASM instead of the C-to-Go transpilation (experimental)
# - kvformat: enables prettier output of log fields (slightly better performance)
# - timetzdata: embed timezone database inside binary (allow setting local time inside Docker containers, at cost of 450KB)
# - notracing: disables compiling-in otel tracing support (reduced binary size, better performance)
# - nometrics: disables compiling-in otel metrics support (reduced binary size, better performance)
# - noerrcaller: disables caller function prefix in errors (slightly better performance, at cost of err readability)
# - debug: enables /debug/pprof endpoint (adds debug, at performance cost)
# - debugenv: enables /debug/pprof endpoint if DEBUG=1 env during runtime (adds debug, at performance cost)
# - moderncsqlite3: reverts to using the C-to-Go transpiled SQLite driver (disables the WASM-based SQLite driver)
log_exec env CGO_ENABLED=0 go build -trimpath -v \
-tags "${GO_BUILDTAGS}" \
-ldflags="${GO_LDFLAGS}" \