2023-03-12 16:00:57 +01:00
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
2021-03-02 18:26:30 +01:00
2021-08-25 15:34:33 +02:00
package bundb
2021-03-02 18:26:30 +01:00
import (
"context"
2021-07-19 18:03:07 +02:00
"crypto/tls"
"crypto/x509"
2021-08-25 15:34:33 +02:00
"database/sql"
2021-07-19 18:03:07 +02:00
"encoding/pem"
2021-03-02 18:26:30 +01:00
"errors"
"fmt"
2024-10-16 14:13:58 +02:00
"math"
2023-09-01 15:13:33 +02:00
"net/url"
2021-07-19 18:03:07 +02:00
"os"
2021-09-20 18:20:21 +02:00
"runtime"
2021-03-03 18:12:02 +01:00
"strings"
2021-03-02 22:52:31 +01:00
"time"
2021-03-02 18:26:30 +01:00
2023-01-17 13:29:44 +01:00
"codeberg.org/gruf/go-bytesize"
2022-08-22 11:21:36 +02:00
"github.com/google/uuid"
2023-05-12 14:33:40 +02:00
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/stdlib"
2021-04-01 20:46:45 +02:00
"github.com/superseriousbusiness/gotosocial/internal/config"
2021-05-15 11:58:11 +02:00
"github.com/superseriousbusiness/gotosocial/internal/db"
2021-08-31 19:27:02 +02:00
"github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations"
2021-05-08 14:25:55 +02:00
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
2022-07-19 10:47:55 +02:00
"github.com/superseriousbusiness/gotosocial/internal/log"
2023-11-20 16:43:55 +01:00
"github.com/superseriousbusiness/gotosocial/internal/metrics"
2022-12-08 18:35:14 +01:00
"github.com/superseriousbusiness/gotosocial/internal/state"
2023-05-09 19:19:48 +02:00
"github.com/superseriousbusiness/gotosocial/internal/tracing"
2021-08-25 15:34:33 +02:00
"github.com/uptrace/bun"
2023-12-16 12:54:53 +01:00
"github.com/uptrace/bun/dialect"
2021-08-25 15:34:33 +02:00
"github.com/uptrace/bun/dialect/pgdialect"
2021-08-29 16:41:41 +02:00
"github.com/uptrace/bun/dialect/sqlitedialect"
2021-08-31 19:27:02 +02:00
"github.com/uptrace/bun/migrate"
2024-11-29 16:03:10 +01:00
"github.com/uptrace/bun/schema"
2021-08-25 15:34:33 +02:00
)
2022-09-26 11:56:01 +02:00
// DBService satisfies the DB interface
type DBService struct {
2021-08-20 12:26:56 +02:00
db . Account
db . Admin
2024-07-23 21:44:31 +02:00
db . AdvancedMigration
2023-08-10 16:08:41 +02:00
db . Application
2021-08-20 12:26:56 +02:00
db . Basic
2024-07-23 21:44:31 +02:00
db . Conversation
2021-08-20 12:26:56 +02:00
db . Domain
2022-05-20 10:34:36 +02:00
db . Emoji
2023-12-18 15:18:25 +01:00
db . HeaderFilter
2021-08-20 12:26:56 +02:00
db . Instance
2024-07-11 16:44:29 +02:00
db . Interaction
2024-03-06 11:15:58 +01:00
db . Filter
2023-05-25 10:37:38 +02:00
db . List
2023-07-29 12:49:14 +02:00
db . Marker
2021-08-20 12:26:56 +02:00
db . Media
db . Mention
2024-03-06 11:18:57 +01:00
db . Move
2021-08-20 12:26:56 +02:00
db . Notification
2023-11-08 15:32:17 +01:00
db . Poll
2021-08-20 12:26:56 +02:00
db . Relationship
2023-01-10 15:19:05 +01:00
db . Report
2023-08-19 14:33:15 +02:00
db . Rule
2023-06-21 18:26:40 +02:00
db . Search
2021-08-25 15:34:33 +02:00
db . Session
2024-09-10 14:34:49 +02:00
db . SinBinStatus
2021-08-20 12:26:56 +02:00
db . Status
2023-03-20 19:10:08 +01:00
db . StatusBookmark
db . StatusFave
2023-07-31 15:47:35 +02:00
db . Tag
2023-10-25 16:04:53 +02:00
db . Thread
2021-08-20 12:26:56 +02:00
db . Timeline
2022-10-03 10:46:11 +02:00
db . User
2022-11-11 12:18:38 +01:00
db . Tombstone
2024-07-30 13:58:31 +02:00
db . WorkerTask
2024-02-07 15:43:27 +01:00
db * bun . DB
2021-03-02 18:26:30 +01:00
}
2023-07-25 10:34:05 +02:00
// GetDB returns the underlying database connection pool.
2022-09-26 11:56:01 +02:00
// Should only be used in testing + exceptional circumstance.
2024-02-07 15:43:27 +01:00
func ( dbService * DBService ) DB ( ) * bun . DB {
2023-07-25 10:34:05 +02:00
return dbService . db
2022-09-26 11:56:01 +02:00
}
2021-10-11 14:37:33 +02:00
func doMigration ( ctx context . Context , db * bun . DB ) error {
2021-08-31 19:27:02 +02:00
migrator := migrate . NewMigrator ( db , migrations . Migrations )
if err := migrator . Init ( ctx ) ; err != nil {
return err
}
group , err := migrator . Migrate ( ctx )
2023-05-12 14:33:40 +02:00
if err != nil && ! strings . Contains ( err . Error ( ) , "no migrations" ) {
2021-08-31 19:27:02 +02:00
return err
}
2023-05-12 14:33:40 +02:00
if group == nil || group . ID == 0 {
2023-02-17 12:02:29 +01:00
log . Info ( ctx , "there are no new migrations to run" )
2021-08-31 19:27:02 +02:00
return nil
}
2023-02-17 12:02:29 +01:00
log . Infof ( ctx , "MIGRATED DATABASE TO %s" , group )
2023-12-16 12:54:53 +01:00
if db . Dialect ( ) . Name ( ) == dialect . SQLite {
2024-03-14 18:40:36 +01:00
log . Info ( ctx ,
"running ANALYZE to update table and index statistics; this will take somewhere between " +
"1-10 minutes, or maybe longer depending on your hardware and database size, please be patient" ,
)
2023-12-16 12:54:53 +01:00
_ , err := db . ExecContext ( ctx , "ANALYZE" )
if err != nil {
log . Warnf ( ctx , "ANALYZE failed, query planner may make poor life choices: %s" , err )
}
}
2021-08-31 19:27:02 +02:00
return nil
}
2021-08-25 15:34:33 +02:00
// NewBunDBService returns a bunDB derived from the provided config, which implements the go-fed DB interface.
// Under the hood, it uses https://github.com/uptrace/bun to create and maintain a database connection.
2022-12-08 18:35:14 +01:00
func NewBunDBService ( ctx context . Context , state * state . State ) ( db . DB , error ) {
2024-11-29 16:03:10 +01:00
var sqldb * sql . DB
var dialect func ( ) schema . Dialect
2021-11-21 17:41:51 +01:00
var err error
2021-12-07 13:31:39 +01:00
2024-11-29 16:03:10 +01:00
switch t := strings . ToLower ( config . GetDbType ( ) ) ; t {
2023-01-17 13:29:44 +01:00
case "postgres" :
2024-11-29 16:03:10 +01:00
sqldb , dialect , err = pgConn ( ctx )
2021-08-25 15:34:33 +02:00
if err != nil {
2021-11-21 17:41:51 +01:00
return nil , err
2021-08-25 15:34:33 +02:00
}
2023-01-17 13:29:44 +01:00
case "sqlite" :
2024-11-29 16:03:10 +01:00
sqldb , dialect , err = sqliteConn ( ctx )
2021-08-29 16:41:41 +02:00
if err != nil {
2021-11-21 17:41:51 +01:00
return nil , err
2021-08-29 16:41:41 +02:00
}
2021-08-25 15:34:33 +02:00
default :
2023-01-17 13:29:44 +01:00
return nil , fmt . Errorf ( "database type %s not supported for bundb" , t )
2021-03-05 18:31:12 +01:00
}
2024-11-29 16:03:10 +01:00
// perform any pending database migrations: this includes the first
// 'migration' on startup which just creates necessary db tables.
//
// Note this uses its own instance of bun.DB as bun will automatically
// store in-memory reflect type schema of any Go models passed to it,
// and we still maintain lots of old model versions in the migrations.
if err := doMigration ( ctx , bunDB ( sqldb , dialect ) ) ; err != nil {
2021-08-31 19:27:02 +02:00
return nil , fmt . Errorf ( "db migration error: %s" , err )
}
2021-09-01 11:08:21 +02:00
2024-11-29 16:03:10 +01:00
// Wrap sql.DB as bun.DB type,
// adding any connection hooks.
db := bunDB ( sqldb , dialect )
2022-09-26 11:56:01 +02:00
ps := & DBService {
2022-12-08 18:35:14 +01:00
Account : & accountDB {
2023-07-25 10:34:05 +02:00
db : db ,
2022-12-08 18:35:14 +01:00
state : state ,
} ,
2021-08-20 12:26:56 +02:00
Admin : & adminDB {
2023-07-25 10:34:05 +02:00
db : db ,
2022-12-08 18:35:14 +01:00
state : state ,
2021-08-20 12:26:56 +02:00
} ,
2024-07-23 21:44:31 +02:00
AdvancedMigration : & advancedMigrationDB {
db : db ,
state : state ,
} ,
2023-08-10 16:08:41 +02:00
Application : & applicationDB {
db : db ,
state : state ,
} ,
2021-08-20 12:26:56 +02:00
Basic : & basicDB {
2023-07-25 10:34:05 +02:00
db : db ,
2021-08-20 12:26:56 +02:00
} ,
2024-07-23 21:44:31 +02:00
Conversation : & conversationDB {
db : db ,
state : state ,
} ,
2022-12-08 18:35:14 +01:00
Domain : & domainDB {
2023-07-25 10:34:05 +02:00
db : db ,
2022-12-08 18:35:14 +01:00
state : state ,
} ,
Emoji : & emojiDB {
2023-07-25 10:34:05 +02:00
db : db ,
2022-12-08 18:35:14 +01:00
state : state ,
} ,
2023-12-18 15:18:25 +01:00
HeaderFilter : & headerFilterDB {
db : db ,
state : state ,
} ,
2021-08-20 12:26:56 +02:00
Instance : & instanceDB {
2023-07-25 10:34:05 +02:00
db : db ,
2023-07-07 11:34:12 +02:00
state : state ,
2021-08-20 12:26:56 +02:00
} ,
2024-07-11 16:44:29 +02:00
Interaction : & interactionDB {
db : db ,
state : state ,
} ,
2024-03-06 11:15:58 +01:00
Filter : & filterDB {
db : db ,
state : state ,
} ,
2023-05-25 10:37:38 +02:00
List : & listDB {
2023-07-25 10:34:05 +02:00
db : db ,
2023-05-25 10:37:38 +02:00
state : state ,
} ,
2023-07-29 12:49:14 +02:00
Marker : & markerDB {
db : db ,
state : state ,
} ,
2021-08-20 12:26:56 +02:00
Media : & mediaDB {
2023-07-25 10:34:05 +02:00
db : db ,
2023-02-13 21:19:51 +01:00
state : state ,
2021-08-20 12:26:56 +02:00
} ,
2022-12-08 18:35:14 +01:00
Mention : & mentionDB {
2023-07-25 10:34:05 +02:00
db : db ,
2022-12-08 18:35:14 +01:00
state : state ,
} ,
2024-03-06 11:18:57 +01:00
Move : & moveDB {
db : db ,
state : state ,
} ,
2022-12-08 18:35:14 +01:00
Notification : & notificationDB {
2023-07-25 10:34:05 +02:00
db : db ,
2022-12-08 18:35:14 +01:00
state : state ,
} ,
2023-11-08 15:32:17 +01:00
Poll : & pollDB {
db : db ,
state : state ,
} ,
2022-12-08 18:35:14 +01:00
Relationship : & relationshipDB {
2023-07-25 10:34:05 +02:00
db : db ,
2022-12-08 18:35:14 +01:00
state : state ,
} ,
2023-01-10 15:19:05 +01:00
Report : & reportDB {
2023-07-25 10:34:05 +02:00
db : db ,
2023-01-10 15:19:05 +01:00
state : state ,
} ,
2023-08-19 14:33:15 +02:00
Rule : & ruleDB {
db : db ,
state : state ,
} ,
2023-06-21 18:26:40 +02:00
Search : & searchDB {
2023-07-25 10:34:05 +02:00
db : db ,
2023-06-21 18:26:40 +02:00
state : state ,
} ,
2021-08-25 15:34:33 +02:00
Session : & sessionDB {
2023-07-25 10:34:05 +02:00
db : db ,
2021-08-20 12:26:56 +02:00
} ,
2024-09-10 14:34:49 +02:00
SinBinStatus : & sinBinStatusDB {
db : db ,
state : state ,
} ,
2022-12-08 18:35:14 +01:00
Status : & statusDB {
2023-07-25 10:34:05 +02:00
db : db ,
2022-12-08 18:35:14 +01:00
state : state ,
} ,
2023-03-20 19:10:08 +01:00
StatusBookmark : & statusBookmarkDB {
2023-07-25 10:34:05 +02:00
db : db ,
2023-03-20 19:10:08 +01:00
state : state ,
} ,
StatusFave : & statusFaveDB {
2023-07-25 10:34:05 +02:00
db : db ,
2023-03-20 19:10:08 +01:00
state : state ,
} ,
2023-07-31 15:47:35 +02:00
Tag : & tagDB {
2024-01-19 13:57:29 +01:00
db : db ,
2023-07-31 15:47:35 +02:00
state : state ,
} ,
2023-10-25 16:04:53 +02:00
Thread : & threadDB {
db : db ,
state : state ,
} ,
2022-12-08 18:35:14 +01:00
Timeline : & timelineDB {
2023-07-25 10:34:05 +02:00
db : db ,
2022-12-08 18:35:14 +01:00
state : state ,
} ,
User : & userDB {
2023-07-25 10:34:05 +02:00
db : db ,
2022-12-08 18:35:14 +01:00
state : state ,
} ,
Tombstone : & tombstoneDB {
2023-07-25 10:34:05 +02:00
db : db ,
2022-12-08 18:35:14 +01:00
state : state ,
} ,
2024-07-30 13:58:31 +02:00
WorkerTask : & workerTaskDB {
db : db ,
} ,
2023-07-25 10:34:05 +02:00
db : db ,
2021-04-01 20:46:45 +02:00
}
2021-03-02 18:26:30 +01:00
2021-08-25 15:34:33 +02:00
// we can confidently return this useable service now
2021-04-01 20:46:45 +02:00
return ps , nil
2021-03-22 22:26:54 +01:00
}
2024-11-29 16:03:10 +01:00
// bunDB returns a new bun.DB for given sql.DB connection pool and dialect
// function. This can be used to apply any necessary opts / hooks as we
// initialize a bun.DB object both before and after performing migrations.
func bunDB ( sqldb * sql . DB , dialect func ( ) schema . Dialect ) * bun . DB {
db := bun . NewDB ( sqldb , dialect ( ) )
// Add our SQL connection hooks.
db . AddQueryHook ( queryHook { } )
if config . GetTracingEnabled ( ) {
db . AddQueryHook ( tracing . InstrumentBun ( ) )
}
if config . GetMetricsEnabled ( ) {
db . AddQueryHook ( metrics . InstrumentBun ( ) )
}
// table registration is needed for many-to-many, see:
// https://bun.uptrace.dev/orm/many-to-many-relation/
for _ , t := range [ ] interface { } {
& gtsmodel . AccountToEmoji { } ,
& gtsmodel . ConversationToStatus { } ,
& gtsmodel . StatusToEmoji { } ,
& gtsmodel . StatusToTag { } ,
& gtsmodel . ThreadToStatus { } ,
} {
db . RegisterModel ( t )
}
return db
}
func pgConn ( ctx context . Context ) ( * sql . DB , func ( ) schema . Dialect , error ) {
2023-01-26 15:12:48 +01:00
opts , err := deriveBunDBPGOptions ( ) //nolint:contextcheck
if err != nil {
2024-11-29 16:03:10 +01:00
return nil , nil , fmt . Errorf ( "could not create bundb postgres options: %w" , err )
2023-01-26 15:12:48 +01:00
}
2024-02-07 15:43:27 +01:00
cfg := stdlib . RegisterConnConfig ( opts )
sqldb , err := sql . Open ( "pgx-gts" , cfg )
if err != nil {
2024-11-29 16:03:10 +01:00
return nil , nil , fmt . Errorf ( "could not open postgres db: %w" , err )
2024-02-07 15:43:27 +01:00
}
2023-01-26 15:12:48 +01:00
// Tune db connections for postgres, see:
// - https://bun.uptrace.dev/guide/running-bun-in-production.html#database-sql
// - https://www.alexedwards.net/blog/configuring-sqldb
sqldb . SetMaxOpenConns ( maxOpenConns ( ) ) // x number of conns per CPU
sqldb . SetMaxIdleConns ( 2 ) // assume default 2; if max idle is less than max open, it will be automatically adjusted
sqldb . SetConnMaxLifetime ( 5 * time . Minute ) // fine to kill old connections
// ping to check the db is there and listening
2024-11-29 16:03:10 +01:00
if err := sqldb . PingContext ( ctx ) ; err != nil {
return nil , nil , fmt . Errorf ( "postgres ping: %w" , err )
2023-01-26 15:12:48 +01:00
}
2023-02-17 12:02:29 +01:00
log . Info ( ctx , "connected to POSTGRES database" )
2024-11-29 16:03:10 +01:00
return sqldb , func ( ) schema . Dialect { return pgdialect . New ( ) } , nil
2023-01-26 15:12:48 +01:00
}
2024-11-29 16:03:10 +01:00
func sqliteConn ( ctx context . Context ) ( * sql . DB , func ( ) schema . Dialect , error ) {
2022-01-30 17:06:28 +01:00
// validate db address has actually been set
2023-01-17 13:29:44 +01:00
address := config . GetDbAddress ( )
if address == "" {
2024-11-29 16:03:10 +01:00
return nil , nil , fmt . Errorf ( "'%s' was not set when attempting to start sqlite" , config . DbAddressFlag ( ) )
2022-01-30 17:06:28 +01:00
}
2021-12-07 13:31:39 +01:00
2023-09-01 15:13:33 +02:00
// Build SQLite connection address with prefs.
2024-09-08 16:14:56 +02:00
address , inMem := buildSQLiteAddress ( address )
2022-08-22 11:21:36 +02:00
2021-11-21 17:41:51 +01:00
// Open new DB instance
2024-02-07 15:43:27 +01:00
sqldb , err := sql . Open ( "sqlite-gts" , address )
2021-11-21 17:41:51 +01:00
if err != nil {
2024-11-29 16:03:10 +01:00
return nil , nil , fmt . Errorf ( "could not open sqlite db with address %s: %w" , address , err )
2021-11-21 17:41:51 +01:00
}
2023-01-26 15:12:48 +01:00
// Tune db connections for sqlite, see:
// - https://bun.uptrace.dev/guide/running-bun-in-production.html#database-sql
// - https://www.alexedwards.net/blog/configuring-sqldb
2023-07-25 10:34:05 +02:00
sqldb . SetMaxOpenConns ( maxOpenConns ( ) ) // x number of conns per CPU
sqldb . SetMaxIdleConns ( 1 ) // only keep max 1 idle connection around
2024-09-08 16:14:56 +02:00
if inMem {
log . Warn ( nil , "using sqlite in-memory mode; all data will be deleted when gts shuts down; this mode should only be used for debugging or running tests" )
// Don't close aged connections as this may wipe the DB.
sqldb . SetConnMaxLifetime ( 0 )
} else {
sqldb . SetConnMaxLifetime ( 5 * time . Minute )
}
2021-11-21 17:41:51 +01:00
// ping to check the db is there and listening
2024-11-29 16:03:10 +01:00
if err := sqldb . PingContext ( ctx ) ; err != nil {
return nil , nil , fmt . Errorf ( "sqlite ping: %w" , err )
2021-11-21 17:41:51 +01:00
}
2024-05-27 17:46:15 +02:00
2023-02-17 12:02:29 +01:00
log . Infof ( ctx , "connected to SQLITE database with address %s" , address )
2023-01-17 13:29:44 +01:00
2024-11-29 16:03:10 +01:00
return sqldb , func ( ) schema . Dialect { return sqlitedialect . New ( ) } , nil
2021-11-21 17:41:51 +01:00
}
2021-03-02 18:26:30 +01:00
/ *
HANDY STUFF
* /
2023-01-26 15:12:48 +01:00
// maxOpenConns returns multiplier * GOMAXPROCS,
2023-01-31 13:46:45 +01:00
// returning just 1 instead if multiplier < 1.
2023-01-26 15:12:48 +01:00
func maxOpenConns ( ) int {
multiplier := config . GetDbMaxOpenConnsMultiplier ( )
if multiplier < 1 {
2023-01-31 13:46:45 +01:00
return 1
2023-01-26 15:12:48 +01:00
}
2024-10-14 00:19:52 +02:00
// Specifically for SQLite databases with
// a journal mode of anything EXCEPT "wal",
// only 1 concurrent connection is supported.
if strings . ToLower ( config . GetDbType ( ) ) == "sqlite" {
journalMode := config . GetDbSqliteJournalMode ( )
journalMode = strings . ToLower ( journalMode )
if journalMode != "wal" {
return 1
}
}
2023-01-26 15:12:48 +01:00
return multiplier * runtime . GOMAXPROCS ( 0 )
}
2021-08-25 15:34:33 +02:00
// deriveBunDBPGOptions takes an application config and returns either a ready-to-use set of options
2021-03-02 18:26:30 +01:00
// with sensible defaults, or an error if it's not satisfied by the provided config.
2021-12-07 13:31:39 +01:00
func deriveBunDBPGOptions ( ) ( * pgx . ConnConfig , error ) {
2024-11-25 16:15:33 +01:00
// If database URL is defined, ignore
// other DB-related configuration fields.
if url := config . GetDbPostgresConnectionString ( ) ; url != "" {
return pgx . ParseConfig ( url )
2024-08-08 14:00:19 +02:00
}
2024-11-25 16:15:33 +01:00
2021-12-21 12:08:27 +01:00
// these are all optional, the db adapter figures out defaults
2022-05-30 14:41:24 +02:00
address := config . GetDbAddress ( )
2021-03-02 22:52:31 +01:00
// validate database
2022-05-30 14:41:24 +02:00
database := config . GetDbDatabase ( )
2021-12-07 13:31:39 +01:00
if database == "" {
2021-03-04 12:07:24 +01:00
return nil , errors . New ( "no database set" )
2021-03-02 18:26:30 +01:00
}
2021-07-19 18:03:07 +02:00
var tlsConfig * tls . Config
2022-05-30 14:41:24 +02:00
switch config . GetDbTLSMode ( ) {
2023-01-17 13:29:44 +01:00
case "" , "disable" :
2021-07-19 18:03:07 +02:00
break // nothing to do
2023-01-17 13:29:44 +01:00
case "enable" :
2021-07-19 18:03:07 +02:00
tlsConfig = & tls . Config {
2023-10-23 14:07:31 +02:00
InsecureSkipVerify : true , //nolint:gosec
2021-07-19 18:03:07 +02:00
}
2023-01-17 13:29:44 +01:00
case "require" :
2021-07-19 18:03:07 +02:00
tlsConfig = & tls . Config {
InsecureSkipVerify : false ,
2022-05-30 14:41:24 +02:00
ServerName : address ,
2021-11-22 08:46:19 +01:00
MinVersion : tls . VersionTLS12 ,
2021-07-19 18:03:07 +02:00
}
}
2022-05-30 14:41:24 +02:00
if certPath := config . GetDbTLSCACert ( ) ; tlsConfig != nil && certPath != "" {
2021-07-19 18:03:07 +02:00
// load the system cert pool first -- we'll append the given CA cert to this
certPool , err := x509 . SystemCertPool ( )
if err != nil {
return nil , fmt . Errorf ( "error fetching system CA cert pool: %s" , err )
}
// open the file itself and make sure there's something in it
2022-05-30 14:41:24 +02:00
caCertBytes , err := os . ReadFile ( certPath )
2021-07-19 18:03:07 +02:00
if err != nil {
2022-05-30 14:41:24 +02:00
return nil , fmt . Errorf ( "error opening CA certificate at %s: %s" , certPath , err )
2021-07-19 18:03:07 +02:00
}
if len ( caCertBytes ) == 0 {
2022-05-30 14:41:24 +02:00
return nil , fmt . Errorf ( "ca cert at %s was empty" , certPath )
2021-07-19 18:03:07 +02:00
}
// make sure we have a PEM block
caPem , _ := pem . Decode ( caCertBytes )
if caPem == nil {
2022-05-30 14:41:24 +02:00
return nil , fmt . Errorf ( "could not parse cert at %s into PEM" , certPath )
2021-07-19 18:03:07 +02:00
}
// parse the PEM block into the certificate
caCert , err := x509 . ParseCertificate ( caPem . Bytes )
if err != nil {
2024-02-07 15:43:27 +01:00
return nil , fmt . Errorf ( "could not parse cert at %s into x509 certificate: %w" , certPath , err )
2021-07-19 18:03:07 +02:00
}
// we're happy, add it to the existing pool and then use this pool in our tls config
certPool . AddCert ( caCert )
tlsConfig . RootCAs = certPool
}
2021-08-25 15:34:33 +02:00
cfg , _ := pgx . ParseConfig ( "" )
2021-12-21 12:08:27 +01:00
if address != "" {
cfg . Host = address
}
2022-06-03 15:40:38 +02:00
if port := config . GetDbPort ( ) ; port > 0 {
2024-10-16 14:13:58 +02:00
if port > math . MaxUint16 {
return nil , errors . New ( "invalid port, must be in range 1-65535" )
}
cfg . Port = uint16 ( port ) // #nosec G115 -- Just validated above.
2021-12-21 12:08:27 +01:00
}
2022-05-30 14:41:24 +02:00
if u := config . GetDbUser ( ) ; u != "" {
cfg . User = u
2021-12-21 12:08:27 +01:00
}
2022-05-30 14:41:24 +02:00
if p := config . GetDbPassword ( ) ; p != "" {
cfg . Password = p
2021-12-21 12:08:27 +01:00
}
if tlsConfig != nil {
cfg . TLSConfig = tlsConfig
}
2021-12-07 13:31:39 +01:00
cfg . Database = database
2022-05-30 14:41:24 +02:00
cfg . RuntimeParams [ "application_name" ] = config . GetApplicationName ( )
2021-03-02 18:26:30 +01:00
2021-08-25 15:34:33 +02:00
return cfg , nil
2021-03-02 18:26:30 +01:00
}
2023-09-01 15:13:33 +02:00
// buildSQLiteAddress will build an SQLite address string from given config input,
// appending user defined SQLite connection preferences (e.g. cache_size, journal_mode etc).
2024-09-08 16:14:56 +02:00
// The returned bool indicates whether this is an in-memory address or not.
func buildSQLiteAddress ( addr string ) ( string , bool ) {
2023-09-01 15:13:33 +02:00
// Notes on SQLite preferences:
//
// - SQLite by itself supports setting a subset of its configuration options
// via URI query arguments in the connection. Namely `mode` and `cache`.
2024-11-29 16:03:10 +01:00
// This is the same situation for our supported SQLite implementations.
2023-09-01 15:13:33 +02:00
//
2024-11-29 16:03:10 +01:00
// - Both implementations have a "shim" around them in the form of a
// `database/sql/driver.Driver{}` implementation.
2023-09-01 15:13:33 +02:00
//
2024-11-29 16:03:10 +01:00
// - The SQLite shims we interface with add support for setting ANY of the
2023-09-01 15:13:33 +02:00
// configuration options via query arguments, through using a special `_pragma`
// query key that specifies SQLite PRAGMAs to set upon opening each connection.
// As such you will see below that most config is set with the `_pragma` key.
//
// - As for why we're setting these PRAGMAs by connection string instead of
// directly executing the PRAGMAs ourselves? That's to ensure that all of
// configuration options are set across _all_ of our SQLite connections, given
// that we are a multi-threaded (not directly in a C way) application and that
// each connection is a separate SQLite instance opening the same database.
// And the `database/sql` package provides transparent connection pooling.
// Some data is shared between connections, for example the `journal_mode`
// as that is set in a bit of the file header, but to be sure with the other
// settings we just add them all to the connection URI string.
//
// - We specifically set the `busy_timeout` PRAGMA before the `journal_mode`.
// When Write-Ahead-Logging (WAL) is enabled, in order to handle the issues
// that may arise between separate concurrent read/write threads racing for
// the same database file (and write-ahead log), SQLite will sometimes return
// an `SQLITE_BUSY` error code, which indicates that the query was aborted
// due to a data race and must be retried. The `busy_timeout` PRAGMA configures
// a function handler that SQLite can use internally to handle these data races,
// in that it will attempt to retry the query until the `busy_timeout` time is
// reached. And for whatever reason (:shrug:) SQLite is very particular about
// setting this BEFORE the `journal_mode` is set, otherwise you can end up
// running into more of these `SQLITE_BUSY` return codes than you might expect.
// Drop anything fancy from DB address
addr = strings . Split ( addr , "?" ) [ 0 ] // drop any provided query strings
addr = strings . TrimPrefix ( addr , "file:" ) // we'll prepend this later ourselves
// build our own SQLite preferences
// as a series of URL encoded values
prefs := make ( url . Values )
// use immediate transaction lock mode to fail quickly if tx can't lock
// see https://pkg.go.dev/modernc.org/sqlite#Driver.Open
prefs . Add ( "_txlock" , "immediate" )
2024-09-08 16:14:56 +02:00
inMem := false
2023-09-01 15:13:33 +02:00
if addr == ":memory:" {
// Use random name for in-memory instead of ':memory:', so
// multiple in-mem databases can be created without conflict.
2024-09-08 16:14:56 +02:00
inMem = true
2024-05-27 17:46:15 +02:00
addr = "/" + uuid . NewString ( )
prefs . Add ( "vfs" , "memdb" )
2023-09-01 15:13:33 +02:00
}
if dur := config . GetDbSqliteBusyTimeout ( ) ; dur > 0 {
// Set the user provided SQLite busy timeout
// NOTE: MUST BE SET BEFORE THE JOURNAL MODE.
prefs . Add ( "_pragma" , fmt . Sprintf ( "busy_timeout(%d)" , dur . Milliseconds ( ) ) )
}
2023-01-26 15:12:48 +01:00
if mode := config . GetDbSqliteJournalMode ( ) ; mode != "" {
2023-09-01 15:13:33 +02:00
// Set the user provided SQLite journal mode.
prefs . Add ( "_pragma" , fmt . Sprintf ( "journal_mode(%s)" , mode ) )
2023-01-26 15:12:48 +01:00
}
if mode := config . GetDbSqliteSynchronous ( ) ; mode != "" {
2023-09-01 15:13:33 +02:00
// Set the user provided SQLite synchronous mode.
prefs . Add ( "_pragma" , fmt . Sprintf ( "synchronous(%s)" , mode ) )
2023-01-26 15:12:48 +01:00
}
2023-09-01 15:13:33 +02:00
if sz := config . GetDbSqliteCacheSize ( ) ; sz > 0 {
2023-01-26 15:12:48 +01:00
// Set the user provided SQLite cache size (in kibibytes)
// Prepend a '-' character to this to indicate to sqlite
// that we're giving kibibytes rather than num pages.
// https://www.sqlite.org/pragma.html#pragma_cache_size
2023-09-01 15:13:33 +02:00
prefs . Add ( "_pragma" , fmt . Sprintf ( "cache_size(-%d)" , uint64 ( sz / bytesize . KiB ) ) )
2023-01-26 15:12:48 +01:00
}
2023-09-01 15:13:33 +02:00
var b strings . Builder
b . WriteString ( "file:" )
b . WriteString ( addr )
b . WriteString ( "?" )
b . WriteString ( prefs . Encode ( ) )
2024-09-08 16:14:56 +02:00
return b . String ( ) , inMem
2023-01-26 15:12:48 +01:00
}