diff --git a/go.mod b/go.mod index 8766becda..78bf1c892 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/miekg/dns v1.1.62 github.com/minio/minio-go/v7 v7.0.81 github.com/mitchellh/mapstructure v1.5.0 - github.com/ncruces/go-sqlite3 v0.21.1 + github.com/ncruces/go-sqlite3 v0.21.2 github.com/oklog/ulid v1.3.1 github.com/prometheus/client_golang v1.20.5 github.com/spf13/cobra v1.8.1 diff --git a/go.sum b/go.sum index f3812e8ac..693cfa1e6 100644 --- a/go.sum +++ b/go.sum @@ -434,8 +434,8 @@ github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/ncruces/go-sqlite3 v0.21.1 h1:cbzIOY3jQrXZWVsBfH9TCFj/iqqMIcJ7PLye4AAEwoQ= -github.com/ncruces/go-sqlite3 v0.21.1/go.mod h1:zxMOaSG5kFYVFK4xQa0pdwIszqxqJ0W0BxBgwdrNjuA= +github.com/ncruces/go-sqlite3 v0.21.2 h1:X7Ao4BwtS9h308lFtZA/stkvrzEHvAdp8g4Gko7Ehjs= +github.com/ncruces/go-sqlite3 v0.21.2/go.mod h1:zxMOaSG5kFYVFK4xQa0pdwIszqxqJ0W0BxBgwdrNjuA= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= diff --git a/vendor/github.com/ncruces/go-sqlite3/README.md b/vendor/github.com/ncruces/go-sqlite3/README.md index d3d5e518f..b370e9638 100644 --- a/vendor/github.com/ncruces/go-sqlite3/README.md +++ b/vendor/github.com/ncruces/go-sqlite3/README.md @@ -74,7 +74,7 @@ This project aims for [high test coverage](https://github.com/ncruces/go-sqlite3 It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and [wazero's](https://tetrate.io/blog/introducing-wazero-from-tetrate/#:~:text=Rock%2Dsolid%20test%20approach) thorough testing. -Every commit is [tested](https://github.com/ncruces/go-sqlite3/wiki/Test-matrix) on +Every commit is [tested](https://github.com/ncruces/go-sqlite3/wiki/Support-matrix) on Linux (amd64/arm64/386/riscv64/ppc64le/s390x), macOS (amd64/arm64), Windows (amd64), FreeBSD (amd64), OpenBSD (amd64), NetBSD (amd64), DragonFly BSD (amd64), illumos (amd64), and Solaris (amd64). diff --git a/vendor/github.com/ncruces/go-sqlite3/embed/sqlite3.wasm b/vendor/github.com/ncruces/go-sqlite3/embed/sqlite3.wasm index 05a6065f4..c312aa62a 100644 Binary files a/vendor/github.com/ncruces/go-sqlite3/embed/sqlite3.wasm and b/vendor/github.com/ncruces/go-sqlite3/embed/sqlite3.wasm differ diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/dotlk/dotlk.go b/vendor/github.com/ncruces/go-sqlite3/internal/dotlk/dotlk.go new file mode 100644 index 000000000..3c8d782d7 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/dotlk/dotlk.go @@ -0,0 +1,29 @@ +package dotlk + +import ( + "errors" + "io/fs" + "os" +) + +// LockShm creates a directory on disk to prevent SQLite +// from using this path for a shared memory file. +func LockShm(name string) error { + err := os.Mkdir(name, 0777) + if errors.Is(err, fs.ErrExist) { + s, err := os.Lstat(name) + if err == nil && s.IsDir() { + return nil + } + } + return err +} + +// Unlock removes the lock or shared memory file. +func Unlock(name string) error { + err := os.Remove(name) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/dotlk/dotlk_other.go b/vendor/github.com/ncruces/go-sqlite3/internal/dotlk/dotlk_other.go new file mode 100644 index 000000000..5399a5f8a --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/dotlk/dotlk_other.go @@ -0,0 +1,13 @@ +//go:build !unix + +package dotlk + +import "os" + +// TryLock returns nil if it acquired the lock, +// fs.ErrExist if another process has the lock. +func TryLock(name string) error { + f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + f.Close() + return err +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/dotlk/dotlk_unix.go b/vendor/github.com/ncruces/go-sqlite3/internal/dotlk/dotlk_unix.go new file mode 100644 index 000000000..177ab30bb --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/dotlk/dotlk_unix.go @@ -0,0 +1,50 @@ +//go:build unix + +package dotlk + +import ( + "errors" + "io/fs" + "os" + "strconv" + + "golang.org/x/sys/unix" +) + +// TryLock returns nil if it acquired the lock, +// fs.ErrExist if another process has the lock. +func TryLock(name string) error { + for retry := true; retry; retry = false { + f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if err == nil { + f.WriteString(strconv.Itoa(os.Getpid())) + f.Close() + return nil + } + if !errors.Is(err, fs.ErrExist) { + return err + } + if !removeStale(name) { + break + } + } + return fs.ErrExist +} + +func removeStale(name string) bool { + buf, err := os.ReadFile(name) + if err != nil { + return errors.Is(err, fs.ErrNotExist) + } + + pid, err := strconv.Atoi(string(buf)) + if err != nil { + return false + } + if unix.Kill(pid, 0) == nil { + return false + } + + err = os.Remove(name) + return err == nil || errors.Is(err, fs.ErrNotExist) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/util/sql3util/parse/sql3parse_table.wasm b/vendor/github.com/ncruces/go-sqlite3/util/sql3util/parse/sql3parse_table.wasm index 28eab8d29..4d3357ea1 100644 Binary files a/vendor/github.com/ncruces/go-sqlite3/util/sql3util/parse/sql3parse_table.wasm and b/vendor/github.com/ncruces/go-sqlite3/util/sql3util/parse/sql3parse_table.wasm differ diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/README.md b/vendor/github.com/ncruces/go-sqlite3/vfs/README.md index 354f8bf33..4e987ce3f 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/README.md +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/README.md @@ -48,11 +48,6 @@ On Unix, this package may use `mmap` to implement [shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index), like SQLite. -With [BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2) -a WAL database can only be accessed by a single proccess. -Other processes that attempt to access a database locked with BSD locks, -will fail with the [`SQLITE_PROTOCOL`](https://sqlite.org/rescode.html#protocol) error code. - On Windows, this package may use `MapViewOfFile`, like SQLite. You can also opt into a cross-platform, in-process, memory sharing implementation diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_bsd.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_bsd.go index fa13ef3ae..4f6fadef4 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/os_bsd.go +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_bsd.go @@ -9,11 +9,11 @@ import ( ) func osGetSharedLock(file *os.File) _ErrorCode { - return osLock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK) + return osFlock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK) } func osGetReservedLock(file *os.File) _ErrorCode { - rc := osLock(file, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK) + rc := osFlock(file, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK) if rc == _BUSY { // The documentation states that a lock is upgraded by // releasing the previous lock, then acquiring the new lock. @@ -37,7 +37,7 @@ func osGetExclusiveLock(file *os.File, state *LockLevel) _ErrorCode { } func osDowngradeLock(file *os.File, _ LockLevel) _ErrorCode { - rc := osLock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK) + rc := osFlock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK) if rc == _BUSY { // The documentation states that a lock is downgraded by // releasing the previous lock then acquiring the new lock. @@ -66,7 +66,36 @@ func osCheckReservedLock(file *os.File) (bool, _ErrorCode) { return lock == unix.F_WRLCK, rc } -func osLock(file *os.File, how int, def _ErrorCode) _ErrorCode { +func osFlock(file *os.File, how int, def _ErrorCode) _ErrorCode { err := unix.Flock(int(file.Fd()), how) return osLockErrorCode(err, def) } + +func osReadLock(file *os.File, start, len int64) _ErrorCode { + return osLock(file, unix.F_RDLCK, start, len, _IOERR_RDLOCK) +} + +func osWriteLock(file *os.File, start, len int64) _ErrorCode { + return osLock(file, unix.F_WRLCK, start, len, _IOERR_LOCK) +} + +func osLock(file *os.File, typ int16, start, len int64, def _ErrorCode) _ErrorCode { + err := unix.FcntlFlock(file.Fd(), unix.F_SETLK, &unix.Flock_t{ + Type: typ, + Start: start, + Len: len, + }) + return osLockErrorCode(err, def) +} + +func osUnlock(file *os.File, start, len int64) _ErrorCode { + err := unix.FcntlFlock(file.Fd(), unix.F_SETLK, &unix.Flock_t{ + Type: unix.F_UNLCK, + Start: start, + Len: len, + }) + if err != nil { + return _IOERR_UNLOCK + } + return _OK +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_dotlk.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_dotlk.go index b00a1865b..7a9c38897 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/os_dotlk.go +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_dotlk.go @@ -7,6 +7,8 @@ import ( "io/fs" "os" "sync" + + "github.com/ncruces/go-sqlite3/internal/dotlk" ) var ( @@ -28,12 +30,10 @@ func osGetSharedLock(file *os.File) _ErrorCode { name := file.Name() locker := vfsDotLocks[name] if locker == nil { - f, err := os.OpenFile(name+".lock", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) - f.Close() - if errors.Is(err, fs.ErrExist) { - return _BUSY // Another process has the lock. - } - if err != nil { + if err := dotlk.TryLock(name + ".lock"); err != nil { + if errors.Is(err, fs.ErrExist) { + return _BUSY // Another process has the lock. + } return _IOERR_LOCK } locker = &vfsDotLocker{} @@ -114,8 +114,7 @@ func osReleaseLock(file *os.File, state LockLevel) _ErrorCode { } if locker.shared == 1 { - err := os.Remove(name + ".lock") - if err != nil && !errors.Is(err, fs.ErrNotExist) { + if err := dotlk.Unlock(name + ".lock"); err != nil { return _IOERR_UNLOCK } delete(vfsDotLocks, name) diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/shm_bsd.go b/vendor/github.com/ncruces/go-sqlite3/vfs/shm_bsd.go index 8e7f27084..10d6dbf61 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/shm_bsd.go +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/shm_bsd.go @@ -4,7 +4,9 @@ package vfs import ( "context" + "errors" "io" + "io/fs" "os" "sync" @@ -71,23 +73,21 @@ func (s *vfsShm) shmOpen() _ErrorCode { return _OK } - // Always open file read-write, as it will be shared. - f, err := os.OpenFile(s.path, - os.O_RDWR|os.O_CREATE|_O_NOFOLLOW, 0666) - if err != nil { - return _CANTOPEN - } - // Closes file if it's not nil. + var f *os.File + // Close file on error. + // Keep this here to avoid confusing checklocks. defer func() { f.Close() }() - fi, err := f.Stat() - if err != nil { - return _IOERR_FSTAT - } - vfsShmListMtx.Lock() defer vfsShmListMtx.Unlock() + // Stat file without opening it. + // Closing it would release all POSIX locks on it. + fi, err := os.Stat(s.path) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return _IOERR_FSTAT + } + // Find a shared file, increase the reference count. for _, g := range vfsShmList { if g != nil && os.SameFile(fi, g.info) { @@ -97,13 +97,33 @@ func (s *vfsShm) shmOpen() _ErrorCode { } } - // Lock and truncate the file. - // The lock is only released by closing the file. - if rc := osLock(f, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK); rc != _OK { + // Always open file read-write, as it will be shared. + f, err = os.OpenFile(s.path, + os.O_RDWR|os.O_CREATE|_O_NOFOLLOW, 0666) + if err != nil { + return _CANTOPEN + } + + // Dead man's switch. + if lock, rc := osTestLock(f, _SHM_DMS, 1); rc != _OK { + return _IOERR_LOCK + } else if lock == unix.F_WRLCK { + return _BUSY + } else if lock == unix.F_UNLCK { + if rc := osWriteLock(f, _SHM_DMS, 1); rc != _OK { + return rc + } + if err := f.Truncate(0); err != nil { + return _IOERR_SHMOPEN + } + } + if rc := osReadLock(f, _SHM_DMS, 1); rc != _OK { return rc } - if err := f.Truncate(0); err != nil { - return _IOERR_SHMOPEN + + fi, err = f.Stat() + if err != nil { + return _IOERR_FSTAT } // Add the new shared file. @@ -157,7 +177,30 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode { s.Lock() defer s.Unlock() - return s.shmMemLock(offset, n, flags) + + // Check if we could obtain/release the lock locally. + rc := s.shmMemLock(offset, n, flags) + if rc != _OK { + return rc + } + + // Obtain/release the appropriate file lock. + switch { + case flags&_SHM_UNLOCK != 0: + return osUnlock(s.File, _SHM_BASE+int64(offset), int64(n)) + case flags&_SHM_SHARED != 0: + rc = osReadLock(s.File, _SHM_BASE+int64(offset), int64(n)) + case flags&_SHM_EXCLUSIVE != 0: + rc = osWriteLock(s.File, _SHM_BASE+int64(offset), int64(n)) + default: + panic(util.AssertErr()) + } + + // Release the local lock. + if rc != _OK { + s.shmMemLock(offset, n, flags^(_SHM_UNLOCK|_SHM_LOCK)) + } + return rc } func (s *vfsShm) shmUnmap(delete bool) { diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/shm_dotlk.go b/vendor/github.com/ncruces/go-sqlite3/vfs/shm_dotlk.go index e302db7e6..17fefe562 100644 --- a/vendor/github.com/ncruces/go-sqlite3/vfs/shm_dotlk.go +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/shm_dotlk.go @@ -6,11 +6,11 @@ import ( "context" "errors" "io/fs" - "os" "sync" "github.com/tetratelabs/wazero/api" + "github.com/ncruces/go-sqlite3/internal/dotlk" "github.com/ncruces/go-sqlite3/internal/util" ) @@ -58,8 +58,7 @@ func (s *vfsShm) Close() error { return nil } - err := os.Remove(s.path) - if err != nil && !errors.Is(err, fs.ErrNotExist) { + if err := dotlk.Unlock(s.path); err != nil { return _IOERR_UNLOCK } delete(vfsShmList, s.path) @@ -82,9 +81,8 @@ func (s *vfsShm) shmOpen() _ErrorCode { return _OK } - // Create a directory on disk to ensure only this process - // uses this path to register a shared memory. - err := os.Mkdir(s.path, 0777) + // Dead man's switch. + err := dotlk.LockShm(s.path) if errors.Is(err, fs.ErrExist) { return _BUSY } diff --git a/vendor/modules.txt b/vendor/modules.txt index bffe1090e..fbb9ddfbc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -520,12 +520,13 @@ github.com/modern-go/reflect2 # github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 ## explicit github.com/munnerz/goautoneg -# github.com/ncruces/go-sqlite3 v0.21.1 +# github.com/ncruces/go-sqlite3 v0.21.2 ## explicit; go 1.21 github.com/ncruces/go-sqlite3 github.com/ncruces/go-sqlite3/driver github.com/ncruces/go-sqlite3/embed github.com/ncruces/go-sqlite3/internal/alloc +github.com/ncruces/go-sqlite3/internal/dotlk github.com/ncruces/go-sqlite3/internal/util github.com/ncruces/go-sqlite3/util/osutil github.com/ncruces/go-sqlite3/util/sql3util