mirror of
1
Fork 0
forgejo/tests/integration/migration-test/migration_test.go

346 lines
11 KiB
Go
Raw Normal View History

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"compress/gzip"
"context"
"database/sql"
"fmt"
"io"
"os"
"path"
"path/filepath"
"regexp"
"sort"
2019-05-06 01:42:29 +02:00
"strings"
"testing"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/migrations"
migrate_base "code.gitea.io/gitea/models/migrations/base"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-22 00:35:11 +02:00
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-22 00:35:11 +02:00
"code.gitea.io/gitea/modules/testlogger"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert"
2019-10-17 11:26:49 +02:00
"xorm.io/xorm"
)
var currentEngine *xorm.Engine
func initMigrationTest(t *testing.T) func() {
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-22 00:35:11 +02:00
log.RegisterEventWriter("test", testlogger.NewTestLoggerWriter)
deferFn := tests.PrintCurrentTest(t, 2)
giteaRoot := base.SetupGiteaRoot()
if giteaRoot == "" {
tests.Printf("Environment variable $GITEA_ROOT not set\n")
os.Exit(1)
}
setting.AppPath = path.Join(giteaRoot, "gitea")
if _, err := os.Stat(setting.AppPath); err != nil {
tests.Printf("Could not find gitea binary at %s\n", setting.AppPath)
os.Exit(1)
}
giteaConf := os.Getenv("GITEA_CONF")
if giteaConf == "" {
tests.Printf("Environment variable $GITEA_CONF not set\n")
os.Exit(1)
} else if !path.IsAbs(giteaConf) {
setting.CustomConf = path.Join(giteaRoot, giteaConf)
} else {
setting.CustomConf = giteaConf
}
unittest.InitSettings()
assert.True(t, len(setting.RepoRootPath) != 0)
assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath))
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
}
for _, ownerDir := range ownerDirs {
if !ownerDir.Type().IsDir() {
continue
}
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
if err != nil {
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
}
for _, repoDir := range repoDirs {
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755)
}
}
assert.NoError(t, git.InitFull(context.Background()))
setting.LoadDBSetting()
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-22 00:35:11 +02:00
setting.InitLoggersForTest()
return deferFn
}
func availableVersions() ([]string, error) {
migrationsDir, err := os.Open("tests/integration/migration-test")
if err != nil {
return nil, err
}
defer migrationsDir.Close()
[DB] run all Forgejo migrations in integration tests The tests at tests/integration/migration-test/migration_test.go will not run any Forgejo migration when using the gitea-*.sql.gz files because they do not contain a ForgejoVersion row which is interpreted as a new Forgejo installation for which there is no need for migration. Create a situation by which the ForgejoVersion table exists and has a version of 0 in tests/integration/migration-test/forgejo-v1.19.0.*.sql.gz thus ensuring all Forgejo migrations are run. The forgejo*.sql.gz files do not have any Gitea related records, which will be interpreted by the Gitea migrations as a new installation that does not need any migration. As a consequence the migration tests run when using forgejo-v1.19.0.*.sql.gz are exclusively about Forgejo migrations. (cherry picked from commit ec8003859c920ac05a071ad9b1d9d8af5a694ac0) (cherry picked from commit 697570ff390656930d41bcdc4194aacc5c7a54f4) (cherry picked from commit f041aec17265ca34943f961d9640e60c96c0f24d) (cherry picked from commit 60463e3bab9dfa6aae7f379fc77309ab52b1d250) (cherry picked from commit b2fc2a7c1338ad2a9bd0a6887443e604dccda53f) (cherry picked from commit fb2759b6afc213d5e2db8f6e523391927c611b23) (cherry picked from commit 37cfc3b2275afd523f53e149781d69a2914fd83b) (cherry picked from commit 832607500a576a40ee1e35897e58ef319dba9788) (cherry picked from commit 143d4007b1b60ad1880b64ce1d4998e77854a05a) (cherry picked from commit a17e803fbfb7908729c2d92fd58360806cce6fbe) (cherry picked from commit 5c1cc8d9f6c9974e399d3b7da6dc901374f70583) (cherry picked from commit 8bfc117458f046d28c6978de17ff407fe4cc43c4)
2023-08-09 13:00:36 +02:00
versionRE, err := regexp.Compile(".*-v(?P<version>.+)\\." + regexp.QuoteMeta(setting.Database.Type.String()) + "\\.sql.gz")
if err != nil {
return nil, err
}
filenames, err := migrationsDir.Readdirnames(-1)
if err != nil {
return nil, err
}
versions := []string{}
for _, filename := range filenames {
if versionRE.MatchString(filename) {
substrings := versionRE.FindStringSubmatch(filename)
versions = append(versions, substrings[1])
}
}
sort.Strings(versions)
return versions, nil
}
func readSQLFromFile(version string) (string, error) {
filename := fmt.Sprintf("tests/integration/migration-test/gitea-v%s.%s.sql.gz", version, setting.Database.Type)
if _, err := os.Stat(filename); os.IsNotExist(err) {
[DB] run all Forgejo migrations in integration tests The tests at tests/integration/migration-test/migration_test.go will not run any Forgejo migration when using the gitea-*.sql.gz files because they do not contain a ForgejoVersion row which is interpreted as a new Forgejo installation for which there is no need for migration. Create a situation by which the ForgejoVersion table exists and has a version of 0 in tests/integration/migration-test/forgejo-v1.19.0.*.sql.gz thus ensuring all Forgejo migrations are run. The forgejo*.sql.gz files do not have any Gitea related records, which will be interpreted by the Gitea migrations as a new installation that does not need any migration. As a consequence the migration tests run when using forgejo-v1.19.0.*.sql.gz are exclusively about Forgejo migrations. (cherry picked from commit ec8003859c920ac05a071ad9b1d9d8af5a694ac0) (cherry picked from commit 697570ff390656930d41bcdc4194aacc5c7a54f4) (cherry picked from commit f041aec17265ca34943f961d9640e60c96c0f24d) (cherry picked from commit 60463e3bab9dfa6aae7f379fc77309ab52b1d250) (cherry picked from commit b2fc2a7c1338ad2a9bd0a6887443e604dccda53f) (cherry picked from commit fb2759b6afc213d5e2db8f6e523391927c611b23) (cherry picked from commit 37cfc3b2275afd523f53e149781d69a2914fd83b) (cherry picked from commit 832607500a576a40ee1e35897e58ef319dba9788) (cherry picked from commit 143d4007b1b60ad1880b64ce1d4998e77854a05a) (cherry picked from commit a17e803fbfb7908729c2d92fd58360806cce6fbe) (cherry picked from commit 5c1cc8d9f6c9974e399d3b7da6dc901374f70583) (cherry picked from commit 8bfc117458f046d28c6978de17ff407fe4cc43c4)
2023-08-09 13:00:36 +02:00
filename = fmt.Sprintf("tests/integration/migration-test/forgejo-v%s.%s.sql.gz", version, setting.Database.Type)
if _, err := os.Stat(filename); os.IsNotExist(err) {
return "", nil
}
}
file, err := os.Open(filename)
if err != nil {
return "", err
}
defer file.Close()
gr, err := gzip.NewReader(file)
if err != nil {
return "", err
}
defer gr.Close()
bytes, err := io.ReadAll(gr)
if err != nil {
return "", err
}
return string(charset.RemoveBOMIfPresent(bytes)), nil
}
func restoreOldDB(t *testing.T, version string) bool {
data, err := readSQLFromFile(version)
assert.NoError(t, err)
if len(data) == 0 {
tests.Printf("No db found to restore for %s version: %s\n", setting.Database.Type, version)
return false
}
switch {
case setting.Database.Type.IsSQLite3():
util.Remove(setting.Database.Path)
err := os.MkdirAll(path.Dir(setting.Database.Path), os.ModePerm)
assert.NoError(t, err)
db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d&_txlock=immediate", setting.Database.Path, setting.Database.Timeout))
assert.NoError(t, err)
defer db.Close()
_, err = db.Exec(data)
assert.NoError(t, err)
db.Close()
case setting.Database.Type.IsMySQL():
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/",
setting.Database.User, setting.Database.Passwd, setting.Database.Host))
assert.NoError(t, err)
defer db.Close()
_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name))
assert.NoError(t, err)
_, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name))
assert.NoError(t, err)
db.Close()
db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?multiStatements=true",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name))
assert.NoError(t, err)
defer db.Close()
_, err = db.Exec(data)
assert.NoError(t, err)
db.Close()
case setting.Database.Type.IsPostgreSQL():
var db *sql.DB
var err error
if setting.Database.Host[0] == '/' {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/?sslmode=%s&host=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.SSLMode, setting.Database.Host))
assert.NoError(t, err)
} else {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
assert.NoError(t, err)
}
defer db.Close()
_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name))
assert.NoError(t, err)
_, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name))
assert.NoError(t, err)
db.Close()
// Check if we need to setup a specific schema
if len(setting.Database.Schema) != 0 {
if setting.Database.Host[0] == '/' {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/%s?sslmode=%s&host=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Name, setting.Database.SSLMode, setting.Database.Host))
} else {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
}
if !assert.NoError(t, err) {
return false
}
defer db.Close()
schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
if !assert.NoError(t, err) || !assert.NotEmpty(t, schrows) {
return false
}
if !schrows.Next() {
// Create and setup a DB schema
_, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema))
assert.NoError(t, err)
}
schrows.Close()
// Make the user's default search path the created schema; this will affect new connections
_, err = db.Exec(fmt.Sprintf(`ALTER USER "%s" SET search_path = %s`, setting.Database.User, setting.Database.Schema))
assert.NoError(t, err)
db.Close()
}
if setting.Database.Host[0] == '/' {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/%s?sslmode=%s&host=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Name, setting.Database.SSLMode, setting.Database.Host))
} else {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
}
assert.NoError(t, err)
defer db.Close()
_, err = db.Exec(data)
assert.NoError(t, err)
db.Close()
case setting.Database.Type.IsMSSQL():
host, port := setting.ParseMSSQLHostPort(setting.Database.Host)
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
host, port, "master", setting.Database.User, setting.Database.Passwd))
assert.NoError(t, err)
defer db.Close()
_, err = db.Exec("DROP DATABASE IF EXISTS [gitea]")
assert.NoError(t, err)
2019-05-06 01:42:29 +02:00
statements := strings.Split(data, "\nGO\n")
for _, statement := range statements {
if len(statement) > 5 && statement[:5] == "USE [" {
dbname := statement[5 : len(statement)-1]
db.Close()
db, err = sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
host, port, dbname, setting.Database.User, setting.Database.Passwd))
assert.NoError(t, err)
defer db.Close()
}
2019-05-06 01:42:29 +02:00
_, err = db.Exec(statement)
assert.NoError(t, err, "Failure whilst running: %s\nError: %v", statement, err)
}
db.Close()
}
return true
}
func wrappedMigrate(x *xorm.Engine) error {
currentEngine = x
return migrations.Migrate(x)
}
func doMigrationTest(t *testing.T, version string) {
defer tests.PrintCurrentTest(t)()
tests.Printf("Performing migration test for %s version: %s\n", setting.Database.Type, version)
if !restoreOldDB(t, version) {
return
}
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-22 00:35:11 +02:00
setting.InitSQLLoggersForCli(log.INFO)
err := db.InitEngineWithMigration(context.Background(), wrappedMigrate)
assert.NoError(t, err)
currentEngine.Close()
beans, _ := db.NamesToBean()
err = db.InitEngineWithMigration(context.Background(), func(x *xorm.Engine) error {
currentEngine = x
return migrate_base.RecreateTables(beans...)(x)
})
assert.NoError(t, err)
currentEngine.Close()
// We do this a second time to ensure that there is not a problem with retained indices
err = db.InitEngineWithMigration(context.Background(), func(x *xorm.Engine) error {
currentEngine = x
return migrate_base.RecreateTables(beans...)(x)
})
assert.NoError(t, err)
currentEngine.Close()
}
func TestMigrations(t *testing.T) {
defer initMigrationTest(t)()
dialect := setting.Database.Type
versions, err := availableVersions()
assert.NoError(t, err)
if len(versions) == 0 {
tests.Printf("No old database versions available to migration test for %s\n", dialect)
return
}
tests.Printf("Preparing to test %d migrations for %s\n", len(versions), dialect)
for _, version := range versions {
t.Run(fmt.Sprintf("Migrate-%s-%s", dialect, version), func(t *testing.T) {
doMigrationTest(t, version)
})
}
}