diff --git a/Makefile b/Makefile index ca4df2b1749..ad7739c07b9 100644 --- a/Makefile +++ b/Makefile @@ -164,7 +164,7 @@ TEST_PGSQL_PASSWORD ?= postgres TEST_PGSQL_SCHEMA ?= gtestschema TEST_MINIO_ENDPOINT ?= minio:9000 TEST_MSSQL_HOST ?= mssql:1433 -TEST_MSSQL_DBNAME ?= gitea +TEST_MSSQL_DBNAME ?= testgitea TEST_MSSQL_USERNAME ?= sa TEST_MSSQL_PASSWORD ?= MwantsaSecurePassword1 diff --git a/cmd/dump.go b/cmd/dump.go index 49f4d9e8946..40b73f69aa1 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -203,8 +203,8 @@ func runDump(ctx context.Context, cmd *cli.Command) error { } }() - targetDBType := cmd.String("database") - if len(targetDBType) > 0 && targetDBType != setting.Database.Type.String() { + targetDBType := setting.DatabaseType(cmd.String("database")) + if targetDBType != "" && targetDBType != setting.Database.Type { log.Info("Dumping database %s => %s...", setting.Database.Type, targetDBType) } else { log.Info("Dumping database...") diff --git a/models/db/conn.go b/models/db/conn.go new file mode 100644 index 00000000000..de6f3cd5ec7 --- /dev/null +++ b/models/db/conn.go @@ -0,0 +1,173 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +type ConnOptions struct { + Type setting.DatabaseType + Host string + Database string + User string + Passwd string + Schema string + SSLMode string + + SQLitePath string + SQLiteBusyTimeout int + SQLiteJournalMode string +} + +type SQLiteConnStrOptions struct { + FilePath string + BusyTimeout int + JournalMode string +} + +func GlobalConnOptions() ConnOptions { + return ConnOptions{ + Type: setting.Database.Type, + Host: setting.Database.Host, + Database: setting.Database.Name, + User: setting.Database.User, + Passwd: setting.Database.Passwd, + Schema: setting.Database.Schema, + SSLMode: setting.Database.SSLMode, + + SQLitePath: setting.Database.Path, + SQLiteBusyTimeout: setting.Database.SQLiteBusyTimeout, + SQLiteJournalMode: setting.Database.SQLiteJournalMode, + } +} + +const sqlDriverPostgresSchema = "postgresschema" + +var makeSQLiteConnStr = func(opts SQLiteConnStrOptions) (string, string, error) { + return "", "", errors.New(`this Gitea binary was not built with SQLite3 support, get an official release or rebuild with: -tags sqlite,sqlite_unlock_notify`) +} + +func ConnStrDefaultDatabase(opts ConnOptions) (string, string, error) { + opts.Database, opts.Schema = "", "" + return ConnStr(opts) +} + +func ConnStr(opts ConnOptions) (string, string, error) { + switch { + case opts.Type.IsMySQL(): + // use unix socket or tcp socket + connType := util.Iif(strings.HasPrefix(opts.Host, "/"), "unix", "tcp") + // allow (Postgres-inspired) default value to work in MySQL + tls := util.Iif(opts.SSLMode == "disable", "false", opts.SSLMode) + // in case the database name is a partial connection string which contains "?" parameters + paramSep := util.Iif(strings.Contains(opts.Database, "?"), "&", "?") + connStr := fmt.Sprintf("%s:%s@%s(%s)/%s%sparseTime=true&tls=%s", opts.User, opts.Passwd, connType, opts.Host, opts.Database, paramSep, tls) + return "mysql", connStr, nil + + case opts.Type.IsPostgreSQL(): + connStr := makePgSQLConnStr(opts.Host, opts.User, opts.Passwd, opts.Database, opts.SSLMode) + driver := util.Iif(opts.Schema == "", "postgres", sqlDriverPostgresSchema) + registerPostgresSchemaDriver() + return driver, connStr, nil + + case opts.Type.IsMSSQL(): + host, port := parseMSSQLHostPort(opts.Host) + connStr := fmt.Sprintf("server=%s; port=%s; user id=%s; password=%s;", host, port, opts.User, opts.Passwd) + if opts.Database != "" { + connStr += "; database=" + opts.Database + } + return "mssql", connStr, nil + + case opts.Type.IsSQLite3(): + if opts.SQLitePath == "" { + return "", "", errors.New("sqlite3 database path cannot be empty") + } + if err := os.MkdirAll(filepath.Dir(opts.SQLitePath), os.ModePerm); err != nil { + return "", "", fmt.Errorf("failed to create directories: %w", err) + } + return makeSQLiteConnStr(SQLiteConnStrOptions{ + FilePath: opts.SQLitePath, + JournalMode: opts.SQLiteJournalMode, + BusyTimeout: opts.SQLiteBusyTimeout, + }) + } + return "", "", fmt.Errorf("unknown database type: %s", opts.Type) +} + +// parsePgSQLHostPort parses given input in various forms defined in +// https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING +// and returns proper host and port number. +func parsePgSQLHostPort(info string) (host, port string) { + if h, p, err := net.SplitHostPort(info); err == nil { + host, port = h, p + } else { + // treat the "info" as "host", if it's an IPv6 address, remove the wrapper + host = info + if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { + host = host[1 : len(host)-1] + } + } + + // set fallback values + if host == "" { + host = "127.0.0.1" + } + if port == "" { + port = "5432" + } + return host, port +} + +func makePgSQLConnStr(dbHost, dbUser, dbPasswd, dbName, dbsslMode string) (connStr string) { + dbName, dbParam, _ := strings.Cut(dbName, "?") + host, port := parsePgSQLHostPort(dbHost) + connURL := url.URL{ + Scheme: "postgres", + User: url.UserPassword(dbUser, dbPasswd), + Host: net.JoinHostPort(host, port), + Path: dbName, + OmitHost: false, + RawQuery: dbParam, + } + query := connURL.Query() + if strings.HasPrefix(host, "/") { // looks like a unix socket + query.Add("host", host) + connURL.Host = ":" + port + } + query.Set("sslmode", dbsslMode) + connURL.RawQuery = query.Encode() + return connURL.String() +} + +// parseMSSQLHostPort splits the host into host and port +func parseMSSQLHostPort(info string) (string, string) { + // the default port "0" might be related to MSSQL's dynamic port, maybe it should be double-confirmed in the future + host, port := "127.0.0.1", "0" + if strings.Contains(info, ":") { + host = strings.Split(info, ":")[0] + port = strings.Split(info, ":")[1] + } else if strings.Contains(info, ",") { + host = strings.Split(info, ",")[0] + port = strings.TrimSpace(strings.Split(info, ",")[1]) + } else if len(info) > 0 { + host = info + } + if host == "" { + host = "127.0.0.1" + } + if port == "" { + port = "0" + } + return host, port +} diff --git a/modules/setting/database_test.go b/models/db/conn_test.go similarity index 88% rename from modules/setting/database_test.go rename to models/db/conn_test.go index a742d54f8cb..ba33d252f2b 100644 --- a/modules/setting/database_test.go +++ b/models/db/conn_test.go @@ -1,7 +1,7 @@ // Copyright 2019 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package setting +package db import ( "testing" @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_parsePostgreSQLHostPort(t *testing.T) { +func TestParsePgSQLHostPort(t *testing.T) { tests := map[string]struct { HostPort string Host string @@ -49,14 +49,14 @@ func Test_parsePostgreSQLHostPort(t *testing.T) { for k, test := range tests { t.Run(k, func(t *testing.T) { t.Log(test.HostPort) - host, port := parsePostgreSQLHostPort(test.HostPort) + host, port := parsePgSQLHostPort(test.HostPort) assert.Equal(t, test.Host, host) assert.Equal(t, test.Port, port) }) } } -func Test_getPostgreSQLConnectionString(t *testing.T) { +func TestMakePgSQLConnStr(t *testing.T) { tests := []struct { Host string User string @@ -103,7 +103,7 @@ func Test_getPostgreSQLConnectionString(t *testing.T) { } for _, test := range tests { - connStr := getPostgreSQLConnectionString(test.Host, test.User, test.Passwd, test.Name, test.SSLMode) + connStr := makePgSQLConnStr(test.Host, test.User, test.Passwd, test.Name, test.SSLMode) assert.Equal(t, test.Output, connStr) } } diff --git a/models/db/sql_postgres_with_schema.go b/models/db/driver_postgresschema.go similarity index 92% rename from models/db/sql_postgres_with_schema.go rename to models/db/driver_postgresschema.go index 812fe4a6a61..b6735007631 100644 --- a/models/db/sql_postgres_with_schema.go +++ b/models/db/driver_postgresschema.go @@ -18,8 +18,8 @@ var registerOnce sync.Once func registerPostgresSchemaDriver() { registerOnce.Do(func() { - sql.Register("postgresschema", &postgresSchemaDriver{}) - dialects.RegisterDriver("postgresschema", dialects.QueryDriver("postgres")) + sql.Register(sqlDriverPostgresSchema, &postgresSchemaDriver{}) + dialects.RegisterDriver(sqlDriverPostgresSchema, dialects.QueryDriver("postgres")) }) } diff --git a/models/db/driver_sqlite_mattn.go b/models/db/driver_sqlite_mattn.go new file mode 100644 index 00000000000..4988a43d3f2 --- /dev/null +++ b/models/db/driver_sqlite_mattn.go @@ -0,0 +1,34 @@ +//go:build sqlite + +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db + +import ( + "fmt" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/setting" + + _ "github.com/mattn/go-sqlite3" +) + +func init() { + setting.SupportedDatabaseTypes = append(setting.SupportedDatabaseTypes, "sqlite3") + makeSQLiteConnStr = makeSQLiteConnStrMattnCGO +} + +func makeSQLiteConnStrMattnCGO(opts SQLiteConnStrOptions) (string, string, error) { + var params []string + params = append(params, "cache=shared") + params = append(params, "mode=rwc") + params = append(params, "_busy_timeout="+strconv.Itoa(opts.BusyTimeout)) + params = append(params, "_txlock=immediate") + if opts.JournalMode != "" { + params = append(params, "_journal_mode="+opts.JournalMode) + } + connStr := fmt.Sprintf("file:%s?%s", opts.FilePath, strings.Join(params, "&")) + return "sqlite3", connStr, nil +} diff --git a/models/db/engine_dump.go b/models/db/engine_dump.go index 63f2d4e093c..1d8d555b448 100644 --- a/models/db/engine_dump.go +++ b/models/db/engine_dump.go @@ -3,10 +3,14 @@ package db -import "xorm.io/xorm/schemas" +import ( + "code.gitea.io/gitea/modules/setting" + + "xorm.io/xorm/schemas" +) // DumpDatabase dumps all data from database according the special database SQL syntax to file system. -func DumpDatabase(filePath, dbType string) error { +func DumpDatabase(filePath string, dbType setting.DatabaseType) error { var tbs []*schemas.Table for _, t := range registeredModels { t, err := xormEngine.TableInfo(t) diff --git a/models/db/engine_init.go b/models/db/engine_init.go index ef5db3ff5ea..65192d3327a 100644 --- a/models/db/engine_init.go +++ b/models/db/engine_init.go @@ -6,7 +6,6 @@ package db import ( "context" "fmt" - "strings" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -24,31 +23,23 @@ func init() { // newXORMEngine returns a new XORM engine from the configuration func newXORMEngine() (*xorm.Engine, error) { - connStr, err := setting.DBConnStr() + connOpts := GlobalConnOptions() + driver, connStr, err := ConnStr(connOpts) if err != nil { return nil, err } - var engine *xorm.Engine - - if setting.Database.Type.IsPostgreSQL() && len(setting.Database.Schema) > 0 { - // OK whilst we sort out our schema issues - create a schema aware postgres - registerPostgresSchemaDriver() - engine, err = xorm.NewEngine("postgresschema", connStr) - } else { - engine, err = xorm.NewEngine(setting.Database.Type.String(), connStr) - } - + engine, err := xorm.NewEngine(driver, connStr) if err != nil { return nil, err } - switch setting.Database.Type { - case "mysql": + switch { + case connOpts.Type.IsMySQL(): engine.Dialect().SetParams(map[string]string{"rowFormat": "DYNAMIC"}) - case "mssql": + case connOpts.Type.IsMSSQL(): engine.Dialect().SetParams(map[string]string{"DEFAULT_VARCHAR": "nvarchar"}) } - engine.SetSchema(setting.Database.Schema) + engine.SetSchema(connOpts.Schema) return engine, nil } @@ -56,10 +47,7 @@ func newXORMEngine() (*xorm.Engine, error) { func InitEngine(ctx context.Context) error { xe, err := newXORMEngine() if err != nil { - if strings.Contains(err.Error(), "SQLite3 support") { - return fmt.Errorf("sqlite3 requires: -tags sqlite,sqlite_unlock_notify\n%w", err) - } - return fmt.Errorf("failed to connect to database: %w", err) + return fmt.Errorf("failed to init database engine: %w", err) } xe.SetMapper(names.GonicMapper{}) diff --git a/models/db/engine_test.go b/models/db/engine_test.go index 1c218df77f3..6a6264b5353 100644 --- a/models/db/engine_test.go +++ b/models/db/engine_test.go @@ -30,7 +30,7 @@ func TestDumpDatabase(t *testing.T) { assert.NoError(t, db.GetEngine(t.Context()).Sync(new(Version))) for _, dbType := range setting.SupportedDatabaseTypes { - assert.NoError(t, db.DumpDatabase(filepath.Join(dir, dbType+".sql"), dbType)) + assert.NoError(t, db.DumpDatabase(filepath.Join(dir, dbType+".sql"), setting.DatabaseType(dbType))) } } diff --git a/models/migrations/base/db_test.go b/models/migrations/base/db_test.go index dc4e502e421..ce6e1169d83 100644 --- a/models/migrations/base/db_test.go +++ b/models/migrations/base/db_test.go @@ -6,17 +6,18 @@ package base import ( "testing" + "code.gitea.io/gitea/models/migrations/migrationtest" "code.gitea.io/gitea/modules/timeutil" "xorm.io/xorm/names" ) func TestMain(m *testing.M) { - MainTest(m) + migrationtest.MainTest(m) } func Test_DropTableColumns(t *testing.T) { - x, deferable := PrepareTestEnv(t, 0) + x, deferable := migrationtest.PrepareTestEnv(t, 0) defer deferable() // FIXME: this logic seems wrong. Need to add an assertion here in the future, but it seems causing failure. if x == nil || t.Failed() { diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go deleted file mode 100644 index 0ec979a5131..00000000000 --- a/models/migrations/base/tests.go +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package base - -import ( - "database/sql" - "fmt" - "os" - "path" - "path/filepath" - "testing" - - "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/testlogger" - "code.gitea.io/gitea/modules/util" - - "github.com/stretchr/testify/require" - "xorm.io/xorm" - "xorm.io/xorm/schemas" -) - -// FIXME: this file shouldn't be in a normal package, it should only be compiled for tests - -func newXORMEngine(t *testing.T) (*xorm.Engine, error) { - if err := db.InitEngine(t.Context()); err != nil { - return nil, err - } - x := unittest.GetXORMEngine() - return x, nil -} - -func deleteDB() error { - switch { - case setting.Database.Type.IsSQLite3(): - if err := util.Remove(setting.Database.Path); err != nil { - return err - } - return os.MkdirAll(path.Dir(setting.Database.Path), os.ModePerm) - - case setting.Database.Type.IsMySQL(): - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/", - setting.Database.User, setting.Database.Passwd, setting.Database.Host)) - if err != nil { - return err - } - defer db.Close() - - if _, err = db.Exec("DROP DATABASE IF EXISTS " + setting.Database.Name); err != nil { - return err - } - - if _, err = db.Exec("CREATE DATABASE IF NOT EXISTS " + setting.Database.Name); err != nil { - return err - } - return nil - case setting.Database.Type.IsPostgreSQL(): - db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s", - setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode)) - if err != nil { - return err - } - defer db.Close() - - if _, err = db.Exec("DROP DATABASE IF EXISTS " + setting.Database.Name); err != nil { - return err - } - - if _, err = db.Exec("CREATE DATABASE " + setting.Database.Name); err != nil { - return err - } - db.Close() - - // Check if we need to set up a specific schema - if len(setting.Database.Schema) != 0 { - db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", - setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode)) - if err != nil { - return err - } - defer db.Close() - - schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema)) - if err != nil { - return err - } - defer schrows.Close() - - if !schrows.Next() { - // Create and set up a DB schema - _, err = db.Exec("CREATE SCHEMA " + setting.Database.Schema) - if err != nil { - return err - } - } - - // Make the user's default search path the created schema; this will affect new connections - _, err = db.Exec(fmt.Sprintf(`ALTER USER "%s" SET search_path = %s`, setting.Database.User, setting.Database.Schema)) - if err != nil { - return err - } - return nil - } - case setting.Database.Type.IsMSSQL(): - host, port := setting.ParseMSSQLHostPort(setting.Database.Host) - db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", - host, port, "master", setting.Database.User, setting.Database.Passwd)) - if err != nil { - return err - } - defer db.Close() - - if _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS [%s]", setting.Database.Name)); err != nil { - return err - } - if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE [%s]", setting.Database.Name)); err != nil { - return err - } - default: - return fmt.Errorf("unsupported database type: %s", setting.Database.Type) - } - - return nil -} - -// PrepareTestEnv prepares the test environment and reset the database. The skip parameter should usually be 0. -// Provide models to be sync'd with the database - in particular any models you expect fixtures to be loaded from. -// -// fixtures in `models/migrations/fixtures/` will be loaded automatically -func PrepareTestEnv(t *testing.T, skip int, syncModels ...any) (*xorm.Engine, func()) { - t.Helper() - ourSkip := 2 - ourSkip += skip - deferFn := testlogger.PrintCurrentTest(t, ourSkip) - giteaRoot := setting.GetGiteaTestSourceRoot() - require.NoError(t, unittest.SyncDirs(filepath.Join(giteaRoot, "tests/gitea-repositories-meta"), setting.RepoRootPath)) - - if err := deleteDB(); err != nil { - t.Fatalf("unable to reset database: %v", err) - return nil, deferFn - } - - x, err := newXORMEngine(t) - require.NoError(t, err) - if x != nil { - oldDefer := deferFn - deferFn = func() { - oldDefer() - if err := x.Close(); err != nil { - t.Errorf("error during close: %v", err) - } - if err := deleteDB(); err != nil { - t.Errorf("unable to reset database: %v", err) - } - } - } - if err != nil { - return x, deferFn - } - - if len(syncModels) > 0 { - if err := x.Sync(syncModels...); err != nil { - t.Errorf("error during sync: %v", err) - return x, deferFn - } - } - - fixturesDir := filepath.Join(giteaRoot, "models", "migrations", "fixtures", t.Name()) - - if _, err := os.Stat(fixturesDir); err == nil { - t.Logf("initializing fixtures from: %s", fixturesDir) - if err := unittest.InitFixtures( - unittest.FixturesOptions{ - Dir: fixturesDir, - }, x); err != nil { - t.Errorf("error whilst initializing fixtures from %s: %v", fixturesDir, err) - return x, deferFn - } - if err := unittest.LoadFixtures(); err != nil { - t.Errorf("error whilst loading fixtures from %s: %v", fixturesDir, err) - return x, deferFn - } - } else if !os.IsNotExist(err) { - t.Errorf("unexpected error whilst checking for existence of fixtures: %v", err) - } else { - t.Logf("no fixtures found in: %s", fixturesDir) - } - - return x, deferFn -} - -func LoadTableSchemasMap(t *testing.T, x *xorm.Engine) map[string]*schemas.Table { - tables, err := x.DBMetas() - require.NoError(t, err) - tableMap := make(map[string]*schemas.Table) - for _, table := range tables { - tableMap[table.Name] = table - } - return tableMap -} - -func mainTest(m *testing.M) int { - testlogger.Init() - err := setting.PrepareIntegrationTestConfig() - if err != nil { - return testlogger.MainErrorf("Unable to prepare integration test config: %v", err) - } - setting.SetupGiteaTestEnv() - - if err = git.InitFull(); err != nil { - return testlogger.MainErrorf("Unable to InitFull: %v", err) - } - setting.LoadDBSetting() - setting.InitLoggersForTest() - return m.Run() -} - -func MainTest(m *testing.M) { - os.Exit(mainTest(m)) -} diff --git a/models/migrations/migrationtest/tests.go b/models/migrations/migrationtest/tests.go new file mode 100644 index 00000000000..ed8bb16ef1a --- /dev/null +++ b/models/migrations/migrationtest/tests.go @@ -0,0 +1,120 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrationtest + +import ( + "os" + "path/filepath" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/testlogger" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "xorm.io/xorm" + "xorm.io/xorm/schemas" +) + +// PrepareTestEnv prepares the test environment and reset the database. The skip parameter should usually be 0. +// Provide models to be sync'd with the database - in particular any models you expect fixtures to be loaded from. +// +// fixtures in `models/migrations/fixtures/` will be loaded automatically +func PrepareTestEnv(t *testing.T, skip int, syncModels ...any) (*xorm.Engine, func()) { + t.Helper() + ourSkip := 2 + ourSkip += skip + deferFn := testlogger.PrintCurrentTest(t, ourSkip) + giteaRoot := setting.GetGiteaTestSourceRoot() + require.NoError(t, unittest.SyncDirs(filepath.Join(giteaRoot, "tests/gitea-repositories-meta"), setting.RepoRootPath)) + + cleanup, err := unittest.ResetTestDatabase() + if err != nil { + t.Fatalf("unable to reset database: %v", err) + return nil, deferFn + } + { + oldDefer := deferFn + deferFn = func() { + cleanup() + oldDefer() + } + } + + err = db.InitEngine(t.Context()) + if !assert.NoError(t, err) { + return nil, deferFn + } + x := unittest.GetXORMEngine() + { + oldDefer := deferFn + deferFn = func() { + _ = x.Close() + oldDefer() + } + } + + if len(syncModels) > 0 { + if err := x.Sync(syncModels...); err != nil { + t.Errorf("error during sync: %v", err) + return x, deferFn + } + } + + fixturesDir := filepath.Join(giteaRoot, "models", "migrations", "fixtures", t.Name()) + + if _, err := os.Stat(fixturesDir); err == nil { + t.Logf("initializing fixtures from: %s", fixturesDir) + if err := unittest.InitFixtures( + unittest.FixturesOptions{ + Dir: fixturesDir, + }, x); err != nil { + t.Errorf("error whilst initializing fixtures from %s: %v", fixturesDir, err) + return x, deferFn + } + if err := unittest.LoadFixtures(); err != nil { + t.Errorf("error whilst loading fixtures from %s: %v", fixturesDir, err) + return x, deferFn + } + } else if !os.IsNotExist(err) { + t.Errorf("unexpected error whilst checking for existence of fixtures: %v", err) + } else { + t.Logf("no fixtures found in: %s", fixturesDir) + } + + return x, deferFn +} + +func LoadTableSchemasMap(t *testing.T, x *xorm.Engine) map[string]*schemas.Table { + tables, err := x.DBMetas() + require.NoError(t, err) + tableMap := make(map[string]*schemas.Table) + for _, table := range tables { + tableMap[table.Name] = table + } + return tableMap +} + +func mainTest(m *testing.M) int { + testlogger.Init() + err := setting.PrepareIntegrationTestConfig() + if err != nil { + return testlogger.MainErrorf("Unable to prepare integration test config: %v", err) + } + setting.SetupGiteaTestEnv() + + if err = git.InitFull(); err != nil { + return testlogger.MainErrorf("Unable to InitFull: %v", err) + } + setting.LoadDBSetting() + setting.InitLoggersForTest() + return m.Run() +} + +func MainTest(m *testing.M) { + os.Exit(mainTest(m)) +} diff --git a/models/migrations/v1_14/main_test.go b/models/migrations/v1_14/main_test.go index 978f88577c3..6ed240c4075 100644 --- a/models/migrations/v1_14/main_test.go +++ b/models/migrations/v1_14/main_test.go @@ -6,9 +6,9 @@ package v1_14 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_14/v176_test.go b/models/migrations/v1_14/v176_test.go index 5c1db4db71c..aa57b5ad1d9 100644 --- a/models/migrations/v1_14/v176_test.go +++ b/models/migrations/v1_14/v176_test.go @@ -6,7 +6,7 @@ package v1_14 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" ) @@ -47,7 +47,7 @@ func Test_RemoveInvalidLabels(t *testing.T) { } // load and prepare the test database - x, deferable := base.PrepareTestEnv(t, 0, new(Comment), new(Issue), new(Repository), new(IssueLabel), new(Label)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(Comment), new(Issue), new(Repository), new(IssueLabel), new(Label)) if x == nil || t.Failed() { defer deferable() return diff --git a/models/migrations/v1_14/v177_test.go b/models/migrations/v1_14/v177_test.go index 263f69f3380..a86fb98830c 100644 --- a/models/migrations/v1_14/v177_test.go +++ b/models/migrations/v1_14/v177_test.go @@ -6,7 +6,7 @@ package v1_14 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "code.gitea.io/gitea/modules/timeutil" "github.com/stretchr/testify/assert" @@ -34,7 +34,7 @@ func Test_DeleteOrphanedIssueLabels(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(IssueLabel), new(Label)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(IssueLabel), new(Label)) if x == nil || t.Failed() { defer deferable() return diff --git a/models/migrations/v1_15/main_test.go b/models/migrations/v1_15/main_test.go index d01585e9978..768bbd310b0 100644 --- a/models/migrations/v1_15/main_test.go +++ b/models/migrations/v1_15/main_test.go @@ -6,9 +6,9 @@ package v1_15 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_15/v181_test.go b/models/migrations/v1_15/v181_test.go index 73b5c1f3d6c..e230c684eab 100644 --- a/models/migrations/v1_15/v181_test.go +++ b/models/migrations/v1_15/v181_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" ) @@ -20,7 +20,7 @@ func Test_AddPrimaryEmail2EmailAddress(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(User)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(User)) if x == nil || t.Failed() { defer deferable() return diff --git a/models/migrations/v1_15/v182_test.go b/models/migrations/v1_15/v182_test.go index 5fc6a0c467e..c0a1378534c 100644 --- a/models/migrations/v1_15/v182_test.go +++ b/models/migrations/v1_15/v182_test.go @@ -6,7 +6,7 @@ package v1_15 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" ) @@ -20,7 +20,7 @@ func Test_AddIssueResourceIndexTable(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(Issue)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(Issue)) if x == nil || t.Failed() { defer deferable() return diff --git a/models/migrations/v1_16/main_test.go b/models/migrations/v1_16/main_test.go index 7f93d6e9e5e..c54424788d6 100644 --- a/models/migrations/v1_16/main_test.go +++ b/models/migrations/v1_16/main_test.go @@ -6,9 +6,9 @@ package v1_16 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_16/v189_test.go b/models/migrations/v1_16/v189_test.go index fb56ac8e116..44424dd3691 100644 --- a/models/migrations/v1_16/v189_test.go +++ b/models/migrations/v1_16/v189_test.go @@ -6,7 +6,7 @@ package v1_16 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "code.gitea.io/gitea/modules/json" "github.com/stretchr/testify/assert" @@ -27,7 +27,7 @@ func (ls *LoginSourceOriginalV189) TableName() string { func Test_UnwrapLDAPSourceCfg(t *testing.T) { // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(LoginSourceOriginalV189)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(LoginSourceOriginalV189)) if x == nil || t.Failed() { defer deferable() return diff --git a/models/migrations/v1_16/v193_test.go b/models/migrations/v1_16/v193_test.go index 2e827f0550b..f68dd6d92d8 100644 --- a/models/migrations/v1_16/v193_test.go +++ b/models/migrations/v1_16/v193_test.go @@ -6,7 +6,7 @@ package v1_16 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" ) @@ -31,7 +31,7 @@ func Test_AddRepoIDForAttachment(t *testing.T) { } // Prepare and load the testing database - x, deferrable := base.PrepareTestEnv(t, 0, new(Attachment), new(Issue), new(Release)) + x, deferrable := migrationtest.PrepareTestEnv(t, 0, new(Attachment), new(Issue), new(Release)) defer deferrable() if x == nil || t.Failed() { return diff --git a/models/migrations/v1_16/v195_test.go b/models/migrations/v1_16/v195_test.go index 946e06e3997..bbfa5e162a8 100644 --- a/models/migrations/v1_16/v195_test.go +++ b/models/migrations/v1_16/v195_test.go @@ -6,7 +6,7 @@ package v1_16 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" ) @@ -21,7 +21,7 @@ func Test_AddTableCommitStatusIndex(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(CommitStatus)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(CommitStatus)) if x == nil || t.Failed() { defer deferable() return diff --git a/models/migrations/v1_16/v210_test.go b/models/migrations/v1_16/v210_test.go index 3b4ac7aa4b1..7bff2572e11 100644 --- a/models/migrations/v1_16/v210_test.go +++ b/models/migrations/v1_16/v210_test.go @@ -6,7 +6,7 @@ package v1_16 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "code.gitea.io/gitea/modules/timeutil" "github.com/stretchr/testify/assert" @@ -44,7 +44,7 @@ func Test_RemigrateU2FCredentials(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(WebauthnCredential), new(U2fRegistration), new(ExpectedWebauthnCredential)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(WebauthnCredential), new(U2fRegistration), new(ExpectedWebauthnCredential)) if x == nil || t.Failed() { defer deferable() return diff --git a/models/migrations/v1_17/main_test.go b/models/migrations/v1_17/main_test.go index 571a4f55a34..86522018719 100644 --- a/models/migrations/v1_17/main_test.go +++ b/models/migrations/v1_17/main_test.go @@ -6,9 +6,9 @@ package v1_17 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_17/v221_test.go b/models/migrations/v1_17/v221_test.go index a2dc0fae554..6fda9b9980e 100644 --- a/models/migrations/v1_17/v221_test.go +++ b/models/migrations/v1_17/v221_test.go @@ -7,7 +7,7 @@ import ( "encoding/base32" "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" ) @@ -38,7 +38,7 @@ func Test_StoreWebauthnCredentialIDAsBytes(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(WebauthnCredential), new(ExpectedWebauthnCredential)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(WebauthnCredential), new(ExpectedWebauthnCredential)) defer deferable() if x == nil || t.Failed() { return diff --git a/models/migrations/v1_18/main_test.go b/models/migrations/v1_18/main_test.go index ebcfb45a941..b8641526f3d 100644 --- a/models/migrations/v1_18/main_test.go +++ b/models/migrations/v1_18/main_test.go @@ -6,9 +6,9 @@ package v1_18 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_18/v229_test.go b/models/migrations/v1_18/v229_test.go index 5722dd35574..638983ad0ba 100644 --- a/models/migrations/v1_18/v229_test.go +++ b/models/migrations/v1_18/v229_test.go @@ -7,7 +7,7 @@ import ( "testing" "code.gitea.io/gitea/models/issues" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" ) @@ -16,7 +16,7 @@ func Test_UpdateOpenMilestoneCounts(t *testing.T) { type ExpectedMilestone issues.Milestone // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(issues.Milestone), new(ExpectedMilestone), new(issues.Issue)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(issues.Milestone), new(ExpectedMilestone), new(issues.Issue)) defer deferable() if x == nil || t.Failed() { return diff --git a/models/migrations/v1_18/v230_test.go b/models/migrations/v1_18/v230_test.go index 25b2f6525da..e5e28ea63fc 100644 --- a/models/migrations/v1_18/v230_test.go +++ b/models/migrations/v1_18/v230_test.go @@ -6,7 +6,7 @@ package v1_18 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" ) @@ -18,7 +18,7 @@ func Test_AddConfidentialClientColumnToOAuth2ApplicationTable(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(oauth2Application)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(oauth2Application)) defer deferable() if x == nil || t.Failed() { return diff --git a/models/migrations/v1_19/main_test.go b/models/migrations/v1_19/main_test.go index 87e807be6e1..784ca0e46ed 100644 --- a/models/migrations/v1_19/main_test.go +++ b/models/migrations/v1_19/main_test.go @@ -6,9 +6,9 @@ package v1_19 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_19/v233_test.go b/models/migrations/v1_19/v233_test.go index 7436ff7483c..3f7900c58f0 100644 --- a/models/migrations/v1_19/v233_test.go +++ b/models/migrations/v1_19/v233_test.go @@ -6,7 +6,7 @@ package v1_19 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/secret" "code.gitea.io/gitea/modules/setting" @@ -39,7 +39,7 @@ func Test_AddHeaderAuthorizationEncryptedColWebhook(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(Webhook), new(ExpectedWebhook), new(HookTask)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(Webhook), new(ExpectedWebhook), new(HookTask)) defer deferable() if x == nil || t.Failed() { return diff --git a/models/migrations/v1_20/main_test.go b/models/migrations/v1_20/main_test.go index 2fd63a7118e..3ceb9a3c66d 100644 --- a/models/migrations/v1_20/main_test.go +++ b/models/migrations/v1_20/main_test.go @@ -6,9 +6,9 @@ package v1_20 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_20/v259_test.go b/models/migrations/v1_20/v259_test.go index 0bf63719e5e..3864eecb78a 100644 --- a/models/migrations/v1_20/v259_test.go +++ b/models/migrations/v1_20/v259_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" ) @@ -66,7 +66,7 @@ func Test_ConvertScopedAccessTokens(t *testing.T) { }) } - x, deferable := base.PrepareTestEnv(t, 0, new(AccessToken)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(AccessToken)) defer deferable() if x == nil || t.Failed() { t.Skip() diff --git a/models/migrations/v1_21/main_test.go b/models/migrations/v1_21/main_test.go index 536a7ade088..daf98d40f4c 100644 --- a/models/migrations/v1_21/main_test.go +++ b/models/migrations/v1_21/main_test.go @@ -6,9 +6,9 @@ package v1_21 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_22/main_test.go b/models/migrations/v1_22/main_test.go index ac8facd6aa0..e02c8a53282 100644 --- a/models/migrations/v1_22/main_test.go +++ b/models/migrations/v1_22/main_test.go @@ -6,9 +6,9 @@ package v1_22 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_22/v283_test.go b/models/migrations/v1_22/v283_test.go index 743f860466f..8e4c9410bd7 100644 --- a/models/migrations/v1_22/v283_test.go +++ b/models/migrations/v1_22/v283_test.go @@ -6,7 +6,7 @@ package v1_22 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" ) @@ -21,7 +21,7 @@ func Test_AddCombinedIndexToIssueUser(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(IssueUser)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(IssueUser)) defer deferable() assert.NoError(t, AddCombinedIndexToIssueUser(x)) diff --git a/models/migrations/v1_22/v286_test.go b/models/migrations/v1_22/v286_test.go index b4a50f6fcb4..1bd7fac2f18 100644 --- a/models/migrations/v1_22/v286_test.go +++ b/models/migrations/v1_22/v286_test.go @@ -6,7 +6,7 @@ package v1_22 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" "xorm.io/xorm" @@ -64,7 +64,7 @@ func PrepareOldRepository(t *testing.T) (*xorm.Engine, func()) { } // Prepare and load the testing database - return base.PrepareTestEnv(t, 0, + return migrationtest.PrepareTestEnv(t, 0, new(Repository), new(CommitStatus), new(RepoArchiver), diff --git a/models/migrations/v1_22/v287_test.go b/models/migrations/v1_22/v287_test.go index 2b42a33c389..21946a662ab 100644 --- a/models/migrations/v1_22/v287_test.go +++ b/models/migrations/v1_22/v287_test.go @@ -7,7 +7,7 @@ import ( "strconv" "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" ) @@ -20,7 +20,7 @@ func Test_UpdateBadgeColName(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(Badge)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(Badge)) defer deferable() if x == nil || t.Failed() { return diff --git a/models/migrations/v1_22/v293_test.go b/models/migrations/v1_22/v293_test.go index c7b643c7e08..bc3a33055cc 100644 --- a/models/migrations/v1_22/v293_test.go +++ b/models/migrations/v1_22/v293_test.go @@ -6,7 +6,7 @@ package v1_22 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "code.gitea.io/gitea/models/project" "github.com/stretchr/testify/assert" @@ -14,7 +14,7 @@ import ( func Test_CheckProjectColumnsConsistency(t *testing.T) { // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(project.Project), new(project.Column)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(project.Project), new(project.Column)) defer deferable() if x == nil || t.Failed() { return diff --git a/models/migrations/v1_22/v294_test.go b/models/migrations/v1_22/v294_test.go index 1cf03d61201..a711b5ec5f0 100644 --- a/models/migrations/v1_22/v294_test.go +++ b/models/migrations/v1_22/v294_test.go @@ -6,7 +6,7 @@ package v1_22 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" "xorm.io/xorm/schemas" @@ -20,7 +20,7 @@ func Test_AddUniqueIndexForProjectIssue(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(ProjectIssue)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(ProjectIssue)) defer deferable() if x == nil || t.Failed() { return diff --git a/models/migrations/v1_23/main_test.go b/models/migrations/v1_23/main_test.go index f7b2caed83d..ffccac0fd3d 100644 --- a/models/migrations/v1_23/main_test.go +++ b/models/migrations/v1_23/main_test.go @@ -6,9 +6,9 @@ package v1_23 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_23/v302_test.go b/models/migrations/v1_23/v302_test.go index b008b6fc03d..1832adf39ab 100644 --- a/models/migrations/v1_23/v302_test.go +++ b/models/migrations/v1_23/v302_test.go @@ -6,7 +6,7 @@ package v1_23 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "code.gitea.io/gitea/modules/timeutil" "github.com/stretchr/testify/assert" @@ -44,7 +44,7 @@ func Test_AddIndexToActionTaskStoppedLogExpired(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(ActionTask)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(ActionTask)) defer deferable() assert.NoError(t, AddIndexToActionTaskStoppedLogExpired(x)) diff --git a/models/migrations/v1_23/v304_test.go b/models/migrations/v1_23/v304_test.go index c3dfa5e7e7e..9af84cd257f 100644 --- a/models/migrations/v1_23/v304_test.go +++ b/models/migrations/v1_23/v304_test.go @@ -6,7 +6,7 @@ package v1_23 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "code.gitea.io/gitea/modules/timeutil" "github.com/stretchr/testify/assert" @@ -33,7 +33,7 @@ func Test_AddIndexForReleaseSha1(t *testing.T) { } // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(Release)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(Release)) defer deferable() assert.NoError(t, AddIndexForReleaseSha1(x)) diff --git a/models/migrations/v1_25/main_test.go b/models/migrations/v1_25/main_test.go index d2c4a4105d3..33c981edb97 100644 --- a/models/migrations/v1_25/main_test.go +++ b/models/migrations/v1_25/main_test.go @@ -6,9 +6,9 @@ package v1_25 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_25/v321_test.go b/models/migrations/v1_25/v321_test.go index 3ef2c68aa33..0749a20e20e 100644 --- a/models/migrations/v1_25/v321_test.go +++ b/models/migrations/v1_25/v321_test.go @@ -6,7 +6,7 @@ package v1_25 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" @@ -44,12 +44,12 @@ func Test_UseLongTextInSomeColumnsAndFixBugs(t *testing.T) { } // Prepare and load the testing database - x, deferrable := base.PrepareTestEnv(t, 0, new(ReviewState), new(PackageProperty), new(Notice)) + x, deferrable := migrationtest.PrepareTestEnv(t, 0, new(ReviewState), new(PackageProperty), new(Notice)) defer deferrable() require.NoError(t, UseLongTextInSomeColumnsAndFixBugs(x)) - tables := base.LoadTableSchemasMap(t, x) + tables := migrationtest.LoadTableSchemasMap(t, x) table := tables["review_state"] column := table.GetColumn("updated_files") assert.Equal(t, "LONGTEXT", column.SQLType.Name) diff --git a/models/migrations/v1_25/v322_test.go b/models/migrations/v1_25/v322_test.go index 78d890704c5..19646140351 100644 --- a/models/migrations/v1_25/v322_test.go +++ b/models/migrations/v1_25/v322_test.go @@ -6,7 +6,7 @@ package v1_25 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" @@ -23,11 +23,11 @@ func Test_ExtendCommentTreePathLength(t *testing.T) { TreePath string `xorm:"VARCHAR(255)"` } - x, deferrable := base.PrepareTestEnv(t, 0, new(Comment)) + x, deferrable := migrationtest.PrepareTestEnv(t, 0, new(Comment)) defer deferrable() require.NoError(t, ExtendCommentTreePathLength(x)) - table := base.LoadTableSchemasMap(t, x)["comment"] + table := migrationtest.LoadTableSchemasMap(t, x)["comment"] column := table.GetColumn("tree_path") assert.Contains(t, []string{"NVARCHAR", "VARCHAR"}, column.SQLType.Name) assert.EqualValues(t, 4000, column.Length) diff --git a/models/migrations/v1_26/main_test.go b/models/migrations/v1_26/main_test.go index 5aa12d553c9..0b271b9bbcf 100644 --- a/models/migrations/v1_26/main_test.go +++ b/models/migrations/v1_26/main_test.go @@ -6,9 +6,9 @@ package v1_26 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_26/v325_test.go b/models/migrations/v1_26/v325_test.go index d4a66fee816..3fd658e01b9 100644 --- a/models/migrations/v1_26/v325_test.go +++ b/models/migrations/v1_26/v325_test.go @@ -6,7 +6,7 @@ package v1_26 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "code.gitea.io/gitea/modules/timeutil" "github.com/stretchr/testify/require" @@ -38,7 +38,7 @@ func Test_FixMissedRepoIDWhenMigrateAttachments(t *testing.T) { } // Prepare and load the testing database - x, deferrable := base.PrepareTestEnv(t, 0, new(Attachment), new(Issue), new(Release)) + x, deferrable := migrationtest.PrepareTestEnv(t, 0, new(Attachment), new(Issue), new(Release)) defer deferrable() require.NoError(t, FixMissedRepoIDWhenMigrateAttachments(x)) diff --git a/models/migrations/v1_26/v326_test.go b/models/migrations/v1_26/v326_test.go index b92eed35f69..a0225eb774c 100644 --- a/models/migrations/v1_26/v326_test.go +++ b/models/migrations/v1_26/v326_test.go @@ -6,7 +6,7 @@ package v1_26 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/test" @@ -57,7 +57,7 @@ func Test_FixCommitStatusTargetURLToUseRunAndJobID(t *testing.T) { TargetURL string } - x, deferable := base.PrepareTestEnv(t, 0, + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(Repository), new(ActionRun), new(ActionRunJob), diff --git a/models/migrations/v1_26/v327_test.go b/models/migrations/v1_26/v327_test.go index 971707be4f9..98e948cf05c 100644 --- a/models/migrations/v1_26/v327_test.go +++ b/models/migrations/v1_26/v327_test.go @@ -6,7 +6,7 @@ package v1_26 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/require" ) @@ -17,7 +17,7 @@ func Test_AddDisabledToActionRunner(t *testing.T) { Name string } - x, deferable := base.PrepareTestEnv(t, 0, new(ActionRunner)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(ActionRunner)) defer deferable() _, err := x.Insert(&ActionRunner{Name: "runner"}) diff --git a/models/migrations/v1_26/v329_test.go b/models/migrations/v1_26/v329_test.go index cab8e79906d..e4bebfb71dc 100644 --- a/models/migrations/v1_26/v329_test.go +++ b/models/migrations/v1_26/v329_test.go @@ -6,7 +6,7 @@ package v1_26 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" ) @@ -22,7 +22,7 @@ func (UserBadgeBefore) TableName() string { } func Test_AddUniqueIndexForUserBadge(t *testing.T) { - x, deferable := base.PrepareTestEnv(t, 0, new(UserBadgeBefore)) + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(UserBadgeBefore)) defer deferable() if x == nil || t.Failed() { return diff --git a/models/migrations/v1_27/main_test.go b/models/migrations/v1_27/main_test.go index e269e3df9a8..0c6a6a24401 100644 --- a/models/migrations/v1_27/main_test.go +++ b/models/migrations/v1_27/main_test.go @@ -6,9 +6,9 @@ package v1_27 import ( "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" ) func TestMain(m *testing.M) { - base.MainTest(m) + migrationtest.MainTest(m) } diff --git a/models/migrations/v1_27/v331_test.go b/models/migrations/v1_27/v331_test.go index 45f467cf9bc..2302fee024f 100644 --- a/models/migrations/v1_27/v331_test.go +++ b/models/migrations/v1_27/v331_test.go @@ -8,7 +8,7 @@ import ( "slices" "testing" - "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/models/migrations/migrationtest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -49,7 +49,7 @@ func (actionArtifactBeforeV331) TableName() string { } func Test_AddActionRunAttemptModel(t *testing.T) { - x, deferable := base.PrepareTestEnv(t, 0, + x, deferable := migrationtest.PrepareTestEnv(t, 0, new(actionRunBeforeV331), new(actionRunJobBeforeV331), new(actionArtifactBeforeV331), @@ -69,7 +69,7 @@ func Test_AddActionRunAttemptModel(t *testing.T) { require.NoError(t, AddActionRunAttemptModel(x)) - tableMap := base.LoadTableSchemasMap(t, x) + tableMap := migrationtest.LoadTableSchemasMap(t, x) attemptTable := tableMap["action_run_attempt"] require.NotNil(t, attemptTable) diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index bd832348e7c..116fdab4968 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -5,6 +5,8 @@ package unittest import ( "context" + "database/sql" + "errors" "fmt" "os" "path/filepath" @@ -102,6 +104,101 @@ func mainTest(m *testing.M, testOptsArg ...*TestOptions) int { return exitStatus } +func ResetTestDatabase() (cleanup func(), err error) { + defer func() { + if cleanup == nil { + cleanup = func() {} + } + }() + + connOpts := db.GlobalConnOptions() + driverDefault, connStrDefault, err := db.ConnStrDefaultDatabase(connOpts) + if err != nil { + return nil, err + } + driverDatabase, connStrDatabase, err := db.ConnStr(connOpts) + if err != nil { + return nil, err + } + + if connOpts.Type.IsSQLite3() { + if !strings.HasSuffix(connOpts.SQLitePath, "-test.db") { + return nil, errors.New(`testing database file for sqlite3 must end in "-test.db"`) + } + _ = os.Remove(connOpts.SQLitePath) + err = os.MkdirAll(filepath.Dir(connOpts.SQLitePath), os.ModePerm) + if err != nil { + return nil, err + } + cleanup = func() { + _ = os.Remove(connOpts.SQLitePath) + _ = os.Remove(filepath.Dir(connOpts.SQLitePath)) + } + return cleanup, nil + } + + if !strings.Contains(connOpts.Database, "test") { + return nil, fmt.Errorf(`testing database name for %s must contain "test"`, connOpts.Database) + } + + quotedDbName := connOpts.Database + if connOpts.Type.IsMSSQL() { + quotedDbName = `[` + connOpts.Database + `]` + } + + sqlExec := func(sqlDB *sql.DB, sql string) error { + _, err := sqlDB.Exec(sql) + if err != nil { + return fmt.Errorf("failed to execute SQL %q: %w", sql, err) + } + return nil + } + + createDatabase := func() error { + sqlDB, err := sql.Open(driverDefault, connStrDefault) + if err != nil { + return err + } + defer sqlDB.Close() + if err = sqlExec(sqlDB, "DROP DATABASE IF EXISTS "+quotedDbName); err != nil { + return err + } + return sqlExec(sqlDB, "CREATE DATABASE "+quotedDbName) + } + if err = createDatabase(); err != nil { + return nil, err + } + + cleanup = func() { + sqlDB, err := sql.Open(driverDefault, connStrDefault) + if err != nil { + return + } + defer sqlDB.Close() + _, _ = sqlDB.Exec("DROP DATABASE IF EXISTS " + quotedDbName) + } + + createDatabaseSchema := func() error { + if !connOpts.Type.IsPostgreSQL() { + return nil + } + if connOpts.Schema == "" { + return nil + } + sqlDB, err := sql.Open(driverDatabase, connStrDatabase) + if err != nil { + return err + } + defer sqlDB.Close() + if err = sqlExec(sqlDB, "DROP SCHEMA IF EXISTS "+connOpts.Schema); err != nil { + return err + } + return sqlExec(sqlDB, "CREATE SCHEMA "+connOpts.Schema) + } + + return cleanup, createDatabaseSchema() +} + // FixturesOptions fixtures needs to be loaded options type FixturesOptions struct { Dir string @@ -110,11 +207,12 @@ type FixturesOptions struct { // CreateTestEngine creates a memory database and loads the fixture data from fixturesDir func CreateTestEngine(opts FixturesOptions) error { - x, err := xorm.NewEngine("sqlite3", "file::memory:?cache=shared&_txlock=immediate") + driver, connStr, err := db.ConnStr(db.ConnOptions{Type: "sqlite3", SQLitePath: ":memory:"}) + if err != nil { + return err + } + x, err := xorm.NewEngine(driver, connStr) if err != nil { - if strings.Contains(err.Error(), "unknown driver") { - return fmt.Errorf("sqlite3 requires: -tags sqlite,sqlite_unlock_notify\n%w", err) - } return err } x.SetMapper(names.GonicMapper{}) diff --git a/modules/setting/database.go b/modules/setting/database.go index 1a4bf648058..2b069a62922 100644 --- a/modules/setting/database.go +++ b/modules/setting/database.go @@ -4,13 +4,7 @@ package setting import ( - "errors" - "fmt" - "net" - "net/url" - "os" "path/filepath" - "strings" "time" ) @@ -20,24 +14,22 @@ var ( // DatabaseTypeNames contains the friendly names for all database types DatabaseTypeNames = map[string]string{"mysql": "MySQL", "postgres": "PostgreSQL", "mssql": "MSSQL", "sqlite3": "SQLite3"} - // EnableSQLite3 use SQLite3, set by build flag - EnableSQLite3 bool - // Database holds the database settings Database = struct { - Type DatabaseType - Host string - Name string - User string - Passwd string - Schema string - SSLMode string - Path string + Type DatabaseType + Host string + Name string + User string + Passwd string + Schema string + SSLMode string + Path string + + SQLiteBusyTimeout int + SQLiteJournalMode string + LogSQL bool - MysqlCharset string CharsetCollation string - Timeout int // seconds - SQLiteJournalMode string DBConnectRetries int DBConnectBackoff time.Duration MaxIdleConns int @@ -47,7 +39,7 @@ var ( AutoMigration bool SlowQueryThreshold time.Duration }{ - Timeout: 500, + SQLiteBusyTimeout: 500, IterateBufferSize: 50, } ) @@ -64,15 +56,14 @@ func loadDBSetting(rootCfg ConfigProvider) { Database.Host = sec.Key("HOST").String() Database.Name = sec.Key("NAME").String() Database.User = sec.Key("USER").String() - if len(Database.Passwd) == 0 { - Database.Passwd = sec.Key("PASSWD").String() - } + Database.Passwd = sec.Key("PASSWD").String() + Database.Schema = sec.Key("SCHEMA").String() Database.SSLMode = sec.Key("SSL_MODE").MustString("disable") Database.CharsetCollation = sec.Key("CHARSET_COLLATION").String() Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db")) - Database.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500) + Database.SQLiteBusyTimeout = sec.Key("SQLITE_TIMEOUT").MustInt(500) Database.SQLiteJournalMode = sec.Key("SQLITE_JOURNAL_MODE").MustString("") Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(2) @@ -91,123 +82,9 @@ func loadDBSetting(rootCfg ConfigProvider) { Database.SlowQueryThreshold = sec.Key("SLOW_QUERY_THRESHOLD").MustDuration(5 * time.Second) } -// DBConnStr returns database connection string -func DBConnStr() (string, error) { - var connStr string - paramSep := "?" - if strings.Contains(Database.Name, paramSep) { - paramSep = "&" - } - switch Database.Type { - case "mysql": - connType := "tcp" - if len(Database.Host) > 0 && Database.Host[0] == '/' { // looks like a unix socket - connType = "unix" - } - tls := Database.SSLMode - if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL - tls = "false" - } - connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%sparseTime=true&tls=%s", - Database.User, Database.Passwd, connType, Database.Host, Database.Name, paramSep, tls) - case "postgres": - connStr = getPostgreSQLConnectionString(Database.Host, Database.User, Database.Passwd, Database.Name, Database.SSLMode) - case "mssql": - host, port := ParseMSSQLHostPort(Database.Host) - connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, Database.Name, Database.User, Database.Passwd) - case "sqlite3": - if !EnableSQLite3 { - return "", errors.New("this Gitea binary was not built with SQLite3 support") - } - if err := os.MkdirAll(filepath.Dir(Database.Path), os.ModePerm); err != nil { - return "", fmt.Errorf("Failed to create directories: %w", err) - } - journalMode := "" - if Database.SQLiteJournalMode != "" { - journalMode = "&_journal_mode=" + Database.SQLiteJournalMode - } - connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d&_txlock=immediate%s", - Database.Path, Database.Timeout, journalMode) - default: - return "", fmt.Errorf("unknown database type: %s", Database.Type) - } - - return connStr, nil -} - -// parsePostgreSQLHostPort parses given input in various forms defined in -// https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING -// and returns proper host and port number. -func parsePostgreSQLHostPort(info string) (host, port string) { - if h, p, err := net.SplitHostPort(info); err == nil { - host, port = h, p - } else { - // treat the "info" as "host", if it's an IPv6 address, remove the wrapper - host = info - if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { - host = host[1 : len(host)-1] - } - } - - // set fallback values - if host == "" { - host = "127.0.0.1" - } - if port == "" { - port = "5432" - } - return host, port -} - -func getPostgreSQLConnectionString(dbHost, dbUser, dbPasswd, dbName, dbsslMode string) (connStr string) { - dbName, dbParam, _ := strings.Cut(dbName, "?") - host, port := parsePostgreSQLHostPort(dbHost) - connURL := url.URL{ - Scheme: "postgres", - User: url.UserPassword(dbUser, dbPasswd), - Host: net.JoinHostPort(host, port), - Path: dbName, - OmitHost: false, - RawQuery: dbParam, - } - query := connURL.Query() - if strings.HasPrefix(host, "/") { // looks like a unix socket - query.Add("host", host) - connURL.Host = ":" + port - } - query.Set("sslmode", dbsslMode) - connURL.RawQuery = query.Encode() - return connURL.String() -} - -// ParseMSSQLHostPort splits the host into host and port -func ParseMSSQLHostPort(info string) (string, string) { - // the default port "0" might be related to MSSQL's dynamic port, maybe it should be double-confirmed in the future - host, port := "127.0.0.1", "0" - if strings.Contains(info, ":") { - host = strings.Split(info, ":")[0] - port = strings.Split(info, ":")[1] - } else if strings.Contains(info, ",") { - host = strings.Split(info, ",")[0] - port = strings.TrimSpace(strings.Split(info, ",")[1]) - } else if len(info) > 0 { - host = info - } - if host == "" { - host = "127.0.0.1" - } - if port == "" { - port = "0" - } - return host, port -} - +// DatabaseType FIXME: it is also used directly with "schemas.DBType", so the names must be consistent type DatabaseType string -func (t DatabaseType) String() string { - return string(t) -} - func (t DatabaseType) IsSQLite3() bool { return t == "sqlite3" } diff --git a/modules/setting/database_sqlite.go b/modules/setting/database_sqlite.go deleted file mode 100644 index c1037cfb27e..00000000000 --- a/modules/setting/database_sqlite.go +++ /dev/null @@ -1,15 +0,0 @@ -//go:build sqlite - -// Copyright 2014 The Gogs Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package setting - -import ( - _ "github.com/mattn/go-sqlite3" -) - -func init() { - EnableSQLite3 = true - SupportedDatabaseTypes = append(SupportedDatabaseTypes, "sqlite3") -} diff --git a/options/locale/locale_en-US.json b/options/locale/locale_en-US.json index 9c6903edc6c..80692acdaf3 100644 --- a/options/locale/locale_en-US.json +++ b/options/locale/locale_en-US.json @@ -3315,7 +3315,6 @@ "admin.config.cache_config": "Cache Configuration", "admin.config.cache_adapter": "Cache Adapter", "admin.config.cache_interval": "Cache Interval", - "admin.config.cache_conn": "Cache Connection", "admin.config.cache_item_ttl": "Cache Item TTL", "admin.config.cache_test": "Test Cache", "admin.config.cache_test_failed": "Failed to probe the cache: %v.", @@ -3330,7 +3329,6 @@ "admin.config.instance_web_banner.message_placeholder": "Banner message (supports markdown)", "admin.config.session_config": "Session Configuration", "admin.config.session_provider": "Session Provider", - "admin.config.provider_config": "Provider Config", "admin.config.cookie_name": "Cookie Name", "admin.config.gc_interval_time": "GC Interval Time", "admin.config.session_life_time": "Session Life Time", diff --git a/routers/init.go b/routers/init.go index 92eab5eaf28..e04b711c4d3 100644 --- a/routers/init.go +++ b/routers/init.go @@ -134,12 +134,6 @@ func InitWebInstalled(ctx context.Context) { external.RegisterRenderers() markup.Init(markup_service.FormalRenderHelperFuncs()) - if setting.EnableSQLite3 { - log.Info("SQLite3 support is enabled") - } else if setting.Database.Type.IsSQLite3() { - log.Fatal("SQLite3 support is disabled, but it is used for database setting. Please get or build a Gitea release with SQLite3 support.") - } - mustInitCtx(ctx, common.InitDBEngine) log.Info("ORM engine initialization successful!") mustInit(system.Init) diff --git a/routers/install/install.go b/routers/install/install.go index a0f32fb939c..718ede6564d 100644 --- a/routers/install/install.go +++ b/routers/install/install.go @@ -76,7 +76,7 @@ func Install(ctx *context.Context) { form.DbSchema = setting.Database.Schema form.SSLMode = setting.Database.SSLMode - curDBType := setting.Database.Type.String() + curDBType := string(setting.Database.Type) if !slices.Contains(setting.SupportedDatabaseTypes, curDBType) { curDBType = "mysql" } @@ -328,7 +328,7 @@ func SubmitInstall(ctx *context.Context) { cfg.Section("").Key("WORK_PATH").SetValue(setting.AppWorkPath) cfg.Section("").Key("RUN_MODE").SetValue("prod") - cfg.Section("database").Key("DB_TYPE").SetValue(setting.Database.Type.String()) + cfg.Section("database").Key("DB_TYPE").SetValue(string(setting.Database.Type)) cfg.Section("database").Key("HOST").SetValue(setting.Database.Host) cfg.Section("database").Key("NAME").SetValue(setting.Database.Name) cfg.Section("database").Key("USER").SetValue(setting.Database.User) diff --git a/routers/web/admin/admin_test.go b/routers/web/admin/admin_test.go index ecdd462f9e7..929faa19680 100644 --- a/routers/web/admin/admin_test.go +++ b/routers/web/admin/admin_test.go @@ -16,64 +16,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestShadowPassword(t *testing.T) { - kases := []struct { - Provider string - CfgItem string - Result string - }{ - { - Provider: "redis", - CfgItem: "network=tcp,addr=:6379,password=gitea,db=0,pool_size=100,idle_timeout=180", - Result: "network=tcp,addr=:6379,password=******,db=0,pool_size=100,idle_timeout=180", - }, - { - Provider: "mysql", - CfgItem: "root:@tcp(localhost:3306)/gitea?charset=utf8", - Result: "root:******@tcp(localhost:3306)/gitea?charset=utf8", - }, - { - Provider: "mysql", - CfgItem: "/gitea?charset=utf8", - Result: "/gitea?charset=utf8", - }, - { - Provider: "mysql", - CfgItem: "user:mypassword@/dbname", - Result: "user:******@/dbname", - }, - { - Provider: "postgres", - CfgItem: "user=pqgotest dbname=pqgotest sslmode=verify-full", - Result: "user=pqgotest dbname=pqgotest sslmode=verify-full", - }, - { - Provider: "postgres", - CfgItem: "user=pqgotest password= dbname=pqgotest sslmode=verify-full", - Result: "user=pqgotest password=****** dbname=pqgotest sslmode=verify-full", - }, - { - Provider: "postgres", - CfgItem: "postgres://user:pass@hostname/dbname", - Result: "postgres://user:******@hostname/dbname", - }, - { - Provider: "couchbase", - CfgItem: "http://dev-couchbase.example.com:8091/", - Result: "http://dev-couchbase.example.com:8091/", - }, - { - Provider: "couchbase", - CfgItem: "http://user:the_password@dev-couchbase.example.com:8091/", - Result: "http://user:******@dev-couchbase.example.com:8091/", - }, - } - - for _, k := range kases { - assert.Equal(t, k.Result, shadowPassword(k.Provider, k.CfgItem)) - } -} - func TestSelfCheckPost(t *testing.T) { defer test.MockVariableValue(&setting.PublicURLDetection)() defer test.MockVariableValue(&setting.AppURL, "http://config/sub/")() diff --git a/routers/web/admin/config.go b/routers/web/admin/config.go index bf48e554dfd..03a15b67134 100644 --- a/routers/web/admin/config.go +++ b/routers/web/admin/config.go @@ -7,8 +7,6 @@ package admin import ( "errors" "net/http" - "net/url" - "strings" system_model "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/modules/cache" @@ -59,63 +57,6 @@ func TestCache(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/-/admin/config") } -func shadowPasswordKV(cfgItem, splitter string) string { - fields := strings.Split(cfgItem, splitter) - for i := range fields { - if strings.HasPrefix(fields[i], "password=") { - fields[i] = "password=******" - break - } - } - return strings.Join(fields, splitter) -} - -func shadowURL(provider, cfgItem string) string { - u, err := url.Parse(cfgItem) - if err != nil { - log.Error("Shadowing Password for %v failed: %v", provider, err) - return cfgItem - } - if u.User != nil { - atIdx := strings.Index(cfgItem, "@") - if atIdx > 0 { - colonIdx := strings.LastIndex(cfgItem[:atIdx], ":") - if colonIdx > 0 { - return cfgItem[:colonIdx+1] + "******" + cfgItem[atIdx:] - } - } - } - return cfgItem -} - -func shadowPassword(provider, cfgItem string) string { - switch provider { - case "redis": - return shadowPasswordKV(cfgItem, ",") - case "mysql": - // root:@tcp(localhost:3306)/macaron?charset=utf8 - atIdx := strings.Index(cfgItem, "@") - if atIdx > 0 { - colonIdx := strings.Index(cfgItem[:atIdx], ":") - if colonIdx > 0 { - return cfgItem[:colonIdx+1] + "******" + cfgItem[atIdx:] - } - } - return cfgItem - case "postgres": - // user=jiahuachen dbname=macaron port=5432 sslmode=disable - if !strings.HasPrefix(cfgItem, "postgres://") { - return shadowPasswordKV(cfgItem, " ") - } - fallthrough - case "couchbase": - return shadowURL(provider, cfgItem) - // postgres://pqgotest:password@localhost/pqgotest?sslmode=verify-full - // Notice: use shadowURL - } - return cfgItem -} - // Config show admin config page func Config(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.config_summary") @@ -150,8 +91,6 @@ func Config(ctx *context.Context) { ctx.Data["CacheAdapter"] = setting.CacheService.Adapter ctx.Data["CacheInterval"] = setting.CacheService.Interval - - ctx.Data["CacheConn"] = shadowPassword(setting.CacheService.Adapter, setting.CacheService.Conn) ctx.Data["CacheItemTTL"] = setting.CacheService.TTL sessionCfg := setting.SessionConfig @@ -169,7 +108,7 @@ func Config(ctx *context.Context) { sessionCfg.Secure = realSession.Secure sessionCfg.Domain = realSession.Domain } - sessionCfg.ProviderConfig = shadowPassword(sessionCfg.Provider, sessionCfg.ProviderConfig) + sessionCfg.ProviderConfig = "" ctx.Data["SessionConfig"] = sessionCfg ctx.Data["Git"] = setting.Git diff --git a/routers/web/healthcheck/check.go b/routers/web/healthcheck/check.go index de9b2c8ec17..116aab886bc 100644 --- a/routers/web/healthcheck/check.go +++ b/routers/web/healthcheck/check.go @@ -111,16 +111,10 @@ func checkDatabase(ctx context.Context, checks checks) status { } if setting.Database.Type.IsSQLite3() && st.Status == pass { - if !setting.EnableSQLite3 { + if _, err := os.Stat(setting.Database.Path); err != nil { st.Status = fail st.Time = getCheckTime() - log.Error("SQLite3 health check failed with error: %v", "this Gitea binary is built without SQLite3 enabled") - } else { - if _, err := os.Stat(setting.Database.Path); err != nil { - st.Status = fail - st.Time = getCheckTime() - log.Error("SQLite3 file exists check failed with error: %v", err) - } + log.Error("SQLite3 file exists check failed with error: %v", err) } } diff --git a/templates/admin/config.tmpl b/templates/admin/config.tmpl index b68f2c1a7ae..c381c5bf1d0 100644 --- a/templates/admin/config.tmpl +++ b/templates/admin/config.tmpl @@ -244,8 +244,6 @@
{{.CacheInterval}} {{ctx.Locale.Tr "tool.raw_seconds"}}
{{end}} {{if .CacheConn}} -
{{ctx.Locale.Tr "admin.config.cache_conn"}}
-
{{.CacheConn}}
{{ctx.Locale.Tr "admin.config.cache_item_ttl"}}
{{.CacheItemTTL}}
{{end}} @@ -266,8 +264,6 @@
{{ctx.Locale.Tr "admin.config.session_provider"}}
{{.SessionConfig.Provider}}
-
{{ctx.Locale.Tr "admin.config.provider_config"}}
-
{{if .SessionConfig.ProviderConfig}}{{.SessionConfig.ProviderConfig}}{{else}}-{{end}}
{{ctx.Locale.Tr "admin.config.cookie_name"}}
{{.SessionConfig.CookieName}}
{{ctx.Locale.Tr "admin.config.gc_interval_time"}}
diff --git a/tests/integration/migration-test/gitea-v1.6.4.mssql.sql.gz b/tests/integration/migration-test/gitea-v1.6.4.mssql.sql.gz index 1b676feda1c..85ba253cf88 100644 Binary files a/tests/integration/migration-test/gitea-v1.6.4.mssql.sql.gz and b/tests/integration/migration-test/gitea-v1.6.4.mssql.sql.gz differ diff --git a/tests/integration/migration-test/gitea-v1.7.0.mssql.sql.gz b/tests/integration/migration-test/gitea-v1.7.0.mssql.sql.gz index bd869cfa583..4f36c7c0cf1 100644 Binary files a/tests/integration/migration-test/gitea-v1.7.0.mssql.sql.gz and b/tests/integration/migration-test/gitea-v1.7.0.mssql.sql.gz differ diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index 2d6fc947a75..d28442b7140 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -11,7 +11,6 @@ import ( "fmt" "io" "os" - "path" "path/filepath" "regexp" "sort" @@ -26,7 +25,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/testlogger" - "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -55,7 +53,7 @@ func availableVersions() ([]string, error) { return nil, err } defer migrationsDir.Close() - versionRE, err := regexp.Compile("gitea-v(?P.+)" + regexp.QuoteMeta("."+setting.Database.Type.String()+".sql.gz")) + versionRE, err := regexp.Compile("gitea-v(?P.+)" + regexp.QuoteMeta("."+string(setting.Database.Type)+".sql.gz")) if err != nil { return nil, err } @@ -64,7 +62,7 @@ func availableVersions() ([]string, error) { if err != nil { return nil, err } - versions := []string{} + var versions []string for _, filename := range filenames { if versionRE.MatchString(filename) { substrings := versionRE.FindStringSubmatch(filename) @@ -76,11 +74,8 @@ func availableVersions() ([]string, error) { } func readSQLFromFile(version string) (string, error) { - filename := filepath.Join(setting.GetGiteaTestSourceRoot(), "tests/integration/migration-test", fmt.Sprintf("gitea-v%s.%s.sql.gz", version, setting.Database.Type)) - - if _, err := os.Stat(filename); os.IsNotExist(err) { - return "", nil - } + filename := fmt.Sprintf("tests/integration/migration-test/gitea-v%s.%s.sql.gz", version, setting.Database.Type) + filename = filepath.Join(setting.GetGiteaTestSourceRoot(), filename) file, err := os.Open(filename) if err != nil { @@ -106,134 +101,51 @@ func restoreOldDB(t *testing.T, version string) { require.NoError(t, err) require.NotEmpty(t, data, "No data found for %s version: %s", setting.Database.Type, version) - switch { - case setting.Database.Type.IsSQLite3(): - util.Remove(setting.Database.Path) - err := os.MkdirAll(path.Dir(setting.Database.Path), os.ModePerm) - assert.NoError(t, err) + cleanup, err := unittest.ResetTestDatabase() + require.NoError(t, err) + _ = cleanup // no clean up yet (not needed at the moment) - db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d&_txlock=immediate", setting.Database.Path, setting.Database.Timeout)) - assert.NoError(t, err) - defer db.Close() + connOpts := db.GlobalConnOptions() - _, err = db.Exec(data) - assert.NoError(t, err) - db.Close() - - case setting.Database.Type.IsMySQL(): - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/", - setting.Database.User, setting.Database.Passwd, setting.Database.Host)) - assert.NoError(t, err) - defer db.Close() - - _, err = db.Exec("DROP DATABASE IF EXISTS " + setting.Database.Name) - assert.NoError(t, err) - - _, err = db.Exec("CREATE DATABASE IF NOT EXISTS " + setting.Database.Name) - assert.NoError(t, err) - db.Close() - - db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?multiStatements=true", - setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name)) - assert.NoError(t, err) - defer db.Close() - - _, err = db.Exec(data) - assert.NoError(t, err) - db.Close() - - case setting.Database.Type.IsPostgreSQL(): - var db *sql.DB - var err error - if setting.Database.Host[0] == '/' { - db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/?sslmode=%s&host=%s", - setting.Database.User, setting.Database.Passwd, setting.Database.SSLMode, setting.Database.Host)) - assert.NoError(t, err) - } else { - db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s", - setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode)) - assert.NoError(t, err) + if !connOpts.Type.IsMSSQL() { + if connOpts.Type.IsMySQL() { + connOpts.Database += "?multiStatements=true" } - defer db.Close() + driver, connStr, err := db.ConnStr(connOpts) + require.NoError(t, err) - _, err = db.Exec("DROP DATABASE IF EXISTS " + setting.Database.Name) - assert.NoError(t, err) + sqlDB, err := sql.Open(driver, connStr) + require.NoError(t, err) + defer sqlDB.Close() - _, err = db.Exec("CREATE DATABASE " + setting.Database.Name) - assert.NoError(t, err) - db.Close() - - // Check if we need to setup a specific schema - if len(setting.Database.Schema) != 0 { - if setting.Database.Host[0] == '/' { - db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/%s?sslmode=%s&host=%s", - setting.Database.User, setting.Database.Passwd, setting.Database.Name, setting.Database.SSLMode, setting.Database.Host)) - } else { - db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", - setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode)) - } - require.NoError(t, err) - defer db.Close() - - schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema)) - require.NoError(t, err) - require.NotEmpty(t, schrows) - - if !schrows.Next() { - // Create and setup a DB schema - _, err = db.Exec("CREATE SCHEMA " + setting.Database.Schema) - assert.NoError(t, err) - } - schrows.Close() - - // Make the user's default search path the created schema; this will affect new connections - _, err = db.Exec(fmt.Sprintf(`ALTER USER "%s" SET search_path = %s`, setting.Database.User, setting.Database.Schema)) - assert.NoError(t, err) - - db.Close() - } - - if setting.Database.Host[0] == '/' { - db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/%s?sslmode=%s&host=%s", - setting.Database.User, setting.Database.Passwd, setting.Database.Name, setting.Database.SSLMode, setting.Database.Host)) - } else { - db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", - setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode)) - } - assert.NoError(t, err) - defer db.Close() - - _, err = db.Exec(data) - assert.NoError(t, err) - db.Close() - - case setting.Database.Type.IsMSSQL(): - host, port := setting.ParseMSSQLHostPort(setting.Database.Host) - db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", - host, port, "master", setting.Database.User, setting.Database.Passwd)) - assert.NoError(t, err) - defer db.Close() - - _, err = db.Exec("DROP DATABASE IF EXISTS [gitea]") - assert.NoError(t, err) - - statements := strings.Split(data, "\nGO\n") - for _, statement := range statements { - if len(statement) > 5 && statement[:5] == "USE [" { - dbname := statement[5 : len(statement)-1] - db.Close() - db, err = sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", - host, port, dbname, setting.Database.User, setting.Database.Passwd)) - assert.NoError(t, err) - defer db.Close() - } - _, err = db.Exec(statement) - assert.NoError(t, err, "Failure whilst running: %s\nError: %v", statement, err) - } - db.Close() - default: - assert.Failf(t, "unsupported database type", "setting.Database.Type=%v", setting.Database.Type) + _, err = sqlDB.Exec(data) + require.NoError(t, err) + return } + + // MSSQL is special. the test fixture will create the [testgitea] database again, so drop it ahead if it exists + driver, connStr, err := db.ConnStrDefaultDatabase(connOpts) + require.NoError(t, err) + sqlDB, err := sql.Open(driver, connStr) + require.NoError(t, err) + + _, err = sqlDB.Exec("DROP DATABASE IF EXISTS [testgitea]") + require.NoError(t, err, "drop existing database testgitea") + + for statement := range strings.SplitSeq(data, "\nGO\n") { + if useStmtAfter, ok := strings.CutPrefix(statement, "USE ["); ok { + _ = sqlDB.Close() + dbname := strings.TrimSuffix(useStmtAfter, "]") // extract the database name from "USE [dbname]" + connOpts.Database = dbname + driver, connStr, err := db.ConnStr(connOpts) + require.NoError(t, err) + sqlDB, err = sql.Open(driver, connStr) + require.NoError(t, err) + } + _, err = sqlDB.Exec(statement) + require.NoError(t, err, "SQL Exec failed when running: %s\nError: %v", statement, err) + } + _ = sqlDB.Close() } func wrappedMigrate(ctx context.Context, x *xorm.Engine) error { diff --git a/tests/sqlite.ini.tmpl b/tests/sqlite.ini.tmpl index 03393c620b9..95a1df283fa 100644 --- a/tests/sqlite.ini.tmpl +++ b/tests/sqlite.ini.tmpl @@ -4,7 +4,7 @@ RUN_MODE = prod [database] DB_TYPE = sqlite3 -PATH = gitea.db +PATH = gitea-test.db [indexer] REPO_INDEXER_ENABLED = true diff --git a/tests/test_utils.go b/tests/test_utils.go index f16754b0562..11e5ac24344 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -4,10 +4,7 @@ package tests import ( - "database/sql" - "fmt" "path/filepath" - "strings" "testing" "code.gitea.io/gitea/models/db" @@ -40,97 +37,14 @@ func InitIntegrationTest() error { } setting.LoadDBSetting() - if err := storage.Init(); err != nil { + cleanupDb, err := unittest.ResetTestDatabase() + if err != nil { return err } + _ = cleanupDb // no clean up yet (not really needed at the moment) - switch { - case setting.Database.Type.IsMySQL(): - { - connType := util.Iif(strings.HasPrefix(setting.Database.Host, "/"), "unix", "tcp") - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@%s(%s)/", - setting.Database.User, setting.Database.Passwd, connType, setting.Database.Host)) - if err != nil { - return err - } - defer db.Close() - if _, err = db.Exec("CREATE DATABASE IF NOT EXISTS " + setting.Database.Name); err != nil { - return err - } - } - case setting.Database.Type.IsPostgreSQL(): - openPostgreSQL := func() (*sql.DB, error) { - if strings.HasPrefix(setting.Database.Host, "/") { - return sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/%s?sslmode=%s&host=%s", - setting.Database.User, setting.Database.Passwd, setting.Database.Name, setting.Database.SSLMode, setting.Database.Host)) - } - return sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", - setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode)) - } - - // create database - { - db, err := openPostgreSQL() - if err != nil { - return err - } - defer db.Close() - dbRows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name)) - if err != nil { - return err - } - defer dbRows.Close() - - if !dbRows.Next() { - if _, err = db.Exec("CREATE DATABASE " + setting.Database.Name); err != nil { - return err - } - } - // Check if we need to set up a specific schema - if setting.Database.Schema == "" { - break - } - db.Close() - } - - // create schema - { - db, err := openPostgreSQL() - if err != nil { - return err - } - defer db.Close() - - schemaRows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema)) - if err != nil { - return err - } - defer schemaRows.Close() - - if !schemaRows.Next() { - // Create and set up a DB schema - if _, err = db.Exec("CREATE SCHEMA " + setting.Database.Schema); err != nil { - return err - } - } - } - - case setting.Database.Type.IsMSSQL(): - { - host, port := setting.ParseMSSQLHostPort(setting.Database.Host) - db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", - host, port, "master", setting.Database.User, setting.Database.Passwd)) - if err != nil { - return err - } - defer db.Close() - if _, err = db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil { - return err - } - } - case setting.Database.Type.IsSQLite3(): - default: - return fmt.Errorf("unsupported database type: %s", setting.Database.Type) + if err := storage.Init(); err != nil { + return err } routers.InitWebInstalled(graceful.GetManager().HammerContext())