diff --git a/Dockerfile b/Dockerfile index 266c48e7..20d25b2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,12 +14,18 @@ RUN apt-get update \ sqlite3 \ nodejs \ npm \ + unzip \ && rm -rf /var/lib/apt/lists/* # golangci-lint RUN curl -fsSL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ | sh -s -- -b /usr/local/bin v1.60.1 +# Download and install DuckDB +RUN curl -fsSL https://github.com/duckdb/duckdb/releases/download/v1.1.0/duckdb_cli-linux-amd64.zip -o duckdb_cli-linux-amd64.zip \ + && unzip duckdb_cli-linux-amd64.zip \ + && mv duckdb /usr/local/bin/duckdb + # download modules COPY go.* /src/ RUN go mod download diff --git a/go.mod b/go.mod index 69769bb3..fc86b121 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/go-sql-driver/mysql v1.8.1 github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.9 + github.com/marcboeker/go-duckdb v1.8.0 github.com/mattn/go-sqlite3 v1.14.23 github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.4 @@ -28,6 +29,7 @@ require ( github.com/ClickHouse/ch-go v0.62.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/apache/arrow/go/v15 v15.0.2 // indirect + github.com/apache/arrow/go/v17 v17.0.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -46,6 +48,7 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index 0781f883..1759fadb 100644 --- a/go.sum +++ b/go.sum @@ -201,6 +201,8 @@ github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer5 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= +github.com/apache/arrow/go/v17 v17.0.0 h1:RRR2bdqKcdbss9Gxy2NS/hK8i4LDMh23L6BbkN5+F54= +github.com/apache/arrow/go/v17 v17.0.0/go.mod h1:jR7QHkODl15PfYyjM2nU+yTLScZ/qfj7OSUZmJ8putc= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -384,8 +386,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/marcboeker/go-duckdb v1.8.0 h1:iOWv1wTL0JIMqpyns6hCf5XJJI4fY6lmJNk+itx5RRo= +github.com/marcboeker/go-duckdb v1.8.0/go.mod h1:2oV8BZv88S16TKGKM+Lwd0g7DX84x0jMxjTInThC8Is= github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0= github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= @@ -762,8 +768,8 @@ golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNq golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= -gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= +gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= +gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= diff --git a/main_cgo.go b/main_cgo.go index 98a36fa1..3ac3b37d 100644 --- a/main_cgo.go +++ b/main_cgo.go @@ -4,5 +4,6 @@ package main import ( + _ "github.com/amacneil/dbmate/v2/pkg/driver/duckdb" _ "github.com/amacneil/dbmate/v2/pkg/driver/sqlite" ) diff --git a/pkg/driver/duckdb/duckdb.go b/pkg/driver/duckdb/duckdb.go new file mode 100644 index 00000000..71968337 --- /dev/null +++ b/pkg/driver/duckdb/duckdb.go @@ -0,0 +1,301 @@ +//go:build cgo +// +build cgo + +// This driver is HEAVILY based on the sqlite driver +// Even many of the comments are applicable. + +// TODO Features: +// - Add support for schema names, sqlite base implementation doesn't have them, duckdb does. +// - See postgres driver for how to do this. +// - Ensure support of non-table objects (views, macros, etc.) + +package duckdb + +import ( + "bytes" + "database/sql" + "fmt" + "io" + "net/url" + "os" + "regexp" + "strings" + + "github.com/amacneil/dbmate/v2/pkg/dbmate" + "github.com/amacneil/dbmate/v2/pkg/dbutil" + + "github.com/lib/pq" + _ "github.com/marcboeker/go-duckdb" // database/sql driver +) + +func init() { + dbmate.RegisterDriver(NewDriver, "duckdb") +} + +type Driver struct { + migrationsTableName string + databaseURL *url.URL + log io.Writer +} + +func NewDriver(config dbmate.DriverConfig) dbmate.Driver { + return &Driver{ + migrationsTableName: config.MigrationsTableName, + databaseURL: config.DatabaseURL, + log: config.Log, + } +} + +// ConnectionString converts a URL into a valid connection string +func ConnectionString(u *url.URL) string { + // duplicate URL and remove scheme + newURL := *u + newURL.Scheme = "" + + if newURL.Opaque == "" && newURL.Path != "" { + // When the DSN is in the form "scheme:/absolute/path" or + // "scheme://absolute/path" or "scheme:///absolute/path", url.Parse + // will consider the file path as : + // - "absolute" as the hostname + // - "path" (and the rest until "?") as the URL path. + // Instead, when the DSN is in the form "scheme:", the (relative) file + // path is stored in the "Opaque" field. + // See: https://pkg.go.dev/net/url#URL + // + // While Opaque is not escaped, the URL Path is. So, if .Path contains + // the file path, we need to un-escape it, and rebuild the full path. + + newURL.Opaque = "//" + newURL.Host + dbutil.MustUnescapePath(newURL.Path) + newURL.Path = "" + } + // trim duplicate leading slashes + str := regexp.MustCompile("^//+").ReplaceAllString(newURL.String(), "/") + + return str +} + +// Open creates a new database connection +func (drv *Driver) Open() (*sql.DB, error) { + return sql.Open("duckdb", ConnectionString(drv.databaseURL)) +} + +func (drv *Driver) CreateDatabase() error { + fmt.Fprintf(drv.log, "Creating: %s\n", ConnectionString(drv.databaseURL)) + + db, err := drv.Open() + if err != nil { + return err + } + defer dbutil.MustClose(db) + + return db.Ping() +} + +// DropDatabase drops the specified database (if it exists) +func (drv *Driver) DropDatabase() error { + path := ConnectionString(drv.databaseURL) + fmt.Fprintf(drv.log, "Dropping: %s\n", path) + + exists, err := drv.DatabaseExists() + if err != nil { + return err + } + if !exists { + return nil + } + + return os.Remove(path) +} + +func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { + migrationsTable := drv.quotedMigrationsTableName() // Use the quoted table name + // load applied migrations + migrations, err := dbutil.QueryColumn(db, + fmt.Sprintf("select format('{}', version) from %s order by version asc", migrationsTable)) + if err != nil { + return nil, err + } + + // build schema migrations table data + var buf bytes.Buffer + buf.WriteString("-- Dbmate schema migrations\n") + + if len(migrations) > 0 { + buf.WriteString( + fmt.Sprintf("INSERT INTO %s (version) VALUES\n ('", migrationsTable) + + strings.Join(migrations, "'),\n ('") + + "');\n") + } else { + return nil, nil + } + + return buf.Bytes(), nil +} + +// DumpSchema returns the current database schema +func (drv *Driver) DumpSchema(db *sql.DB) ([]byte, error) { + queryString := `SELECT sql FROM ( + SELECT COALESCE(sql, format('CREATE SCHEMA {};', schema_name)) AS sql from duckdb_schemas() where internal=false + UNION ALL + SELECT sql from duckdb_sequences() + UNION ALL + SELECT sql from duckdb_tables() where internal=false + UNION ALL + SELECT sql from duckdb_indexes() + UNION ALL + SELECT sql from duckdb_views() WHERE internal=false AND sql is not null + UNION ALL + SELECT macro_definition from duckdb_functions() WHERE internal=false and macro_definition is not null + ) WHERE sql IS NOT NULL; + ` + rows, err := db.Query(queryString) + if err != nil { + return nil, err + } + defer rows.Close() + + var schema []byte + + // Iterate over the rows and build the schema + for rows.Next() { + var sqlStmt string + if err := rows.Scan(&sqlStmt); err != nil { + return nil, err + } + // Append each SQL statement to the schema slice + schema = append(schema, []byte(sqlStmt+"\n")...) + } + + // Check for any errors encountered during iteration + if err := rows.Err(); err != nil { + return nil, err + } + + // Add any migrations to the schema + migrations, err := drv.schemaMigrationsDump(db) + if err != nil { + return nil, err + } + + // Append the migrations to the schema + schema = append(schema, migrations...) + + // Trim leading comments or unnecessary lines from the schema + return dbutil.TrimLeadingSQLComments(schema) +} + +// DatabaseExists determines whether the database exists +func (drv *Driver) DatabaseExists() (bool, error) { + _, err := os.Stat(ConnectionString(drv.databaseURL)) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, err + } + + return true, nil +} + +// MigrationsTableExists checks if the schema_migrations table exists +func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) { + exists := false + // TODO: Change this query. duckdb supports schemas and tables. + // May need to look at another drive to see how they handle this. + err := db.QueryRow("SELECT 1 FROM sqlite_master "+ + "WHERE type='table' AND name=$1", + drv.migrationsTableName). + Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + + return exists, err +} + +// CreateMigrationsTable creates the schema migrations table +func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { + _, err := db.Exec(fmt.Sprintf( + "create table if not exists %s (version varchar(128) primary key)", + drv.quotedMigrationsTableName())) + + return err +} + +// SelectMigrations returns a list of applied migrations +// with an optional limit (in descending order) +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { + query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName()) + if limit >= 0 { + query = fmt.Sprintf("%s limit %d", query, limit) + } + rows, err := db.Query(query) + if err != nil { + return nil, err + } + + defer dbutil.MustClose(rows) + + migrations := map[string]bool{} + for rows.Next() { + var version string + if err := rows.Scan(&version); err != nil { + return nil, err + } + + migrations[version] = true + } + + if err = rows.Err(); err != nil { + return nil, err + } + + return migrations, nil +} + +// InsertMigration adds a new migration record +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { + _, err := db.Exec( + fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), + version) + + return err +} + +// DeleteMigration removes a migration record +func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error { + _, err := db.Exec( + fmt.Sprintf("delete from %s where version = ?", drv.quotedMigrationsTableName()), + version) + + return err +} + +// Ping verifies a connection to the database. Due to the way DuckDB works, by +// testing whether the database is valid, it will automatically create the database +// if it does not already exist. +func (drv *Driver) Ping() error { + db, err := drv.Open() + if err != nil { + return err + } + defer dbutil.MustClose(db) + + return db.Ping() +} + +// Return a normalized version of the driver-specific error type. +func (drv *Driver) QueryError(query string, err error) error { + return &dbmate.QueryError{Err: err, Query: query} +} + +func (drv *Driver) quotedMigrationsTableName() string { + return drv.quoteIdentifier(drv.migrationsTableName) +} + +// quoteIdentifier quotes a table or column name +// we fall back to lib/pq implementation since both use ansi standard (double quotes) +// and mattn/go-duckdb doesn't provide a sqlite-specific equivalent +func (drv *Driver) quoteIdentifier(s string) string { + return pq.QuoteIdentifier(s) +} diff --git a/pkg/driver/duckdb/duckdb_test.go b/pkg/driver/duckdb/duckdb_test.go new file mode 100644 index 00000000..6dac82c8 --- /dev/null +++ b/pkg/driver/duckdb/duckdb_test.go @@ -0,0 +1,415 @@ +//go:build cgo +// +build cgo + +package duckdb + +import ( + "database/sql" + "os" + "strings" + "testing" + + "github.com/amacneil/dbmate/v2/pkg/dbmate" + "github.com/amacneil/dbmate/v2/pkg/dbtest" + "github.com/amacneil/dbmate/v2/pkg/dbutil" + + "github.com/stretchr/testify/require" +) + +func testDuckDBDriver(t *testing.T) *Driver { + u := dbtest.MustParseURL(t, "duckdb:dbmate_test.duckdb") + drv, err := dbmate.New(u).Driver() + require.NoError(t, err) + + return drv.(*Driver) +} + +func prepTestDuckDBDB(t *testing.T) *sql.DB { + drv := testDuckDBDriver(t) + + // drop any existing database + err := drv.DropDatabase() + require.NoError(t, err) + + // create database + err = drv.CreateDatabase() + require.NoError(t, err) + + // connect database + db, err := drv.Open() + require.NoError(t, err) + + return db +} + +func TestGetDriver(t *testing.T) { + db := dbmate.New(dbtest.MustParseURL(t, "DuckDB://")) + drvInterface, err := db.Driver() + require.NoError(t, err) + + // driver should have URL and default migrations table set + drv, ok := drvInterface.(*Driver) + require.True(t, ok) + require.Equal(t, db.DatabaseURL.String(), drv.databaseURL.String()) + require.Equal(t, "schema_migrations", drv.migrationsTableName) +} + +func TestConnectionString(t *testing.T) { + t.Run("relative", func(t *testing.T) { + u := dbtest.MustParseURL(t, "DuckDB:foo/bar.duckdb?mode=ro") + require.Equal(t, "foo/bar.duckdb?mode=ro", ConnectionString(u)) + }) + + t.Run("relative with dot", func(t *testing.T) { + u := dbtest.MustParseURL(t, "duckdb:./foo/bar.duckdb3?mode=ro") + require.Equal(t, "./foo/bar.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("relative with double dot", func(t *testing.T) { + u := dbtest.MustParseURL(t, "duckdb:../foo/bar.duckdb3?mode=ro") + require.Equal(t, "../foo/bar.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("absolute", func(t *testing.T) { + u := dbtest.MustParseURL(t, "duckdb:/tmp/foo.duckdb3?mode=ro") + require.Equal(t, "/tmp/foo.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("two slashes", func(t *testing.T) { + // interpreted as absolute path + u := dbtest.MustParseURL(t, "duckdb://tmp/foo.duckdb3?mode=ro") + require.Equal(t, "/tmp/foo.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("three slashes", func(t *testing.T) { + // interpreted as absolute path + u := dbtest.MustParseURL(t, "duckdb:///tmp/foo.duckdb3?mode=ro") + require.Equal(t, "/tmp/foo.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("four slashes", func(t *testing.T) { + // interpreted as absolute path + // supported for backwards compatibility + u := dbtest.MustParseURL(t, "duckdb:////tmp/foo.duckdb3?mode=ro") + require.Equal(t, "/tmp/foo.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("relative with space", func(t *testing.T) { + u := dbtest.MustParseURL(t, "duckdb:foo bar.duckdb3?mode=ro") + require.Equal(t, "foo bar.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("relative with space and dot", func(t *testing.T) { + u := dbtest.MustParseURL(t, "duckdb:./foo bar.duckdb3?mode=ro") + require.Equal(t, "./foo bar.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("relative with space and double dot", func(t *testing.T) { + u := dbtest.MustParseURL(t, "duckdb:../foo bar.duckdb3?mode=ro") + require.Equal(t, "../foo bar.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("absolute with space", func(t *testing.T) { + u := dbtest.MustParseURL(t, "duckdb:/foo bar.duckdb3?mode=ro") + require.Equal(t, "/foo bar.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("two slashes with space in path", func(t *testing.T) { + // interpreted as absolute path + u := dbtest.MustParseURL(t, "duckdb://tmp/foo bar.duckdb3?mode=ro") + require.Equal(t, "/tmp/foo bar.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("three slashes with space in path", func(t *testing.T) { + // interpreted as absolute path + u := dbtest.MustParseURL(t, "duckdb:///tmp/foo bar.duckdb3?mode=ro") + require.Equal(t, "/tmp/foo bar.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("three slashes with space in path (1st dir)", func(t *testing.T) { + // interpreted as absolute path + u := dbtest.MustParseURL(t, "duckdb:///tm p/foo bar.duckdb3?mode=ro") + require.Equal(t, "/tm p/foo bar.duckdb3?mode=ro", ConnectionString(u)) + }) + + t.Run("four slashes with space", func(t *testing.T) { + // interpreted as absolute path + // supported for backwards compatibility + u := dbtest.MustParseURL(t, "duckdb:////tmp/foo bar.duckdb3?mode=ro") + require.Equal(t, "/tmp/foo bar.duckdb3?mode=ro", ConnectionString(u)) + }) +} + +func TestDuckDBCreateDropDatabase(t *testing.T) { + drv := testDuckDBDriver(t) + path := ConnectionString(drv.databaseURL) + + // drop any existing database + err := drv.DropDatabase() + require.NoError(t, err) + + // create database + err = drv.CreateDatabase() + require.NoError(t, err) + + // check that database exists + _, err = os.Stat(path) + require.NoError(t, err) + + // drop the database + err = drv.DropDatabase() + require.NoError(t, err) + + // check that database no longer exists + _, err = os.Stat(path) + require.NotNil(t, err) + require.Equal(t, true, os.IsNotExist(err)) +} + +func TestDuckDBDumpSchema(t *testing.T) { + drv := testDuckDBDriver(t) + drv.migrationsTableName = "test_migrations" + + // prepare database + db := prepTestDuckDBDB(t) + defer dbutil.MustClose(db) + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + // insert migration + err = drv.InsertMigration(db, "abc1") + require.NoError(t, err) + err = drv.InsertMigration(db, "abc2") + require.NoError(t, err) + + // create a table + _, err = db.Exec("CREATE TABLE t (id INTEGER PRIMARY KEY)") + require.NoError(t, err) + + // create a schema + _, err = db.Exec("CREATE SCHEMA foo") + require.NoError(t, err) + + // Create a view + _, err = db.Exec("CREATE VIEW v AS SELECT * FROM t LIMIT 0") + require.NoError(t, err) + // Create a Sequence + _, err = db.Exec("CREATE SEQUENCE my_seq") + require.NoError(t, err) + + // DumpSchema should return schema + schema, err := drv.DumpSchema(db) + require.NoError(t, err) + + // CREATE SCHEMA foo + require.Contains(t, string(schema), "CREATE SCHEMA foo;") + require.Contains(t, string(schema), "CREATE TABLE t(id INTEGER PRIMARY KEY);") + require.Contains(t, string(schema), "CREATE TABLE test_migrations") + require.Contains(t, string(schema), "CREATE SEQUENCE my_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START 1 NO CYCLE") + require.Contains(t, string(schema), "CREATE VIEW v AS SELECT * FROM t LIMIT 0") + require.Contains(t, string(schema), "\n-- Dbmate schema migrations\n"+ + "INSERT INTO \"test_migrations\" (version) VALUES\n"+ + " ('abc1'),\n"+ + " ('abc2');\n") +} + +func TestDuckDBDatabaseExists(t *testing.T) { + drv := testDuckDBDriver(t) + + // drop any existing database + err := drv.DropDatabase() + require.NoError(t, err) + + // DatabaseExists should return false + exists, err := drv.DatabaseExists() + require.NoError(t, err) + require.Equal(t, false, exists) + + // create database + err = drv.CreateDatabase() + require.NoError(t, err) + + // DatabaseExists should return true + exists, err = drv.DatabaseExists() + require.NoError(t, err) + require.Equal(t, true, exists) +} + +func TestDuckDBCreateMigrationsTable(t *testing.T) { + t.Run("default table", func(t *testing.T) { + drv := testDuckDBDriver(t) + db := prepTestDuckDBDB(t) + defer dbutil.MustClose(db) + + // migrations table should not exist + count := 0 + err := db.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.Error(t, err) + require.True(t, strings.HasPrefix(err.Error(), "Catalog Error: Table with name schema_migrations does not exist!")) + + // create table + err = drv.CreateMigrationsTable(db) + require.NoError(t, err) + + // migrations table should exist + err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.NoError(t, err) + + // create table should be idempotent + err = drv.CreateMigrationsTable(db) + require.NoError(t, err) + }) + + t.Run("custom table", func(t *testing.T) { + drv := testDuckDBDriver(t) + drv.migrationsTableName = "test_migrations" + + db := prepTestDuckDBDB(t) + defer dbutil.MustClose(db) + + // migrations table should not exist + count := 0 + err := db.QueryRow("select count(*) from test_migrations").Scan(&count) + require.Error(t, err) + require.True(t, strings.HasPrefix(err.Error(), "Catalog Error: Table with name test_migrations does not exist!")) + + // create table + err = drv.CreateMigrationsTable(db) + require.NoError(t, err) + + // migrations table should exist + err = db.QueryRow("select count(*) from test_migrations").Scan(&count) + require.NoError(t, err) + + // create table should be idempotent + err = drv.CreateMigrationsTable(db) + require.NoError(t, err) + }) +} + +func TestDuckDBSelectMigrations(t *testing.T) { + drv := testDuckDBDriver(t) + drv.migrationsTableName = "test_migrations" + + db := prepTestDuckDBDB(t) + defer dbutil.MustClose(db) + + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + _, err = db.Exec(`insert into test_migrations (version) + values ('abc2'), ('abc1'), ('abc3')`) + require.NoError(t, err) + + migrations, err := drv.SelectMigrations(db, -1) + require.NoError(t, err) + require.Equal(t, true, migrations["abc1"]) + require.Equal(t, true, migrations["abc2"]) + require.Equal(t, true, migrations["abc2"]) + + // test limit param + migrations, err = drv.SelectMigrations(db, 1) + require.NoError(t, err) + require.Equal(t, true, migrations["abc3"]) + require.Equal(t, false, migrations["abc1"]) + require.Equal(t, false, migrations["abc2"]) +} + +func TestDuckDBInsertMigration(t *testing.T) { + drv := testDuckDBDriver(t) + drv.migrationsTableName = "test_migrations" + + db := prepTestDuckDBDB(t) + defer dbutil.MustClose(db) + + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + count := 0 + err = db.QueryRow("select count(*) from test_migrations").Scan(&count) + require.NoError(t, err) + require.Equal(t, 0, count) + + // insert migration + err = drv.InsertMigration(db, "abc1") + require.NoError(t, err) + + err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'"). + Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestDuckDBDeleteMigration(t *testing.T) { + drv := testDuckDBDriver(t) + drv.migrationsTableName = "test_migrations" + + db := prepTestDuckDBDB(t) + defer dbutil.MustClose(db) + + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + _, err = db.Exec(`insert into test_migrations (version) + values ('abc1'), ('abc2')`) + require.NoError(t, err) + + err = drv.DeleteMigration(db, "abc2") + require.NoError(t, err) + + count := 0 + err = db.QueryRow("select count(*) from test_migrations").Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestDuckDBPing(t *testing.T) { + drv := testDuckDBDriver(t) + path := ConnectionString(drv.databaseURL) + + // drop any existing database + err := drv.DropDatabase() + require.NoError(t, err) + + // ping database + err = drv.Ping() + require.NoError(t, err) + + // check that the database was created (sqlite-only behavior) + _, err = os.Stat(path) + require.NoError(t, err) + + // drop the database + err = drv.DropDatabase() + require.NoError(t, err) + + // create directory where database file is expected + err = os.Mkdir(path, 0755) + require.NoError(t, err) + defer func() { + err = os.RemoveAll(path) + require.NoError(t, err) + }() + + // ping database should fail + err = drv.Ping() + require.Error(t, err) + + require.Contains(t, err.Error(), "could not open database: duckdb error: IO Error: Could not read from file") +} + +func TestDuckDBQuotedMigrationsTableName(t *testing.T) { + t.Run("default name", func(t *testing.T) { + drv := testDuckDBDriver(t) + name := drv.quotedMigrationsTableName() + require.Equal(t, `"schema_migrations"`, name) + }) + + t.Run("custom name", func(t *testing.T) { + drv := testDuckDBDriver(t) + drv.migrationsTableName = "fooMigrations" + + name := drv.quotedMigrationsTableName() + require.Equal(t, `"fooMigrations"`, name) + }) +}