feat: duckdb driver

Dockerfile

@@ -14,12 +14,18 @@ RUN apt-get update \
     sqlite3 \
     nodejs \
     npm \
+    unzip \
   && rm -rf /var/lib/apt/lists/*
 
 # golangci-lint
 RUN curl -fsSL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \
   | sh -s -- -b /usr/local/bin v1.60.1
 
+# Download and install DuckDB
+RUN curl -fsSL https://github.com/duckdb/duckdb/releases/download/v1.1.0/duckdb_cli-linux-amd64.zip -o duckdb_cli-linux-amd64.zip \
+  && unzip duckdb_cli-linux-amd64.zip \
+  && mv duckdb /usr/local/bin/duckdb
+
 # download modules
 COPY go.* /src/
 RUN go mod download

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/joho/godotenv v1.5.1
 	github.com/lib/pq v1.10.9
+	github.com/marcboeker/go-duckdb v1.8.0
 	github.com/mattn/go-sqlite3 v1.14.23
 	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.27.4
@@ -28,6 +29,7 @@ require (
 	github.com/ClickHouse/ch-go v0.62.0 // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/apache/arrow/go/v15 v15.0.2 // indirect
+	github.com/apache/arrow/go/v17 v17.0.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -46,6 +48,7 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/paulmach/orb v0.11.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect

go.sum

@@ -201,6 +201,8 @@ github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer5
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE=
 github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA=
+github.com/apache/arrow/go/v17 v17.0.0 h1:RRR2bdqKcdbss9Gxy2NS/hK8i4LDMh23L6BbkN5+F54=
+github.com/apache/arrow/go/v17 v17.0.0/go.mod h1:jR7QHkODl15PfYyjM2nU+yTLScZ/qfj7OSUZmJ8putc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -384,8 +386,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/marcboeker/go-duckdb v1.8.0 h1:iOWv1wTL0JIMqpyns6hCf5XJJI4fY6lmJNk+itx5RRo=
+github.com/marcboeker/go-duckdb v1.8.0/go.mod h1:2oV8BZv88S16TKGKM+Lwd0g7DX84x0jMxjTInThC8Is=
 github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0=
 github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
@@ -762,8 +768,8 @@ golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNq
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
-gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o=
-gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY=
+gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ=
+gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=

main_cgo.go

@@ -4,5 +4,6 @@
 package main
 
 import (
+	_ "github.com/amacneil/dbmate/v2/pkg/driver/duckdb"
 	_ "github.com/amacneil/dbmate/v2/pkg/driver/sqlite"
 )

pkg/driver/duckdb/duckdb.go

@@ -0,0 +1,301 @@
+//go:build cgo
+// +build cgo
+
+// This driver is HEAVILY based on the sqlite driver
+// Even many of the comments are applicable.
+
+// TODO Features:
+// - Add support for schema names, sqlite base implementation doesn't have them, duckdb does.
+// 		- See postgres driver for how to do this.
+// - Ensure support of non-table objects (views, macros, etc.)
+
+package duckdb
+
+import (
+	"bytes"
+	"database/sql"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/amacneil/dbmate/v2/pkg/dbmate"
+	"github.com/amacneil/dbmate/v2/pkg/dbutil"
+
+	"github.com/lib/pq"
+	_ "github.com/marcboeker/go-duckdb" // database/sql driver
+)
+
+func init() {
+	dbmate.RegisterDriver(NewDriver, "duckdb")
+}
+
+type Driver struct {
+	migrationsTableName string
+	databaseURL         *url.URL
+	log                 io.Writer
+}
+
+func NewDriver(config dbmate.DriverConfig) dbmate.Driver {
+	return &Driver{
+		migrationsTableName: config.MigrationsTableName,
+		databaseURL:         config.DatabaseURL,
+		log:                 config.Log,
+	}
+}
+
+// ConnectionString converts a URL into a valid connection string
+func ConnectionString(u *url.URL) string {
+	// duplicate URL and remove scheme
+	newURL := *u
+	newURL.Scheme = ""
+
+	if newURL.Opaque == "" && newURL.Path != "" {
+		// When the DSN is in the form "scheme:/absolute/path" or
+		// "scheme://absolute/path" or "scheme:///absolute/path", url.Parse
+		// will consider the file path as :
+		// - "absolute" as the hostname
+		// - "path" (and the rest until "?") as the URL path.
+		// Instead, when the DSN is in the form "scheme:", the (relative) file
+		// path is stored in the "Opaque" field.
+		// See: https://pkg.go.dev/net/url#URL
+		//
+		// While Opaque is not escaped, the URL Path is. So, if .Path contains
+		// the file path, we need to un-escape it, and rebuild the full path.
+
+		newURL.Opaque = "//" + newURL.Host + dbutil.MustUnescapePath(newURL.Path)
+		newURL.Path = ""
+	}
+	// trim duplicate leading slashes
+	str := regexp.MustCompile("^//+").ReplaceAllString(newURL.String(), "/")
+
+	return str
+}
+
+// Open creates a new database connection
+func (drv *Driver) Open() (*sql.DB, error) {
+	return sql.Open("duckdb", ConnectionString(drv.databaseURL))
+}
+
+func (drv *Driver) CreateDatabase() error {
+	fmt.Fprintf(drv.log, "Creating: %s\n", ConnectionString(drv.databaseURL))
+
+	db, err := drv.Open()
+	if err != nil {
+		return err
+	}
+	defer dbutil.MustClose(db)
+
+	return db.Ping()
+}
+
+// DropDatabase drops the specified database (if it exists)
+func (drv *Driver) DropDatabase() error {
+	path := ConnectionString(drv.databaseURL)
+	fmt.Fprintf(drv.log, "Dropping: %s\n", path)
+
+	exists, err := drv.DatabaseExists()
+	if err != nil {
+		return err
+	}
+	if !exists {
+		return nil
+	}
+
+	return os.Remove(path)
+}
+
+func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) {
+	migrationsTable := drv.quotedMigrationsTableName() // Use the quoted table name
+	// load applied migrations
+	migrations, err := dbutil.QueryColumn(db,
+		fmt.Sprintf("select format('{}', version) from %s order by version asc", migrationsTable))
+	if err != nil {
+		return nil, err
+	}
+
+	// build schema migrations table data
+	var buf bytes.Buffer
+	buf.WriteString("-- Dbmate schema migrations\n")
+
+	if len(migrations) > 0 {
+		buf.WriteString(
+			fmt.Sprintf("INSERT INTO %s (version) VALUES\n  ('", migrationsTable) +
+				strings.Join(migrations, "'),\n  ('") +
+				"');\n")
+	} else {
+		return nil, nil
+	}
+
+	return buf.Bytes(), nil
+}
+
+// DumpSchema returns the current database schema
+func (drv *Driver) DumpSchema(db *sql.DB) ([]byte, error) {
+	queryString := `SELECT sql FROM (
+	SELECT COALESCE(sql, format('CREATE SCHEMA {};', schema_name)) AS sql from duckdb_schemas() where internal=false
+	UNION ALL
+	SELECT sql from duckdb_sequences()
+	UNION ALL
+	SELECT sql from duckdb_tables() where internal=false
+	UNION ALL
+	SELECT sql from duckdb_indexes()
+	UNION ALL
+	SELECT sql from duckdb_views() WHERE internal=false AND sql is not null
+	UNION ALL
+	SELECT macro_definition from duckdb_functions() WHERE internal=false and macro_definition is not null
+	) WHERE sql IS NOT NULL;
+	`
+	rows, err := db.Query(queryString)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var schema []byte
+
+	// Iterate over the rows and build the schema
+	for rows.Next() {
+		var sqlStmt string
+		if err := rows.Scan(&sqlStmt); err != nil {
+			return nil, err
+		}
+		// Append each SQL statement to the schema slice
+		schema = append(schema, []byte(sqlStmt+"\n")...)
+	}
+
+	// Check for any errors encountered during iteration
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	// Add any migrations to the schema
+	migrations, err := drv.schemaMigrationsDump(db)
+	if err != nil {
+		return nil, err
+	}
+
+	// Append the migrations to the schema
+	schema = append(schema, migrations...)
+
+	// Trim leading comments or unnecessary lines from the schema
+	return dbutil.TrimLeadingSQLComments(schema)
+}
+
+// DatabaseExists determines whether the database exists
+func (drv *Driver) DatabaseExists() (bool, error) {
+	_, err := os.Stat(ConnectionString(drv.databaseURL))
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
+// MigrationsTableExists checks if the schema_migrations table exists
+func (drv *Driver) MigrationsTableExists(db *sql.DB) (bool, error) {
+	exists := false
+	// TODO: Change this query. duckdb supports schemas and tables.
+	// May need to look at another drive to see how they handle this.
+	err := db.QueryRow("SELECT 1 FROM sqlite_master "+
+		"WHERE type='table' AND name=$1",
+		drv.migrationsTableName).
+		Scan(&exists)
+	if err == sql.ErrNoRows {
+		return false, nil
+	}
+
+	return exists, err
+}
+
+// CreateMigrationsTable creates the schema migrations table
+func (drv *Driver) CreateMigrationsTable(db *sql.DB) error {
+	_, err := db.Exec(fmt.Sprintf(
+		"create table if not exists %s (version varchar(128) primary key)",
+		drv.quotedMigrationsTableName()))
+
+	return err
+}
+
+// SelectMigrations returns a list of applied migrations
+// with an optional limit (in descending order)
+func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) {
+	query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName())
+	if limit >= 0 {
+		query = fmt.Sprintf("%s limit %d", query, limit)
+	}
+	rows, err := db.Query(query)
+	if err != nil {
+		return nil, err
+	}
+
+	defer dbutil.MustClose(rows)
+
+	migrations := map[string]bool{}
+	for rows.Next() {
+		var version string
+		if err := rows.Scan(&version); err != nil {
+			return nil, err
+		}
+
+		migrations[version] = true
+	}
+
+	if err = rows.Err(); err != nil {
+		return nil, err
+	}
+
+	return migrations, nil
+}
+
+// InsertMigration adds a new migration record
+func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error {
+	_, err := db.Exec(
+		fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()),
+		version)
+
+	return err
+}
+
+// DeleteMigration removes a migration record
+func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error {
+	_, err := db.Exec(
+		fmt.Sprintf("delete from %s where version = ?", drv.quotedMigrationsTableName()),
+		version)
+
+	return err
+}
+
+// Ping verifies a connection to the database. Due to the way DuckDB works, by
+// testing whether the database is valid, it will automatically create the database
+// if it does not already exist.
+func (drv *Driver) Ping() error {
+	db, err := drv.Open()
+	if err != nil {
+		return err
+	}
+	defer dbutil.MustClose(db)
+
+	return db.Ping()
+}
+
+// Return a normalized version of the driver-specific error type.
+func (drv *Driver) QueryError(query string, err error) error {
+	return &dbmate.QueryError{Err: err, Query: query}
+}
+
+func (drv *Driver) quotedMigrationsTableName() string {
+	return drv.quoteIdentifier(drv.migrationsTableName)
+}
+
+// quoteIdentifier quotes a table or column name
+// we fall back to lib/pq implementation since both use ansi standard (double quotes)
+// and mattn/go-duckdb doesn't provide a sqlite-specific equivalent
+func (drv *Driver) quoteIdentifier(s string) string {
+	return pq.QuoteIdentifier(s)
+}