Moved Sqlite migration package into separate package

.forgejo/workflows/build.yaml

@@ -0,0 +1,10 @@
+on: [push]
+jobs:
+  test:
+    runs-on: docker
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '^1.22'
+      - run: go test .

db.go

@@ -0,0 +1,51 @@
+package migration
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"github.com/Southclaws/fault"
+	"io"
+)
+
+func (m *Migrator) runMigrationFile(ctx context.Context, tx *sql.Tx, filename string) error {
+	f, err := m.fs.Open(filename)
+	if err != nil {
+		return fault.Wrap(err)
+	}
+	defer f.Close()
+
+	bts, err := io.ReadAll(f)
+	if err != nil {
+		return fault.Wrap(err)
+	}
+
+	if _, err := tx.ExecContext(ctx, string(bts)); err != nil {
+		return fault.Wrap(err)
+	}
+	return nil
+}
+
+func (m *Migrator) getVersion(ctx context.Context, tx *sql.Tx) (userVersion int, err error) {
+	err = tx.QueryRowContext(ctx, `SELECT * FROM pragma_user_version;`).Scan(&userVersion)
+	return userVersion, err
+}
+
+func (m *Migrator) setVersion(ctx context.Context, tx *sql.Tx, userVersion int) (err error) {
+	_, err = tx.ExecContext(ctx, fmt.Sprintf("PRAGMA user_version = %v", userVersion))
+	return err
+}
+
+func (m *Migrator) inTX(ctx context.Context, txFn func(tx *sql.Tx) error) error {
+	tx, err := m.db.BeginTx(ctx, nil)
+	if err != nil {
+		return fault.Wrap(err)
+	}
+
+	if err := txFn(tx); err != nil {
+		_ = tx.Rollback()
+		return fault.Wrap(err)
+	}
+
+	return fault.Wrap(tx.Commit())
+}

go.mod

@@ -0,0 +1,30 @@
+module lmika.dev/pkg/litemigrate
+
+go 1.22.4
+
+require (
+	github.com/Southclaws/fault v0.8.1
+	github.com/lmika/blogging-tools v0.0.0-20240630114557-8db2b3aa93e6
+	github.com/lmika/gopkgs v0.0.0-20240408110817-a02f6fc67d1f
+	github.com/stretchr/testify v1.9.0
+	modernc.org/sqlite v1.33.1
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
+	modernc.org/libc v1.55.3 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.8.0 // indirect
+	modernc.org/strutil v1.2.0 // indirect
+	modernc.org/token v1.1.0 // indirect
+)

go.sum

@@ -0,0 +1,63 @@
+github.com/Southclaws/fault v0.8.1 h1:mgqqdC6kUBQ6ExMALZ0nNaDfNJD5h2+wq3se5mAyX+8=
+github.com/Southclaws/fault v0.8.1/go.mod h1:VUVkAWutC59SL16s6FTqf3I6I2z77RmnaW5XRz4bLOE=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
+github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/lmika/blogging-tools v0.0.0-20240630114557-8db2b3aa93e6 h1:WHM70gRzPTKHSKm/wf2kp3HErXzMpmcqfkGjHlhtu1Y=
+github.com/lmika/blogging-tools v0.0.0-20240630114557-8db2b3aa93e6/go.mod h1:w4rGqiE0+/FDUNWiIhfPGSPfT948/9Yw+cja12zOb4o=
+github.com/lmika/gopkgs v0.0.0-20240408110817-a02f6fc67d1f h1:tz68Lhc1oR15HVz69IGbtdukdH0x70kBDEvvj5pTXyE=
+github.com/lmika/gopkgs v0.0.0-20240408110817-a02f6fc67d1f/go.mod h1:zHQvhjGXRro/Xp2C9dbC+ZUpE0gL4GYW75x1lk7hwzI=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
+golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
+golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
+modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
+modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
+modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
+modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
+modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
+modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
+modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
+modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
+modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
+modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
+modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
+modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
+modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
+modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
+modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
+modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM=
+modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k=
+modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
+modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

migration.go

@@ -0,0 +1,125 @@
+package migration
+
+import (
+	"context"
+	"database/sql"
+	"github.com/Southclaws/fault"
+	"github.com/lmika/gopkgs/fp/maps"
+	"io/fs"
+	"log/slog"
+	"regexp"
+	"sort"
+	"strconv"
+)
+
+type Migrator struct {
+	fs fs.FS
+	db *sql.DB
+
+	preStepHooks []func(filename string)
+}
+
+func New(fs fs.FS, db *sql.DB, opts ...Option) *Migrator {
+	migrator := &Migrator{
+		db: db,
+		fs: fs,
+	}
+
+	for _, opt := range opts {
+		opt(migrator)
+	}
+	return migrator
+}
+
+func (m *Migrator) Version(ctx context.Context) (userVersion int, err error) {
+	if err := m.inTX(ctx, func(tx *sql.Tx) error {
+		userVersion, err = m.getVersion(ctx, tx)
+		return err
+	}); err != nil {
+		return 0, fault.Wrap(err)
+	}
+
+	return int(userVersion), nil
+}
+
+func (m *Migrator) MigrateUp(ctx context.Context) error {
+	sf, err := m.readFiles()
+	if err != nil {
+		return fault.Wrap(err)
+	} else if len(sf) == 0 {
+		return nil
+	}
+
+	return m.inTX(ctx, func(tx *sql.Tx) error {
+		latestVersion := sf[len(sf)-1].ver
+
+		currentVersion, err := m.getVersion(ctx, tx)
+		if err != nil {
+			return fault.Wrap(err)
+		}
+
+		if currentVersion == latestVersion {
+			slog.Debug("no DB migration necessary", "current_version", currentVersion, "latest_version", latestVersion)
+			return nil
+		}
+
+		slog.Debug("starting migration", "current_version", currentVersion, "latest_version", latestVersion)
+		for _, mf := range sf {
+			if mf.ver <= currentVersion {
+				continue
+			}
+
+			for _, h := range m.preStepHooks {
+				h(mf.upFile)
+			}
+
+			if err := m.runMigrationFile(ctx, tx, mf.upFile); err != nil {
+				return err
+			}
+		}
+
+		return m.setVersion(ctx, tx, latestVersion)
+	})
+}
+
+func (m *Migrator) readFiles() ([]schemaFiles, error) {
+	de, err := fs.ReadDir(m.fs, ".")
+	if err != nil {
+		return nil, fault.Wrap(err)
+	}
+
+	verFiles := make(map[int]schemaFiles)
+	for _, f := range de {
+		parts := migrateFileName.FindStringSubmatch(f.Name())
+		if len(parts) != 3 {
+			continue
+		}
+
+		versionID, err := strconv.Atoi(parts[1])
+		if err != nil {
+			continue
+		}
+
+		vf := verFiles[versionID]
+		vf.ver = versionID
+		if parts[2] == "up" {
+			vf.upFile = f.Name()
+		} else {
+			vf.downFile = f.Name()
+		}
+
+		verFiles[versionID] = vf
+	}
+
+	files := maps.Values(verFiles)
+	sort.Slice(files, func(i, j int) bool { return files[i].ver < files[j].ver })
+	return files, nil
+}
+
+var migrateFileName = regexp.MustCompile(`0*([0-9]+)[-_].*\.(up|down)\.sql`)
+
+type schemaFiles struct {
+	ver      int
+	upFile   string
+	downFile string
+}

migration_test.go

@@ -0,0 +1,90 @@
+package migration_test
+
+import (
+	"context"
+	"database/sql"
+	"github.com/lmika/blogging-tools/providers/db/migration"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"path/filepath"
+	"testing"
+	"testing/fstest"
+
+	_ "modernc.org/sqlite"
+)
+
+func TestMigrator_MigrateUp(t *testing.T) {
+	ctx := context.Background()
+
+	db, closeFn := newDB(t)
+	defer closeFn()
+
+	fileMigrated := make([]string, 0)
+	migrator := migration.New(testMigration, db, migration.WithPreStepHook(func(filename string) {
+		fileMigrated = append(fileMigrated, filename)
+	}))
+
+	// Check initial version
+	ver, err := migrator.Version(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, ver)
+
+	// Run migration
+	err = migrator.MigrateUp(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"01_base_schema.up.sql", "02_insert_things.up.sql"}, fileMigrated)
+
+	// Check new version
+	ver, err = migrator.Version(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, ver)
+
+	// Verify that migration worked
+	var some string
+	row := db.QueryRow(`SELECT some FROM thing LIMIT 1`)
+	err = row.Scan(&some)
+	assert.NoError(t, err)
+	assert.Equal(t, "bla", some)
+
+	fileMigrated = make([]string, 0)
+
+	// Check that running migration nops
+	err = migrator.MigrateUp(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, []string{}, fileMigrated)
+
+	ver, err = migrator.Version(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, ver)
+}
+
+func newDB(t *testing.T) (*sql.DB, func()) {
+	t.Helper()
+
+	tmpDir, err := os.MkdirTemp("", "")
+	if err != nil {
+		t.Errorf("cannot create mkdir: %v", err)
+	}
+
+	drvr, err := sql.Open("sqlite", filepath.Join(tmpDir, "test.db"))
+	if err != nil {
+		t.Errorf("cannot open driver: %v", err)
+	}
+
+	return drvr, func() {
+		drvr.Close()
+	}
+}
+
+var testMigration = fstest.MapFS{
+	"01_base_schema.up.sql": {
+		Data: []byte(`
+			CREATE TABLE thing (some text);
+        `),
+	},
+	"02_insert_things.up.sql": {
+		Data: []byte(`
+			INSERT INTO thing (some) VALUES ("bla");
+        `),
+	},
+}

opts.go

@@ -0,0 +1,9 @@
+package migration
+
+type Option func(m *Migrator)
+
+func WithPreStepHook(fn func(filename string)) Option {
+	return func(m *Migrator) {
+		m.preStepHooks = append(m.preStepHooks, fn)
+	}
+}