From 6d8ba4295997e07fc782301a6e8e37b0fc0c3b28 Mon Sep 17 00:00:00 2001 From: Leon Mika Date: Sat, 28 Sep 2024 09:04:07 +1000 Subject: [PATCH] Moved Sqlite migration package into separate package --- .forgejo/workflows/build.yaml | 10 +++ db.go | 51 ++++++++++++++ go.mod | 30 ++++++++ go.sum | 63 +++++++++++++++++ migration.go | 125 ++++++++++++++++++++++++++++++++++ migration_test.go | 90 ++++++++++++++++++++++++ opts.go | 9 +++ 7 files changed, 378 insertions(+) create mode 100644 .forgejo/workflows/build.yaml create mode 100644 db.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 migration.go create mode 100644 migration_test.go create mode 100644 opts.go diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml new file mode 100644 index 0000000..507b960 --- /dev/null +++ b/.forgejo/workflows/build.yaml @@ -0,0 +1,10 @@ +on: [push] +jobs: + test: + runs-on: docker + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v5 + with: + go-version: '^1.22' + - run: go test . diff --git a/db.go b/db.go new file mode 100644 index 0000000..986f0d4 --- /dev/null +++ b/db.go @@ -0,0 +1,51 @@ +package migration + +import ( + "context" + "database/sql" + "fmt" + "github.com/Southclaws/fault" + "io" +) + +func (m *Migrator) runMigrationFile(ctx context.Context, tx *sql.Tx, filename string) error { + f, err := m.fs.Open(filename) + if err != nil { + return fault.Wrap(err) + } + defer f.Close() + + bts, err := io.ReadAll(f) + if err != nil { + return fault.Wrap(err) + } + + if _, err := tx.ExecContext(ctx, string(bts)); err != nil { + return fault.Wrap(err) + } + return nil +} + +func (m *Migrator) getVersion(ctx context.Context, tx *sql.Tx) (userVersion int, err error) { + err = tx.QueryRowContext(ctx, `SELECT * FROM pragma_user_version;`).Scan(&userVersion) + return userVersion, err +} + +func (m *Migrator) setVersion(ctx context.Context, tx *sql.Tx, userVersion int) (err error) { + _, err = tx.ExecContext(ctx, fmt.Sprintf("PRAGMA user_version = %v", userVersion)) + return err +} + +func (m *Migrator) inTX(ctx context.Context, txFn func(tx *sql.Tx) error) error { + tx, err := m.db.BeginTx(ctx, nil) + if err != nil { + return fault.Wrap(err) + } + + if err := txFn(tx); err != nil { + _ = tx.Rollback() + return fault.Wrap(err) + } + + return fault.Wrap(tx.Commit()) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..38a2c1c --- /dev/null +++ b/go.mod @@ -0,0 +1,30 @@ +module lmika.dev/pkg/litemigrate + +go 1.22.4 + +require ( + github.com/Southclaws/fault v0.8.1 + github.com/lmika/blogging-tools v0.0.0-20240630114557-8db2b3aa93e6 + github.com/lmika/gopkgs v0.0.0-20240408110817-a02f6fc67d1f + github.com/stretchr/testify v1.9.0 + modernc.org/sqlite v1.33.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.22.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/strutil v1.2.0 // indirect + modernc.org/token v1.1.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a278bba --- /dev/null +++ b/go.sum @@ -0,0 +1,63 @@ +github.com/Southclaws/fault v0.8.1 h1:mgqqdC6kUBQ6ExMALZ0nNaDfNJD5h2+wq3se5mAyX+8= +github.com/Southclaws/fault v0.8.1/go.mod h1:VUVkAWutC59SL16s6FTqf3I6I2z77RmnaW5XRz4bLOE= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/lmika/blogging-tools v0.0.0-20240630114557-8db2b3aa93e6 h1:WHM70gRzPTKHSKm/wf2kp3HErXzMpmcqfkGjHlhtu1Y= +github.com/lmika/blogging-tools v0.0.0-20240630114557-8db2b3aa93e6/go.mod h1:w4rGqiE0+/FDUNWiIhfPGSPfT948/9Yw+cja12zOb4o= +github.com/lmika/gopkgs v0.0.0-20240408110817-a02f6fc67d1f h1:tz68Lhc1oR15HVz69IGbtdukdH0x70kBDEvvj5pTXyE= +github.com/lmika/gopkgs v0.0.0-20240408110817-a02f6fc67d1f/go.mod h1:zHQvhjGXRro/Xp2C9dbC+ZUpE0gL4GYW75x1lk7hwzI= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM= +modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/migration.go b/migration.go new file mode 100644 index 0000000..af061bc --- /dev/null +++ b/migration.go @@ -0,0 +1,125 @@ +package migration + +import ( + "context" + "database/sql" + "github.com/Southclaws/fault" + "github.com/lmika/gopkgs/fp/maps" + "io/fs" + "log/slog" + "regexp" + "sort" + "strconv" +) + +type Migrator struct { + fs fs.FS + db *sql.DB + + preStepHooks []func(filename string) +} + +func New(fs fs.FS, db *sql.DB, opts ...Option) *Migrator { + migrator := &Migrator{ + db: db, + fs: fs, + } + + for _, opt := range opts { + opt(migrator) + } + return migrator +} + +func (m *Migrator) Version(ctx context.Context) (userVersion int, err error) { + if err := m.inTX(ctx, func(tx *sql.Tx) error { + userVersion, err = m.getVersion(ctx, tx) + return err + }); err != nil { + return 0, fault.Wrap(err) + } + + return int(userVersion), nil +} + +func (m *Migrator) MigrateUp(ctx context.Context) error { + sf, err := m.readFiles() + if err != nil { + return fault.Wrap(err) + } else if len(sf) == 0 { + return nil + } + + return m.inTX(ctx, func(tx *sql.Tx) error { + latestVersion := sf[len(sf)-1].ver + + currentVersion, err := m.getVersion(ctx, tx) + if err != nil { + return fault.Wrap(err) + } + + if currentVersion == latestVersion { + slog.Debug("no DB migration necessary", "current_version", currentVersion, "latest_version", latestVersion) + return nil + } + + slog.Debug("starting migration", "current_version", currentVersion, "latest_version", latestVersion) + for _, mf := range sf { + if mf.ver <= currentVersion { + continue + } + + for _, h := range m.preStepHooks { + h(mf.upFile) + } + + if err := m.runMigrationFile(ctx, tx, mf.upFile); err != nil { + return err + } + } + + return m.setVersion(ctx, tx, latestVersion) + }) +} + +func (m *Migrator) readFiles() ([]schemaFiles, error) { + de, err := fs.ReadDir(m.fs, ".") + if err != nil { + return nil, fault.Wrap(err) + } + + verFiles := make(map[int]schemaFiles) + for _, f := range de { + parts := migrateFileName.FindStringSubmatch(f.Name()) + if len(parts) != 3 { + continue + } + + versionID, err := strconv.Atoi(parts[1]) + if err != nil { + continue + } + + vf := verFiles[versionID] + vf.ver = versionID + if parts[2] == "up" { + vf.upFile = f.Name() + } else { + vf.downFile = f.Name() + } + + verFiles[versionID] = vf + } + + files := maps.Values(verFiles) + sort.Slice(files, func(i, j int) bool { return files[i].ver < files[j].ver }) + return files, nil +} + +var migrateFileName = regexp.MustCompile(`0*([0-9]+)[-_].*\.(up|down)\.sql`) + +type schemaFiles struct { + ver int + upFile string + downFile string +} diff --git a/migration_test.go b/migration_test.go new file mode 100644 index 0000000..98db1bf --- /dev/null +++ b/migration_test.go @@ -0,0 +1,90 @@ +package migration_test + +import ( + "context" + "database/sql" + "github.com/lmika/blogging-tools/providers/db/migration" + "github.com/stretchr/testify/assert" + "os" + "path/filepath" + "testing" + "testing/fstest" + + _ "modernc.org/sqlite" +) + +func TestMigrator_MigrateUp(t *testing.T) { + ctx := context.Background() + + db, closeFn := newDB(t) + defer closeFn() + + fileMigrated := make([]string, 0) + migrator := migration.New(testMigration, db, migration.WithPreStepHook(func(filename string) { + fileMigrated = append(fileMigrated, filename) + })) + + // Check initial version + ver, err := migrator.Version(ctx) + assert.NoError(t, err) + assert.Equal(t, 0, ver) + + // Run migration + err = migrator.MigrateUp(ctx) + assert.NoError(t, err) + assert.Equal(t, []string{"01_base_schema.up.sql", "02_insert_things.up.sql"}, fileMigrated) + + // Check new version + ver, err = migrator.Version(ctx) + assert.NoError(t, err) + assert.Equal(t, 2, ver) + + // Verify that migration worked + var some string + row := db.QueryRow(`SELECT some FROM thing LIMIT 1`) + err = row.Scan(&some) + assert.NoError(t, err) + assert.Equal(t, "bla", some) + + fileMigrated = make([]string, 0) + + // Check that running migration nops + err = migrator.MigrateUp(ctx) + assert.NoError(t, err) + assert.Equal(t, []string{}, fileMigrated) + + ver, err = migrator.Version(ctx) + assert.NoError(t, err) + assert.Equal(t, 2, ver) +} + +func newDB(t *testing.T) (*sql.DB, func()) { + t.Helper() + + tmpDir, err := os.MkdirTemp("", "") + if err != nil { + t.Errorf("cannot create mkdir: %v", err) + } + + drvr, err := sql.Open("sqlite", filepath.Join(tmpDir, "test.db")) + if err != nil { + t.Errorf("cannot open driver: %v", err) + } + + return drvr, func() { + drvr.Close() + } +} + +var testMigration = fstest.MapFS{ + "01_base_schema.up.sql": { + Data: []byte(` + CREATE TABLE thing (some text); + `), + }, + "02_insert_things.up.sql": { + Data: []byte(` + INSERT INTO thing (some) VALUES ("bla"); + `), + }, +} diff --git a/opts.go b/opts.go new file mode 100644 index 0000000..04f2e6c --- /dev/null +++ b/opts.go @@ -0,0 +1,9 @@ +package migration + +type Option func(m *Migrator) + +func WithPreStepHook(fn func(filename string)) Option { + return func(m *Migrator) { + m.preStepHooks = append(m.preStepHooks, fn) + } +}