From 5f435b744015453a938d33bd03785dea1325e6f4 Mon Sep 17 00:00:00 2001 From: Erik Winter Date: Sun, 8 Sep 2024 11:17:49 +0200 Subject: [PATCH] sqlite repo --- .gitignore | 1 + go.mod | 12 +++ go.sum | 25 ++++++ sync-service/handler.go | 2 + sync-service/handler_test.go | 41 --------- sync-service/main.go | 10 ++- sync-service/planner.go | 7 ++ sync-service/sqlite.go | 158 +++++++++++++++++++++++++++++++++++ 8 files changed, 213 insertions(+), 43 deletions(-) create mode 100644 .gitignore create mode 100644 sync-service/sqlite.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e7e12b7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +test.db* diff --git a/go.mod b/go.mod index d4e3a15..a18923b 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,18 @@ module code.ewintr.nl/planner go 1.21.5 require ( + github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.22.0 // indirect + modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/sqlite v1.33.0 // indirect + modernc.org/strutil v1.2.0 // indirect + modernc.org/token v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index 493949e..a4ebb5e 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,29 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/sqlite v1.33.0 h1:WWkA/T2G17okiLGgKAj4/RMIvgyMT19yQ038160IeYk= +modernc.org/sqlite v1.33.0/go.mod h1:9uQ9hF/pCZoYZK73D/ud5Z7cIRIILSZI8NdIemVMTX8= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/sync-service/handler.go b/sync-service/handler.go index 966c98a..f3043e5 100644 --- a/sync-service/handler.go +++ b/sync-service/handler.go @@ -74,6 +74,7 @@ func (s *Server) SyncGet(w http.ResponseWriter, r *http.Request) { } fmt.Fprint(w, string(body)) + s.logger.Info("served get sync") } func (s *Server) SyncPost(w http.ResponseWriter, r *http.Request) { @@ -99,6 +100,7 @@ func (s *Server) SyncPost(w http.ResponseWriter, r *http.Request) { } w.WriteHeader(http.StatusNoContent) + s.logger.Info("served get sync") } // ShiftPath splits off the first component of p, which will be cleaned of diff --git a/sync-service/handler_test.go b/sync-service/handler_test.go index e324312..583d225 100644 --- a/sync-service/handler_test.go +++ b/sync-service/handler_test.go @@ -192,44 +192,3 @@ func TestSyncPost(t *testing.T) { }) } } - -/* -func TestSyncHandler(t *testing.T) { - t.Parallel() - - for _, tc := range []struct { - name string - items []item - method string - body .ChangeSummary - expStatus int - expBody .ChangeSummary - }{ - { - name: "empty", - expStatus: http.StatusOK, - }, - { - name: "full sync", - }, - } { - t.Run(tc.name, func(t *testing.T) { - mem := NewMemory() - for _, i := range tc.items { - mem.Update(i) - } - sh := .NewSyncHandler(mem) - req, err := http.NewRequest(tc.method, "/sync", nil) - if err != nil { - t.Errorf("exp nil, got %v", err) - } - rec := httptest.NewRecorder() - sh(rec, req) - res := rec.Result() - if res.StatusCode != tc.expStatus { - t.Errorf("exp %d, got %d", tc.expStatus, res.StatusCode) - } - }) - } -} -*/ diff --git a/sync-service/main.go b/sync-service/main.go index 365e71b..4004000 100644 --- a/sync-service/main.go +++ b/sync-service/main.go @@ -22,10 +22,16 @@ func main() { os.Exit(1) } - mem := NewMemory() + //mem := NewMemory() + repo, err := NewSqlite("test.db") + if err != nil { + fmt.Printf("could not open sqlite db: %s", err.Error()) + os.Exit(1) + } + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) - go http.ListenAndServe(fmt.Sprintf(":%d", port), NewServer(mem, apiKey, logger)) + go http.ListenAndServe(fmt.Sprintf(":%d", port), NewServer(repo, apiKey, logger)) logger.Info("service started") diff --git a/sync-service/planner.go b/sync-service/planner.go index 533fcae..778fe68 100644 --- a/sync-service/planner.go +++ b/sync-service/planner.go @@ -6,8 +6,15 @@ import ( "github.com/google/uuid" ) +type Kind string + +const ( + KindTask Kind = "task" +) + type Syncable struct { ID string + Kind Kind Updated time.Time Deleted bool Item string diff --git a/sync-service/sqlite.go b/sync-service/sqlite.go new file mode 100644 index 0000000..b6be41b --- /dev/null +++ b/sync-service/sqlite.go @@ -0,0 +1,158 @@ +package main + +import ( + "database/sql" + "errors" + "fmt" + "time" + + _ "modernc.org/sqlite" +) + +const ( + timestampFormat = "2006-01-02 15:04:05" +) + +var migrations = []string{ + `CREATE TABLE items ("id" TEXT UNIQUE, "kind" TEXT, "updated" TIMESTAMP, "body" TEXT)`, + `PRAGMA journal_mode=WAL`, + `PRAGMA synchronous=NORMAL`, + `PRAGMA cache_size=2000`, +} + +var ( + ErrInvalidConfiguration = errors.New("invalid configuration") + ErrIncompatibleSQLMigration = errors.New("incompatible migration") + ErrNotEnoughSQLMigrations = errors.New("already more migrations than wanted") + ErrSqliteFailure = errors.New("sqlite returned an error") +) + +type Sqlite struct { + db *sql.DB +} + +func NewSqlite(dbPath string) (*Sqlite, error) { + db, err := sql.Open("sqlite", dbPath) + if err != nil { + return &Sqlite{}, fmt.Errorf("%w: %v", ErrInvalidConfiguration, err) + } + + s := &Sqlite{ + db: db, + } + + if err := s.migrate(migrations); err != nil { + return &Sqlite{}, err + } + + return s, nil +} + +func (s *Sqlite) Update(item Syncable) error { + if _, err := s.db.Exec(` +INSERT INTO items +(id, kind, updated, body) +VALUES +(?, ?, ?, ?) +ON CONFLICT(id) DO UPDATE +SET +kind=?, +updated=?, +body=?`, + item.ID, item.Kind, item.Updated.Format(timestampFormat), item.Item, + item.Kind, item.Updated.Format(timestampFormat), item.Item); err != nil { + return fmt.Errorf("%w: %v", ErrSqliteFailure, err) + } + return nil +} + +func (s *Sqlite) Updated(t time.Time) ([]Syncable, error) { + rows, err := s.db.Query(` +SELECT id, kind, updated, body +FROM items +WHERE updated > ?`, t.Format(timestampFormat)) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrSqliteFailure, err) + } + + result := make([]Syncable, 0) + defer rows.Close() + for rows.Next() { + var item Syncable + if err := rows.Scan(&item.ID, &item.Kind, &item.Updated, &item.Item); err != nil { + return nil, fmt.Errorf("%w: %v", ErrSqliteFailure, err) + } + result = append(result, item) + } + + return result, nil +} + +func (s *Sqlite) migrate(wanted []string) error { + // admin table + if _, err := s.db.Exec(` +CREATE TABLE IF NOT EXISTS migration +("id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, "query" TEXT) +`); err != nil { + return err + } + + // find existing + rows, err := s.db.Query(`SELECT query FROM migration ORDER BY id`) + if err != nil { + return fmt.Errorf("%w: %v", ErrSqliteFailure, err) + } + + existing := []string{} + for rows.Next() { + var query string + if err := rows.Scan(&query); err != nil { + return fmt.Errorf("%w: %v", ErrSqliteFailure, err) + } + existing = append(existing, string(query)) + } + rows.Close() + + // compare + missing, err := compareMigrations(wanted, existing) + if err != nil { + return fmt.Errorf("%w: %v", ErrSqliteFailure, err) + } + + // execute missing + for _, query := range missing { + if _, err := s.db.Exec(string(query)); err != nil { + return fmt.Errorf("%w: %v", ErrSqliteFailure, err) + } + + // register + if _, err := s.db.Exec(` +INSERT INTO migration +(query) VALUES (?) +`, query); err != nil { + return fmt.Errorf("%w: %v", ErrSqliteFailure, err) + } + } + + return nil +} + +func compareMigrations(wanted, existing []string) ([]string, error) { + needed := []string{} + if len(wanted) < len(existing) { + return []string{}, ErrNotEnoughSQLMigrations + } + + for i, want := range wanted { + switch { + case i >= len(existing): + needed = append(needed, want) + case want == existing[i]: + // do nothing + case want != existing[i]: + return []string{}, fmt.Errorf("%w: %v", ErrIncompatibleSQLMigration, want) + } + } + + return needed, nil +}