Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix(francis): keep SQLite tuning on the actor store and stop fsyncing on commit (#4372)
Kyle Mendella4376c5c224e
diff --git a/backend/internal/database/database.go b/backend/internal/database/database.goindex 7aa4135fd..44241b6dd 100644--- a/backend/internal/database/database.go+++ b/backend/internal/database/database.go@@ -449,47 +449,8 @@ func ParseSQLiteConnectionString(connString string) (string, error) {return "", fmt.Errorf("failed to parse SQLite connection string: %w", err)}- qs := make(url.Values, len(connStringUrl.Query()))- for k, v := range connStringUrl.Query() {- switch k {- case "_auto_vacuum", "_vacuum":- qs.Add("_pragma", "auto_vacuum("+v[0]+")")- case "_busy_timeout", "_timeout":- qs.Add("_pragma", "busy_timeout("+v[0]+")")- case "_case_sensitive_like", "_cslike":- qs.Add("_pragma", "case_sensitive_like("+v[0]+")")- case "_foreign_keys", "_fk":- // Foreign-key enforcement is required on every pooled connection.- continue- case "_locking_mode", "_locking":- qs.Add("_pragma", "locking_mode("+v[0]+")")- case "_secure_delete":- qs.Add("_pragma", "secure_delete("+v[0]+")")- case "_synchronous", "_sync":- qs.Add("_pragma", "synchronous("+v[0]+")")- case "_journal_mode":- qs.Add("_pragma", "journal_mode("+v[0]+")")- case "_txlock":- qs.Add("_txlock", v[0])- case "_pragma":- for _, pragma := range v {- name := strings.TrimSpace(pragma)- if end := strings.IndexAny(name, "=( \t\r\n"); end >= 0 {- name = name[:end]- }- if _, unqualified, found := strings.Cut(name, "."); found {- name = unqualified- }- if strings.EqualFold(strings.Trim(name, "\"'`[]"), "foreign_keys") {- continue- }- qs.Add("_pragma", pragma)- }- default:- qs[k] = v- }- }-+ qs := sqliteutil.NormalizeQuery(connStringUrl.Query())+ // Foreign-key enforcement is required on every pooled connection.qs.Add("_pragma", "foreign_keys(1)")connStringUrl.RawQuery = qs.Encode()return connStringUrl.String(), nildiff --git a/backend/pkg/utils/francis/provider.go b/backend/pkg/utils/francis/provider.goindex b9dd7da83..aaad6b958 100644--- a/backend/pkg/utils/francis/provider.go+++ b/backend/pkg/utils/francis/provider.go@@ -17,6 +17,8 @@ import ("github.com/italypaleale/francis/components/postgres""github.com/italypaleale/francis/components/sqlite""github.com/italypaleale/francis/host/local"++ sqliteutil "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/sqlite")// legacyDrops remove the Francis schema that lived in the Arcane SQLite database@@ -42,8 +44,8 @@ func StorePath(databasePath string) string {return strings.TrimSuffix(databasePath, extension) + ".francis" + cmp.Or(extension, ".db")}-// StoreURL resolves Francis storage: a sibling SQLite file without application-// pragmas, or the Postgres URL unchanged.+// StoreURL resolves Francis storage: a sibling SQLite file with the Arcane SQLite+// settings except foreign_keys, which Francis enforces, or the Postgres URL unchanged.func StoreURL(databaseURL string) (string, error) {switch {case strings.HasPrefix(databaseURL, "file:"):@@ -60,7 +62,17 @@ func StoreURL(databaseURL string) (string, error) {if databasePath == "" || strings.HasPrefix(databasePath, ":memory:") || parsed.Query().Get("mode") == "memory" {return "", errors.New("actor storage requires a file-backed SQLite database")}- return "file:" + (&url.URL{Path: StorePath(databasePath)}).EscapedPath(), nil+ query := sqliteutil.NormalizeQuery(parsed.Query())+ // The sibling file must be creatable and writable whatever mode opens arcane.db.+ query.Del("mode")+ if !sqliteutil.HasPragma(query, "busy_timeout") {+ query.Add("_pragma", "busy_timeout(2500)")+ }+ // NORMAL stops WAL commits from holding the write lock through an fsync.+ if !sqliteutil.HasPragma(query, "synchronous") {+ query.Add("_pragma", "synchronous(NORMAL)")+ }+ return "file:" + (&url.URL{Path: StorePath(databasePath)}).EscapedPath() + "?" + query.Encode(), nilcase strings.HasPrefix(databaseURL, "postgres"):return databaseURL, nildefault:@@ -182,6 +194,7 @@ func MigrateLegacyStore(ctx context.Context, mainDB *sql.DB, databaseURL string)if err = tx.Commit(); err != nil {return fmt.Errorf("commit legacy actor storage cleanup: %w", err)}- slog.InfoContext(ctx, "Migrated actor state to its dedicated database", "path", strings.TrimPrefix(storeURL, "file:"))+ storePath, _, _ := strings.Cut(strings.TrimPrefix(storeURL, "file:"), "?")+ slog.InfoContext(ctx, "Migrated actor state to its dedicated database", "path", storePath)return nil}diff --git a/backend/pkg/utils/francis/provider_test.go b/backend/pkg/utils/francis/provider_test.goindex 6d48a9a28..1b2a65b7e 100644--- a/backend/pkg/utils/francis/provider_test.go+++ b/backend/pkg/utils/francis/provider_test.go@@ -18,13 +18,17 @@ func TestMigrateLegacyStore(t *testing.T) {databaseURL := "file:" + mainPath + "?_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_txlock=immediate"storeURL, err := StoreURL(databaseURL)require.NoError(t, err)- require.Equal(t, "file:"+filepath.Join(filepath.Dir(mainPath), "arcane.francis.db"), storeURL)+ const defaults = "_pragma=busy_timeout%282500%29&_pragma=synchronous%28NORMAL%29"+ require.Equal(t, "file:"+filepath.Join(filepath.Dir(mainPath), "arcane.francis.db")+"?_pragma=journal_mode%28WAL%29&"+defaults+"&_txlock=immediate", storeURL)for input, expected := range map[string]string{- "file:/data/arcane%23prod.db?_txlock=immediate": "file:/data/arcane%23prod.francis.db",- "file:/data/arcane%2Edb": "file:/data/arcane.francis.db",- "file:data/arcane%23prod%2Edb": "file:data/arcane%23prod.francis.db",- "file:data/arcane": "file:data/arcane.francis.db",- "postgres://arcane@localhost/arcane?sslmode=disable": "postgres://arcane@localhost/arcane?sslmode=disable",+ "file:/data/arcane%23prod.db?_txlock=immediate": "file:/data/arcane%23prod.francis.db?" + defaults + "&_txlock=immediate",+ "file:/data/arcane%2Edb": "file:/data/arcane.francis.db?" + defaults,+ "file:data/arcane%23prod%2Edb": "file:data/arcane%23prod.francis.db?" + defaults,+ "file:data/arcane": "file:data/arcane.francis.db?" + defaults,+ "file:arcane.db?_pragma=busy_timeout(15000)&_pragma=foreign_keys(1)&_fk=1": "file:arcane.francis.db?_pragma=busy_timeout%2815000%29&_pragma=synchronous%28NORMAL%29",+ "file:arcane.db?_sync=FULL&mode=rw": "file:arcane.francis.db?_pragma=synchronous%28FULL%29&_pragma=busy_timeout%282500%29",+ "file:arcane.db?_pragma=journal_mode(DELETE)&_pragma=synchronous(EXTRA)": "file:arcane.francis.db?_pragma=journal_mode%28DELETE%29&_pragma=synchronous%28EXTRA%29&_pragma=busy_timeout%282500%29",+ "postgres://arcane@localhost/arcane?sslmode=disable": "postgres://arcane@localhost/arcane?sslmode=disable",} {storeURL, err = StoreURL(input)require.NoError(t, err)diff --git a/backend/pkg/utils/francis/runtime.go b/backend/pkg/utils/francis/runtime.goindex e1e15bdfe..8f23d971d 100644--- a/backend/pkg/utils/francis/runtime.go+++ b/backend/pkg/utils/francis/runtime.go@@ -120,7 +120,7 @@ func (r *Runtime) Store() (*gorm.DB, error) {}dialector := postgres.Open(r.storeURL)if strings.HasPrefix(r.storeURL, "file:") {- dialector = sqlite.Open(r.storeURL + "?_pragma=busy_timeout(2500)")+ dialector = sqlite.Open(r.storeURL)}store, err := gorm.Open(dialector, &gorm.Config{Logger: logger.Discard})if err != nil {@@ -281,10 +281,12 @@ func (r *Runtime) publishReady(ctx context.Context, host *local.Host, runErrorsclose(r.ready)go func() {err := <-runErrors- r.finish(err)if ctx.Err() != nil || onFailure == nil {+ r.finish(err)return}+ // onFailure reports the error, so Stop must not return it again.+ r.finish(nil)if err == nil {err = errors.New("actor host stopped unexpectedly")}diff --git a/backend/pkg/utils/sqlite/dsn.go b/backend/pkg/utils/sqlite/dsn.gonew file mode 100644index 000000000..33ace8db3--- /dev/null+++ b/backend/pkg/utils/sqlite/dsn.go@@ -0,0 +1,64 @@+package sqlite++import (+ "net/url"+ "slices"+ "strings"+)++// NormalizeQuery converts mattn-style SQLite DSN options to _pragma values and+// drops every foreign_keys setting so callers control enforcement.+func NormalizeQuery(query url.Values) url.Values {+ qs := make(url.Values, len(query))+ for k, v := range query {+ switch k {+ case "_auto_vacuum", "_vacuum":+ qs.Add("_pragma", "auto_vacuum("+v[0]+")")+ case "_busy_timeout", "_timeout":+ qs.Add("_pragma", "busy_timeout("+v[0]+")")+ case "_case_sensitive_like", "_cslike":+ qs.Add("_pragma", "case_sensitive_like("+v[0]+")")+ case "_foreign_keys", "_fk":+ continue+ case "_locking_mode", "_locking":+ qs.Add("_pragma", "locking_mode("+v[0]+")")+ case "_secure_delete":+ qs.Add("_pragma", "secure_delete("+v[0]+")")+ case "_synchronous", "_sync":+ qs.Add("_pragma", "synchronous("+v[0]+")")+ case "_journal_mode":+ qs.Add("_pragma", "journal_mode("+v[0]+")")+ case "_txlock":+ qs.Add("_txlock", v[0])+ case "_pragma":+ for _, pragma := range v {+ if strings.EqualFold(pragmaName(pragma), "foreign_keys") {+ continue+ }+ qs.Add("_pragma", pragma)+ }+ default:+ qs[k] = v+ }+ }+ return qs+}++// HasPragma reports whether query sets the named _pragma.+func HasPragma(query url.Values, name string) bool {+ return slices.ContainsFunc(query["_pragma"], func(pragma string) bool {+ return strings.EqualFold(pragmaName(pragma), name)+ })+}++// pragmaName strips arguments, schema qualifiers, and quotes from a _pragma value.+func pragmaName(pragma string) string {+ name := strings.TrimSpace(pragma)+ if end := strings.IndexAny(name, "=( \t\r\n"); end >= 0 {+ name = name[:end]+ }+ if _, unqualified, found := strings.Cut(name, "."); found {+ name = unqualified+ }+ return strings.Trim(name, "\"'`[]")+}