Skip to content

Modern Docker Management, Designed for Everyone

fix(francis): keep SQLite tuning on the actor store and stop fsyncing on commit (#4372)

Kyle Mendella4376c5c224e
diff --git a/backend/internal/database/database.go b/backend/internal/database/database.go
index 7aa4135fd..44241b6dd 100644
--- a/backend/internal/database/database.go
+++ b/backend/internal/database/database.go
@@ -449,47 +449,8 @@ func ParseSQLiteConnectionString(connString string) (string, error) {
return "", fmt.Errorf("failed to parse SQLite connection string: %w", err)
}
- qs := make(url.Values, len(connStringUrl.Query()))
- for k, v := range connStringUrl.Query() {
- switch k {
- case "_auto_vacuum", "_vacuum":
- qs.Add("_pragma", "auto_vacuum("+v[0]+")")
- case "_busy_timeout", "_timeout":
- qs.Add("_pragma", "busy_timeout("+v[0]+")")
- case "_case_sensitive_like", "_cslike":
- qs.Add("_pragma", "case_sensitive_like("+v[0]+")")
- case "_foreign_keys", "_fk":
- // Foreign-key enforcement is required on every pooled connection.
- continue
- case "_locking_mode", "_locking":
- qs.Add("_pragma", "locking_mode("+v[0]+")")
- case "_secure_delete":
- qs.Add("_pragma", "secure_delete("+v[0]+")")
- case "_synchronous", "_sync":
- qs.Add("_pragma", "synchronous("+v[0]+")")
- case "_journal_mode":
- qs.Add("_pragma", "journal_mode("+v[0]+")")
- case "_txlock":
- qs.Add("_txlock", v[0])
- case "_pragma":
- for _, pragma := range v {
- name := strings.TrimSpace(pragma)
- if end := strings.IndexAny(name, "=( \t\r\n"); end >= 0 {
- name = name[:end]
- }
- if _, unqualified, found := strings.Cut(name, "."); found {
- name = unqualified
- }
- if strings.EqualFold(strings.Trim(name, "\"'`[]"), "foreign_keys") {
- continue
- }
- qs.Add("_pragma", pragma)
- }
- default:
- qs[k] = v
- }
- }
-
+ qs := sqliteutil.NormalizeQuery(connStringUrl.Query())
+ // Foreign-key enforcement is required on every pooled connection.
qs.Add("_pragma", "foreign_keys(1)")
connStringUrl.RawQuery = qs.Encode()
return connStringUrl.String(), nil
diff --git a/backend/pkg/utils/francis/provider.go b/backend/pkg/utils/francis/provider.go
index b9dd7da83..aaad6b958 100644
--- a/backend/pkg/utils/francis/provider.go
+++ b/backend/pkg/utils/francis/provider.go
@@ -17,6 +17,8 @@ import (
"github.com/italypaleale/francis/components/postgres"
"github.com/italypaleale/francis/components/sqlite"
"github.com/italypaleale/francis/host/local"
+
+ sqliteutil "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/sqlite"
)
// legacyDrops remove the Francis schema that lived in the Arcane SQLite database
@@ -42,8 +44,8 @@ func StorePath(databasePath string) string {
return strings.TrimSuffix(databasePath, extension) + ".francis" + cmp.Or(extension, ".db")
}
-// StoreURL resolves Francis storage: a sibling SQLite file without application
-// pragmas, or the Postgres URL unchanged.
+// StoreURL resolves Francis storage: a sibling SQLite file with the Arcane SQLite
+// settings except foreign_keys, which Francis enforces, or the Postgres URL unchanged.
func StoreURL(databaseURL string) (string, error) {
switch {
case strings.HasPrefix(databaseURL, "file:"):
@@ -60,7 +62,17 @@ func StoreURL(databaseURL string) (string, error) {
if databasePath == "" || strings.HasPrefix(databasePath, ":memory:") || parsed.Query().Get("mode") == "memory" {
return "", errors.New("actor storage requires a file-backed SQLite database")
}
- return "file:" + (&url.URL{Path: StorePath(databasePath)}).EscapedPath(), nil
+ query := sqliteutil.NormalizeQuery(parsed.Query())
+ // The sibling file must be creatable and writable whatever mode opens arcane.db.
+ query.Del("mode")
+ if !sqliteutil.HasPragma(query, "busy_timeout") {
+ query.Add("_pragma", "busy_timeout(2500)")
+ }
+ // NORMAL stops WAL commits from holding the write lock through an fsync.
+ if !sqliteutil.HasPragma(query, "synchronous") {
+ query.Add("_pragma", "synchronous(NORMAL)")
+ }
+ return "file:" + (&url.URL{Path: StorePath(databasePath)}).EscapedPath() + "?" + query.Encode(), nil
case strings.HasPrefix(databaseURL, "postgres"):
return databaseURL, nil
default:
@@ -182,6 +194,7 @@ func MigrateLegacyStore(ctx context.Context, mainDB *sql.DB, databaseURL string)
if err = tx.Commit(); err != nil {
return fmt.Errorf("commit legacy actor storage cleanup: %w", err)
}
- slog.InfoContext(ctx, "Migrated actor state to its dedicated database", "path", strings.TrimPrefix(storeURL, "file:"))
+ storePath, _, _ := strings.Cut(strings.TrimPrefix(storeURL, "file:"), "?")
+ slog.InfoContext(ctx, "Migrated actor state to its dedicated database", "path", storePath)
return nil
}
diff --git a/backend/pkg/utils/francis/provider_test.go b/backend/pkg/utils/francis/provider_test.go
index 6d48a9a28..1b2a65b7e 100644
--- a/backend/pkg/utils/francis/provider_test.go
+++ b/backend/pkg/utils/francis/provider_test.go
@@ -18,13 +18,17 @@ func TestMigrateLegacyStore(t *testing.T) {
databaseURL := "file:" + mainPath + "?_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_txlock=immediate"
storeURL, err := StoreURL(databaseURL)
require.NoError(t, err)
- require.Equal(t, "file:"+filepath.Join(filepath.Dir(mainPath), "arcane.francis.db"), storeURL)
+ const defaults = "_pragma=busy_timeout%282500%29&_pragma=synchronous%28NORMAL%29"
+ require.Equal(t, "file:"+filepath.Join(filepath.Dir(mainPath), "arcane.francis.db")+"?_pragma=journal_mode%28WAL%29&"+defaults+"&_txlock=immediate", storeURL)
for input, expected := range map[string]string{
- "file:/data/arcane%23prod.db?_txlock=immediate": "file:/data/arcane%23prod.francis.db",
- "file:/data/arcane%2Edb": "file:/data/arcane.francis.db",
- "file:data/arcane%23prod%2Edb": "file:data/arcane%23prod.francis.db",
- "file:data/arcane": "file:data/arcane.francis.db",
- "postgres://arcane@localhost/arcane?sslmode=disable": "postgres://arcane@localhost/arcane?sslmode=disable",
+ "file:/data/arcane%23prod.db?_txlock=immediate": "file:/data/arcane%23prod.francis.db?" + defaults + "&_txlock=immediate",
+ "file:/data/arcane%2Edb": "file:/data/arcane.francis.db?" + defaults,
+ "file:data/arcane%23prod%2Edb": "file:data/arcane%23prod.francis.db?" + defaults,
+ "file:data/arcane": "file:data/arcane.francis.db?" + defaults,
+ "file:arcane.db?_pragma=busy_timeout(15000)&_pragma=foreign_keys(1)&_fk=1": "file:arcane.francis.db?_pragma=busy_timeout%2815000%29&_pragma=synchronous%28NORMAL%29",
+ "file:arcane.db?_sync=FULL&mode=rw": "file:arcane.francis.db?_pragma=synchronous%28FULL%29&_pragma=busy_timeout%282500%29",
+ "file:arcane.db?_pragma=journal_mode(DELETE)&_pragma=synchronous(EXTRA)": "file:arcane.francis.db?_pragma=journal_mode%28DELETE%29&_pragma=synchronous%28EXTRA%29&_pragma=busy_timeout%282500%29",
+ "postgres://arcane@localhost/arcane?sslmode=disable": "postgres://arcane@localhost/arcane?sslmode=disable",
} {
storeURL, err = StoreURL(input)
require.NoError(t, err)
diff --git a/backend/pkg/utils/francis/runtime.go b/backend/pkg/utils/francis/runtime.go
index e1e15bdfe..8f23d971d 100644
--- a/backend/pkg/utils/francis/runtime.go
+++ b/backend/pkg/utils/francis/runtime.go
@@ -120,7 +120,7 @@ func (r *Runtime) Store() (*gorm.DB, error) {
}
dialector := postgres.Open(r.storeURL)
if strings.HasPrefix(r.storeURL, "file:") {
- dialector = sqlite.Open(r.storeURL + "?_pragma=busy_timeout(2500)")
+ dialector = sqlite.Open(r.storeURL)
}
store, err := gorm.Open(dialector, &gorm.Config{Logger: logger.Discard})
if err != nil {
@@ -281,10 +281,12 @@ func (r *Runtime) publishReady(ctx context.Context, host *local.Host, runErrors
close(r.ready)
go func() {
err := <-runErrors
- r.finish(err)
if ctx.Err() != nil || onFailure == nil {
+ r.finish(err)
return
}
+ // onFailure reports the error, so Stop must not return it again.
+ r.finish(nil)
if err == nil {
err = errors.New("actor host stopped unexpectedly")
}
diff --git a/backend/pkg/utils/sqlite/dsn.go b/backend/pkg/utils/sqlite/dsn.go
new file mode 100644
index 000000000..33ace8db3
--- /dev/null
+++ b/backend/pkg/utils/sqlite/dsn.go
@@ -0,0 +1,64 @@
+package sqlite
+
+import (
+ "net/url"
+ "slices"
+ "strings"
+)
+
+// NormalizeQuery converts mattn-style SQLite DSN options to _pragma values and
+// drops every foreign_keys setting so callers control enforcement.
+func NormalizeQuery(query url.Values) url.Values {
+ qs := make(url.Values, len(query))
+ for k, v := range query {
+ switch k {
+ case "_auto_vacuum", "_vacuum":
+ qs.Add("_pragma", "auto_vacuum("+v[0]+")")
+ case "_busy_timeout", "_timeout":
+ qs.Add("_pragma", "busy_timeout("+v[0]+")")
+ case "_case_sensitive_like", "_cslike":
+ qs.Add("_pragma", "case_sensitive_like("+v[0]+")")
+ case "_foreign_keys", "_fk":
+ continue
+ case "_locking_mode", "_locking":
+ qs.Add("_pragma", "locking_mode("+v[0]+")")
+ case "_secure_delete":
+ qs.Add("_pragma", "secure_delete("+v[0]+")")
+ case "_synchronous", "_sync":
+ qs.Add("_pragma", "synchronous("+v[0]+")")
+ case "_journal_mode":
+ qs.Add("_pragma", "journal_mode("+v[0]+")")
+ case "_txlock":
+ qs.Add("_txlock", v[0])
+ case "_pragma":
+ for _, pragma := range v {
+ if strings.EqualFold(pragmaName(pragma), "foreign_keys") {
+ continue
+ }
+ qs.Add("_pragma", pragma)
+ }
+ default:
+ qs[k] = v
+ }
+ }
+ return qs
+}
+
+// HasPragma reports whether query sets the named _pragma.
+func HasPragma(query url.Values, name string) bool {
+ return slices.ContainsFunc(query["_pragma"], func(pragma string) bool {
+ return strings.EqualFold(pragmaName(pragma), name)
+ })
+}
+
+// pragmaName strips arguments, schema qualifiers, and quotes from a _pragma value.
+func pragmaName(pragma string) string {
+ name := strings.TrimSpace(pragma)
+ if end := strings.IndexAny(name, "=( \t\r\n"); end >= 0 {
+ name = name[:end]
+ }
+ if _, unqualified, found := strings.Cut(name, "."); found {
+ name = unqualified
+ }
+ return strings.Trim(name, "\"'`[]")
+}