Skip to content

The only tool you need to know what is happening and how to fix it.

feat: autovacuum on sqlite

ddux61496769b36f
diff --git a/backend/app/db/db_sqlite.go b/backend/app/db/db_sqlite.go
index ca12e3aa..4d83d1f3 100644
--- a/backend/app/db/db_sqlite.go
+++ b/backend/app/db/db_sqlite.go
@@ -73,6 +73,7 @@ func openSQLite(path string, telemetry bool) (*sql.DB, error) {
"_pragma=temp_store(MEMORY)",
"_pragma=mmap_size(1073741824)",
"_pragma=wal_autocheckpoint(50000)",
+ "_pragma=auto_vacuum(incremental)",
)
} else {
params = append(params, "_pragma=foreign_keys(ON)")
diff --git a/backend/app/retention/sqlite.go b/backend/app/retention/sqlite.go
index cf5d6bdc..0fbbdcc6 100644
--- a/backend/app/retention/sqlite.go
+++ b/backend/app/retention/sqlite.go
@@ -73,5 +73,21 @@ func runSQLiteRetention(ctx context.Context, days int) {
traceway.CaptureException(fmt.Errorf("retention: delete from telemetry.%s failed: %w", tgt.table, err))
}
}
+ reclaimTelemetryDisk(ctx)
+ }
+}
+
+func reclaimTelemetryDisk(ctx context.Context) {
+ for _, stmt := range []string{
+ "PRAGMA incremental_vacuum",
+ "PRAGMA optimize",
+ "PRAGMA wal_checkpoint(TRUNCATE)",
+ } {
+ if ctx.Err() != nil {
+ return
+ }
+ if _, err := db.TelemetryDB.ExecContext(ctx, stmt); err != nil {
+ traceway.CaptureException(fmt.Errorf("retention: telemetry maintenance %q failed: %w", stmt, err))
+ }
}
}
diff --git a/backend/app/retention/sqlite_test.go b/backend/app/retention/sqlite_test.go
index 7e761e40..b1c89c0f 100644
--- a/backend/app/retention/sqlite_test.go
+++ b/backend/app/retention/sqlite_test.go
@@ -6,6 +6,8 @@ import (
"context"
"database/sql"
"fmt"
+ "os"
+ "path/filepath"
"testing"
"time"
@@ -167,6 +169,56 @@ func TestRunSQLiteRetention_PrunesProfilingTables(t *testing.T) {
}
}
+func TestRunSQLiteRetention_TruncatesWALAfterPass(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "telemetry.db")
+ dsn := "file:" + path + "?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=synchronous(NORMAL)&_pragma=auto_vacuum(incremental)&_pragma=wal_autocheckpoint(1000000)"
+
+ telDB, err := sql.Open("sqlite", dsn)
+ if err != nil {
+ t.Fatalf("open telemetry: %v", err)
+ }
+ telDB.SetMaxOpenConns(1)
+ if err := telDB.Ping(); err != nil {
+ t.Fatalf("ping telemetry: %v", err)
+ }
+
+ for _, tgt := range telemetryRetentionTargets {
+ ddl := fmt.Sprintf("CREATE TABLE %s (id TEXT, project_id TEXT, %s DATETIME NOT NULL)", tgt.table, tgt.column)
+ if _, err := telDB.Exec(ddl); err != nil {
+ t.Fatalf("ddl %s: %v", tgt.table, err)
+ }
+ }
+
+ prevTelDB, prevDriver := db.TelemetryDB, db.Driver
+ db.TelemetryDB = telDB
+ db.Driver = lit.SQLite
+ t.Cleanup(func() {
+ telDB.Close()
+ db.TelemetryDB = prevTelDB
+ db.Driver = prevDriver
+ })
+
+ old := time.Now().UTC().AddDate(0, 0, -45)
+ for range 2000 {
+ insertWithTime(t, telDB, "endpoints", "recorded_at", old)
+ }
+
+ walPath := path + "-wal"
+ info, err := os.Stat(walPath)
+ if err != nil || info.Size() == 0 {
+ t.Fatalf("expected WAL to grow before retention (err=%v)", err)
+ }
+
+ runSQLiteRetention(context.Background(), 30)
+
+ assertCount(t, telDB, "endpoints", 0)
+
+ if info, err := os.Stat(walPath); err == nil && info.Size() > 0 {
+ t.Errorf("expected WAL truncated to 0 after retention, got %d bytes", info.Size())
+ }
+}
+
func assertCount(t *testing.T, ex *sql.DB, table string, want int) {
t.Helper()
if got := countRows(t, ex, table); got != want {