Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(store): cut WAL write amplification on 64 KiB-page indexes (#1419)

Martin Vogeleb99608357a3
diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c
index ba41b877c..8b13aa6d4 100644
--- a/src/pipeline/pipeline.c
+++ b/src/pipeline/pipeline.c
@@ -2267,10 +2267,15 @@ int cbm_pipeline_publish_staged(char *stage_path, const cbm_pipeline_generation_
free(stage_path);
return CBM_PIPELINE_PERSIST_FAILED;
}
- bool ok = cbm_store_exec(store, "PRAGMA synchronous=FULL;") == CBM_STORE_OK;
- ok = ok && cbm_store_delete_file_hashes(store, generation->project) == CBM_STORE_OK &&
- cbm_store_upsert_file_hash_batch(store, generation->manifest,
- generation->manifest_count) == CBM_STORE_OK;
+ /* No synchronous=FULL for these writes (#1419): the stage is private until
+ * the atomic rename and a crash discards it, so an fsync per WAL commit
+ * protects nothing here. The store's NORMAL level is SQLite's
+ * corruption-safe setting under WAL, and cbm_store_seal_for_atomic_publish()
+ * raises this connection to FULL for the checkpoint that makes the
+ * published file durable. */
+ bool ok = cbm_store_delete_file_hashes(store, generation->project) == CBM_STORE_OK &&
+ cbm_store_upsert_file_hash_batch(store, generation->manifest,
+ generation->manifest_count) == CBM_STORE_OK;
/* LSP surfaces belong to the generation: written inside the same staging
* store, before the atomic rename, so graph and surface data can never
* publish separately. The delete guards the incremental path, whose
diff --git a/src/store/store.c b/src/store/store.c
index d66ef0ffb..64c5a96ea 100644
--- a/src/store/store.c
+++ b/src/store/store.c
@@ -539,6 +539,16 @@ int64_t cbm_store_resolve_mmap_size(void) {
return (int64_t)parsed;
}
+/* #1419: page cache for read-write on-disk connections, in SQLite's negative
+ * KiB form. sqlite_writer.c gives every index 64 KiB pages, so SQLite's
+ * default 2000 KiB holds only ~31 of them, while a delta's purge and patch
+ * dirty pages across the node and edge tables and all of their indexes. The
+ * transaction then spills dirty pages into the WAL and rewrites them over and
+ * over (5.2 GiB of WAL writes for a one-line change on a 294 MB django index).
+ * 64 MiB = 1024 pages, the budget bulk mode already used; the cache only grows
+ * as pages are touched. Query connections keep the default. */
+static const char ST_WRITE_CACHE_SQL[] = "PRAGMA cache_size = -65536;";
+
/* Configure connection pragmas.
* in_memory — :memory: DB (synchronous OFF, no journal file).
* read_only — query-only connection opened SQLITE_OPEN_READONLY. Runs
@@ -594,8 +604,9 @@ static int configure_pragmas(cbm_store_t *s, bool in_memory, bool read_only) {
* cbm_store_checkpoint's SIGBUS note), so without a size limit the -wal
* file only ever grows; journal_size_limit truncates it back to N bytes
* on the next successful
- * reset. N is far above the healthy WAL (~4 MiB under the default
- * 1000-page autocheckpoint), so normal indexing never triggers
+ * reset. N is far above the healthy WAL (~64 MiB: the default
+ * 1000-page autocheckpoint at the 64 KiB page size sqlite_writer.c
+ * gives every index), so normal indexing never triggers
* truncate/regrow churn — it only fires after abnormal growth.
* Shared/live paths do NOT use a TRUNCATE checkpoint: truncating the WAL
* to zero can raise SIGBUS in a sibling process that has the DB mmap'd
@@ -604,6 +615,10 @@ static int configure_pragmas(cbm_store_t *s, bool in_memory, bool read_only) {
if (rc != CBM_STORE_OK) {
return rc;
}
+ rc = exec_sql(s, ST_WRITE_CACHE_SQL);
+ if (rc != CBM_STORE_OK) {
+ return rc;
+ }
char mmap_sql[ST_BUF_64];
snprintf(mmap_sql, sizeof(mmap_sql), "PRAGMA mmap_size = %lld;",
(long long)cbm_store_resolve_mmap_size());
@@ -1811,12 +1826,12 @@ int cbm_store_begin_bulk(cbm_store_t *s) {
* because the in-memory rollback journal is lost on crash.
* WAL mode is crash-safe: uncommitted WAL entries are simply discarded
* on the next open. Performance is preserved via synchronous=OFF and a
- * larger cache, which are safe with WAL. */
+ * 64 MiB cache, which are safe with WAL. */
int rc = exec_sql(s, "PRAGMA synchronous = OFF;");
if (rc != CBM_STORE_OK) {
return rc;
}
- return exec_sql(s, "PRAGMA cache_size = -65536;"); /* CBM_SZ_64 MB */
+ return exec_sql(s, ST_WRITE_CACHE_SQL);
}
int cbm_store_end_bulk(cbm_store_t *s) {
@@ -1824,7 +1839,8 @@ int cbm_store_end_bulk(cbm_store_t *s) {
if (rc != CBM_STORE_OK) {
return rc;
}
- return exec_sql(s, "PRAGMA cache_size = -2000;"); /* default ~2 MB */
+ /* Back to the read-write cache, not SQLite's 2000 KiB default (#1419). */
+ return exec_sql(s, ST_WRITE_CACHE_SQL);
}
int cbm_store_drop_indexes(cbm_store_t *s) {
diff --git a/src/store/store.h b/src/store/store.h
index 06ea1fbb7..4eb0be080 100644
--- a/src/store/store.h
+++ b/src/store/store.h
@@ -427,11 +427,12 @@ int cbm_store_rollback(cbm_store_t *s);
/* ── Bulk write optimization ────────────────────────────────────── */
-/* Tune pragmas for bulk write throughput (synchronous=OFF, large cache).
+/* Tune pragmas for bulk write throughput (synchronous=OFF, 64 MiB cache).
* WAL journal mode is preserved throughout for crash safety. */
int cbm_store_begin_bulk(cbm_store_t *s);
-/* Restore normal pragmas (synchronous=NORMAL, default cache) after bulk writes. */
+/* Restore normal pragmas (synchronous=NORMAL, the 64 MiB read-write cache)
+ * after bulk writes. */
int cbm_store_end_bulk(cbm_store_t *s);
/* Drop user indexes for faster bulk inserts. */
diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c
index 03a5cad70..55d6aa092 100644
--- a/tests/test_pipeline.c
+++ b/tests/test_pipeline.c
@@ -2916,6 +2916,115 @@ TEST(pipeline_incremental_repoints_call_reference_without_stale_edge) {
PASS();
}
+/* #1419: every index publishes through a private staging DB in WAL mode that
+ * cbm_store_seal_for_atomic_publish() checkpoints at synchronous=FULL and
+ * switches to DELETE mode before the atomic rename. A crash before that rename
+ * discards the stage, so a WAL commit at FULL only adds an fsync per
+ * transaction without protecting anything; SQLite documents NORMAL as
+ * corruption-safe under WAL. The probe reads the EFFECTIVE synchronous level
+ * of the committing handle from a WAL hook on every connection the run opens.
+ *
+ * SQLite installs its default autocheckpoint WAL hook after auto-extensions
+ * run, so the auto-extension installs a statement trace and the trace installs
+ * the probe once the connection is fully open. The probe keeps SQLite's
+ * default 1000-frame autocheckpoint, so it does not change what the run
+ * writes. */
+enum { I1419_SYNC_FULL = 2, I1419_DEFAULT_AUTOCHECKPOINT = 1000 };
+static atomic_int g_i1419_wal_commits;
+static atomic_int g_i1419_full_sync_commits;
+
+static int i1419_wal_probe(void *arg, sqlite3 *db, const char *db_name, int wal_frames) {
+ (void)arg;
+ atomic_fetch_add(&g_i1419_wal_commits, 1);
+ sqlite3_stmt *stmt = NULL;
+ if (sqlite3_prepare_v2(db, "PRAGMA synchronous;", -1, &stmt, NULL) == SQLITE_OK &&
+ sqlite3_step(stmt) == SQLITE_ROW && sqlite3_column_int(stmt, 0) >= I1419_SYNC_FULL) {
+ atomic_fetch_add(&g_i1419_full_sync_commits, 1);
+ }
+ sqlite3_finalize(stmt);
+ if (wal_frames >= I1419_DEFAULT_AUTOCHECKPOINT) {
+ (void)sqlite3_wal_checkpoint(db, db_name);
+ }
+ return SQLITE_OK;
+}
+
+static int i1419_trace(unsigned type, void *ctx, void *stmt, void *sql) {
+ (void)type;
+ (void)ctx;
+ (void)sql;
+ (void)sqlite3_wal_hook(sqlite3_db_handle((sqlite3_stmt *)stmt), i1419_wal_probe, NULL);
+ return 0;
+}
+
+static int i1419_install_probe(sqlite3 *db, char **errmsg, const sqlite3_api_routines *api) {
+ (void)errmsg;
+ (void)api;
+ return sqlite3_trace_v2(db, SQLITE_TRACE_STMT, i1419_trace, NULL);
+}
+
+static int i1419_index(const char *repo, const char *db_path) {
+ cbm_pipeline_t *p = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL);
+ if (!p) {
+ return -1;
+ }
+ int rc = cbm_pipeline_run(p);
+ cbm_pipeline_free(p);
+ return rc;
+}
+
+TEST(pipeline_staged_publish_commits_below_full_sync_issue1419) {
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_i1419_XXXXXX");
+ if (!cbm_mkdtemp(tmp)) {
+ FAIL("tmpdir");
+ }
+ /* Equal lengths + equal mtimes: the second run must route on content
+ * bytes to the closure-repair (delta) publish, not a full rebuild. */
+ static const char initial_source[] =
+ "package syncprobe\n"
+ "func alphaSyncTarget() {}\n"
+ "func bravoSyncTarget() {}\n"
+ "func syncProbeAccept(callback func()) {}\n"
+ "func syncProbeSite() { syncProbeAccept(alphaSyncTarget) }\n";
+ static const char replacement_source[] =
+ "package syncprobe\n"
+ "func alphaSyncTarget() {}\n"
+ "func bravoSyncTarget() {}\n"
+ "func syncProbeAccept(callback func()) {}\n"
+ "func syncProbeSite() { syncProbeAccept(bravoSyncTarget) }\n";
+ ASSERT_EQ(sizeof(initial_source), sizeof(replacement_source));
+ write_temp_file(tmp, "probe.go", initial_source);
+ char source_path[512];
+ snprintf(source_path, sizeof(source_path), "%s/probe.go", tmp);
+ const time_t fixed_mtime = 1700000000;
+ ASSERT_EQ(pipeline_test_set_mtime(source_path, fixed_mtime, 123456789L), 0);
+ char db_path[512];
+ snprintf(db_path, sizeof(db_path), "%s/i1419.db", tmp);
+
+ atomic_store(&g_i1419_wal_commits, 0);
+ atomic_store(&g_i1419_full_sync_commits, 0);
+ ASSERT_EQ(sqlite3_auto_extension((void (*)(void))i1419_install_probe), SQLITE_OK);
+ /* Both publish routes run before any assertion, so the probe is always
+ * unregistered again even when a run fails. */
+ int full_rc = i1419_index(tmp, db_path);
+ write_temp_file(tmp, "probe.go", replacement_source);
+ int mtime_rc = pipeline_test_set_mtime(source_path, fixed_mtime, 123456789L);
+ cbm_pipeline_incremental_test_reset_faults();
+ int delta_rc = i1419_index(tmp, db_path);
+ cbm_incremental_route_t delta_route = cbm_pipeline_incremental_test_last_route();
+ (void)sqlite3_cancel_auto_extension((void (*)(void))i1419_install_probe);
+ th_rmtree(tmp);
+
+ ASSERT_EQ(full_rc, 0);
+ ASSERT_EQ(mtime_rc, 0);
+ ASSERT_EQ(delta_rc, 0);
+ ASSERT_EQ(delta_route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
+ /* Non-vacuous: both publishes commit through the WAL. */
+ ASSERT_GT(atomic_load(&g_i1419_wal_commits), 0);
+ ASSERT_EQ(atomic_load(&g_i1419_full_sync_commits), 0);
+ PASS();
+}
+
/* SQL DDL becomes first-class Table/View nodes wired into FROM/JOIN lineage,
* while the shared name registry must NOT leak those relations into other
* languages' textual resolution: a Python call or identifier sharing the
@@ -16556,6 +16665,7 @@ SUITE(pipeline) {
SUITE(pipeline_semantic_manifest_repro) {
RUN_TEST(incremental_downgrade_preserves_scope_and_artifact_across_change_noop_delete);
RUN_TEST(pipeline_incremental_repoints_call_reference_without_stale_edge);
+ RUN_TEST(pipeline_staged_publish_commits_below_full_sync_issue1419);
RUN_TEST(pipeline_sql_lineage_and_relation_isolation);
RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage);
RUN_TEST(pipeline_dbt_jinja_lineage);
diff --git a/tests/test_store_pragmas.c b/tests/test_store_pragmas.c
index 757d46363..a81d73cbd 100644
--- a/tests/test_store_pragmas.c
+++ b/tests/test_store_pragmas.c
@@ -11,6 +11,8 @@
#include "test_framework.h"
#include "test_helpers.h"
#include <store/store.h>
+#include "sqlite3.h" /* vendored/sqlite3 — read pragmas back on the store's own handle */
+#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -98,8 +100,9 @@ TEST(journal_size_limit_bounds_wal_issue1083) {
cbm_store_t *s = cbm_store_open_path(tmp_path);
ASSERT(s != NULL);
- /* 256 MiB — far above the healthy WAL (~4 MiB), so no truncate/regrow churn
- * in normal operation; it only fires after abnormal (starved) growth. */
+ /* 256 MiB — far above the healthy WAL (~64 MiB: 1000 autocheckpoint pages
+ * of 64 KiB), so no truncate/regrow churn in normal operation; it only
+ * fires after abnormal (starved) growth. */
ASSERT(cbm_store_journal_size_limit(s) == (int64_t)268435456);
cbm_store_close(s);
@@ -113,6 +116,86 @@ TEST(journal_size_limit_bounds_wal_issue1083) {
PASS();
}
+/* #1419: effective pragmas per store role, read on the handle itself —
+ * synchronous is per-connection and never persisted, so an external sqlite3
+ * shell reports its own default (FULL), not what the indexer runs with. */
+static int role_pragma_int(cbm_store_t *s, const char *sql) {
+ sqlite3_stmt *stmt = NULL;
+ int value = -1;
+ if (sqlite3_prepare_v2(cbm_store_get_db(s), sql, -1, &stmt, NULL) == SQLITE_OK &&
+ sqlite3_step(stmt) == SQLITE_ROW) {
+ value = sqlite3_column_int(stmt, 0);
+ }
+ sqlite3_finalize(stmt);
+ return value;
+}
+
+static bool role_journal_mode_is(cbm_store_t *s, const char *want) {
+ sqlite3_stmt *stmt = NULL;
+ bool match = false;
+ if (sqlite3_prepare_v2(cbm_store_get_db(s), "PRAGMA journal_mode;", -1, &stmt, NULL) ==
+ SQLITE_OK &&
+ sqlite3_step(stmt) == SQLITE_ROW) {
+ const char *mode = (const char *)sqlite3_column_text(stmt, 0);
+ match = mode && strcmp(mode, want) == 0;
+ }
+ sqlite3_finalize(stmt);
+ return match;
+}
+
+TEST(store_role_pragmas_issue1419) {
+ enum {
+ SYNC_OFF = 0,
+ SYNC_NORMAL = 1,
+ SYNC_FULL = 2,
+ /* 64 MiB = 1024 of the 64 KiB pages every index is written with. */
+ WRITE_CACHE_KIB = -65536,
+ SQLITE_DEFAULT_CACHE_KIB = -2000,
+ };
+ char tmp_path[256];
+ snprintf(tmp_path, sizeof(tmp_path), "%s/cbm_test_roles_%d.db", cbm_tmpdir(), (int)getpid());
+ unlink(tmp_path);
+
+ /* Read-write role (live ADR writes, staging generations): WAL at NORMAL,
+ * with a page cache that holds more than a few 64 KiB pages. */
+ cbm_store_t *s = cbm_store_open_path(tmp_path);
+ ASSERT(s != NULL);
+ ASSERT_TRUE(role_journal_mode_is(s, "wal"));
+ ASSERT_EQ(role_pragma_int(s, "PRAGMA synchronous;"), SYNC_NORMAL);
+ ASSERT_EQ(role_pragma_int(s, "PRAGMA cache_size;"), WRITE_CACHE_KIB);
+ /* Bulk role: sync off for the write burst, then back to the read-write
+ * settings rather than SQLite's defaults. */
+ ASSERT_EQ(cbm_store_begin_bulk(s), CBM_STORE_OK);
+ ASSERT_EQ(role_pragma_int(s, "PRAGMA synchronous;"), SYNC_OFF);
+ ASSERT_EQ(cbm_store_end_bulk(s), CBM_STORE_OK);
+ ASSERT_EQ(role_pragma_int(s, "PRAGMA synchronous;"), SYNC_NORMAL);
+ ASSERT_EQ(role_pragma_int(s, "PRAGMA cache_size;"), WRITE_CACHE_KIB);
+ /* Seal role: the durable checkpoint before the atomic rename runs at FULL
+ * and leaves a self-contained DELETE-mode file. */
+ ASSERT_EQ(cbm_store_seal_for_atomic_publish(s), CBM_STORE_OK);
+ ASSERT_EQ(role_pragma_int(s, "PRAGMA synchronous;"), SYNC_FULL);
+ ASSERT_TRUE(role_journal_mode_is(s, "delete"));
+ cbm_store_close(s);
+
+ /* Query role: read-only, never switches a sealed file back to WAL, and
+ * keeps SQLite's small default cache (one per request, so it stays cheap). */
+ cbm_store_t *q = cbm_store_open_path_query(tmp_path);
+ ASSERT(q != NULL);
+ ASSERT_EQ(sqlite3_db_readonly(cbm_store_get_db(q), "main"), 1);
+ ASSERT_TRUE(role_journal_mode_is(q, "delete"));
+ ASSERT_EQ(role_pragma_int(q, "PRAGMA cache_size;"), SQLITE_DEFAULT_CACHE_KIB);
+ cbm_store_close(q);
+
+ unlink(tmp_path);
+ char tmp_wal[300];
+ char tmp_shm[300];
+ snprintf(tmp_wal, sizeof(tmp_wal), "%s-wal", tmp_path);
+ snprintf(tmp_shm, sizeof(tmp_shm), "%s-shm", tmp_path);
+ unlink(tmp_wal);
+ unlink(tmp_shm);
+ PASS();
+}
+
/* Pagination-cursor generation: minted per DB file, bumped per index run.
* Same store + reads only -> stable; upsert_project (every index run's choke
* point) -> changes; two distinct DB files can never share a generation even
@@ -264,6 +347,7 @@ TEST(corrupt_page_scan_returns_error_not_truncation) {
SUITE(store_pragmas) {
RUN_TEST(journal_size_limit_bounds_wal_issue1083);
+ RUN_TEST(store_role_pragmas_issue1419);
RUN_TEST(store_generation_tracks_mutations);
RUN_TEST(store_generation_rejects_malformed_metadata_atomically);
RUN_TEST(corrupt_page_scan_returns_error_not_truncation);