Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(store): cut WAL write amplification on 64 KiB-page indexes (#1419)
Martin Vogeleb99608357a3
diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.cindex ba41b877c..8b13aa6d4 100644--- a/src/pipeline/pipeline.c+++ b/src/pipeline/pipeline.c@@ -2267,10 +2267,15 @@ int cbm_pipeline_publish_staged(char *stage_path, const cbm_pipeline_generation_free(stage_path);return CBM_PIPELINE_PERSIST_FAILED;}- bool ok = cbm_store_exec(store, "PRAGMA synchronous=FULL;") == CBM_STORE_OK;- ok = ok && cbm_store_delete_file_hashes(store, generation->project) == CBM_STORE_OK &&- cbm_store_upsert_file_hash_batch(store, generation->manifest,- generation->manifest_count) == CBM_STORE_OK;+ /* No synchronous=FULL for these writes (#1419): the stage is private until+ * the atomic rename and a crash discards it, so an fsync per WAL commit+ * protects nothing here. The store's NORMAL level is SQLite's+ * corruption-safe setting under WAL, and cbm_store_seal_for_atomic_publish()+ * raises this connection to FULL for the checkpoint that makes the+ * published file durable. */+ bool ok = cbm_store_delete_file_hashes(store, generation->project) == CBM_STORE_OK &&+ cbm_store_upsert_file_hash_batch(store, generation->manifest,+ generation->manifest_count) == CBM_STORE_OK;/* LSP surfaces belong to the generation: written inside the same staging* store, before the atomic rename, so graph and surface data can never* publish separately. The delete guards the incremental path, whosediff --git a/src/store/store.c b/src/store/store.cindex d66ef0ffb..64c5a96ea 100644--- a/src/store/store.c+++ b/src/store/store.c@@ -539,6 +539,16 @@ int64_t cbm_store_resolve_mmap_size(void) {return (int64_t)parsed;}+/* #1419: page cache for read-write on-disk connections, in SQLite's negative+ * KiB form. sqlite_writer.c gives every index 64 KiB pages, so SQLite's+ * default 2000 KiB holds only ~31 of them, while a delta's purge and patch+ * dirty pages across the node and edge tables and all of their indexes. The+ * transaction then spills dirty pages into the WAL and rewrites them over and+ * over (5.2 GiB of WAL writes for a one-line change on a 294 MB django index).+ * 64 MiB = 1024 pages, the budget bulk mode already used; the cache only grows+ * as pages are touched. Query connections keep the default. */+static const char ST_WRITE_CACHE_SQL[] = "PRAGMA cache_size = -65536;";+/* Configure connection pragmas.* in_memory — :memory: DB (synchronous OFF, no journal file).* read_only — query-only connection opened SQLITE_OPEN_READONLY. Runs@@ -594,8 +604,9 @@ static int configure_pragmas(cbm_store_t *s, bool in_memory, bool read_only) {* cbm_store_checkpoint's SIGBUS note), so without a size limit the -wal* file only ever grows; journal_size_limit truncates it back to N bytes* on the next successful- * reset. N is far above the healthy WAL (~4 MiB under the default- * 1000-page autocheckpoint), so normal indexing never triggers+ * reset. N is far above the healthy WAL (~64 MiB: the default+ * 1000-page autocheckpoint at the 64 KiB page size sqlite_writer.c+ * gives every index), so normal indexing never triggers* truncate/regrow churn — it only fires after abnormal growth.* Shared/live paths do NOT use a TRUNCATE checkpoint: truncating the WAL* to zero can raise SIGBUS in a sibling process that has the DB mmap'd@@ -604,6 +615,10 @@ static int configure_pragmas(cbm_store_t *s, bool in_memory, bool read_only) {if (rc != CBM_STORE_OK) {return rc;}+ rc = exec_sql(s, ST_WRITE_CACHE_SQL);+ if (rc != CBM_STORE_OK) {+ return rc;+ }char mmap_sql[ST_BUF_64];snprintf(mmap_sql, sizeof(mmap_sql), "PRAGMA mmap_size = %lld;",(long long)cbm_store_resolve_mmap_size());@@ -1811,12 +1826,12 @@ int cbm_store_begin_bulk(cbm_store_t *s) {* because the in-memory rollback journal is lost on crash.* WAL mode is crash-safe: uncommitted WAL entries are simply discarded* on the next open. Performance is preserved via synchronous=OFF and a- * larger cache, which are safe with WAL. */+ * 64 MiB cache, which are safe with WAL. */int rc = exec_sql(s, "PRAGMA synchronous = OFF;");if (rc != CBM_STORE_OK) {return rc;}- return exec_sql(s, "PRAGMA cache_size = -65536;"); /* CBM_SZ_64 MB */+ return exec_sql(s, ST_WRITE_CACHE_SQL);}int cbm_store_end_bulk(cbm_store_t *s) {@@ -1824,7 +1839,8 @@ int cbm_store_end_bulk(cbm_store_t *s) {if (rc != CBM_STORE_OK) {return rc;}- return exec_sql(s, "PRAGMA cache_size = -2000;"); /* default ~2 MB */+ /* Back to the read-write cache, not SQLite's 2000 KiB default (#1419). */+ return exec_sql(s, ST_WRITE_CACHE_SQL);}int cbm_store_drop_indexes(cbm_store_t *s) {diff --git a/src/store/store.h b/src/store/store.hindex 06ea1fbb7..4eb0be080 100644--- a/src/store/store.h+++ b/src/store/store.h@@ -427,11 +427,12 @@ int cbm_store_rollback(cbm_store_t *s);/* ── Bulk write optimization ────────────────────────────────────── */-/* Tune pragmas for bulk write throughput (synchronous=OFF, large cache).+/* Tune pragmas for bulk write throughput (synchronous=OFF, 64 MiB cache).* WAL journal mode is preserved throughout for crash safety. */int cbm_store_begin_bulk(cbm_store_t *s);-/* Restore normal pragmas (synchronous=NORMAL, default cache) after bulk writes. */+/* Restore normal pragmas (synchronous=NORMAL, the 64 MiB read-write cache)+ * after bulk writes. */int cbm_store_end_bulk(cbm_store_t *s);/* Drop user indexes for faster bulk inserts. */diff --git a/tests/test_pipeline.c b/tests/test_pipeline.cindex 03a5cad70..55d6aa092 100644--- a/tests/test_pipeline.c+++ b/tests/test_pipeline.c@@ -2916,6 +2916,115 @@ TEST(pipeline_incremental_repoints_call_reference_without_stale_edge) {PASS();}+/* #1419: every index publishes through a private staging DB in WAL mode that+ * cbm_store_seal_for_atomic_publish() checkpoints at synchronous=FULL and+ * switches to DELETE mode before the atomic rename. A crash before that rename+ * discards the stage, so a WAL commit at FULL only adds an fsync per+ * transaction without protecting anything; SQLite documents NORMAL as+ * corruption-safe under WAL. The probe reads the EFFECTIVE synchronous level+ * of the committing handle from a WAL hook on every connection the run opens.+ *+ * SQLite installs its default autocheckpoint WAL hook after auto-extensions+ * run, so the auto-extension installs a statement trace and the trace installs+ * the probe once the connection is fully open. The probe keeps SQLite's+ * default 1000-frame autocheckpoint, so it does not change what the run+ * writes. */+enum { I1419_SYNC_FULL = 2, I1419_DEFAULT_AUTOCHECKPOINT = 1000 };+static atomic_int g_i1419_wal_commits;+static atomic_int g_i1419_full_sync_commits;++static int i1419_wal_probe(void *arg, sqlite3 *db, const char *db_name, int wal_frames) {+ (void)arg;+ atomic_fetch_add(&g_i1419_wal_commits, 1);+ sqlite3_stmt *stmt = NULL;+ if (sqlite3_prepare_v2(db, "PRAGMA synchronous;", -1, &stmt, NULL) == SQLITE_OK &&+ sqlite3_step(stmt) == SQLITE_ROW && sqlite3_column_int(stmt, 0) >= I1419_SYNC_FULL) {+ atomic_fetch_add(&g_i1419_full_sync_commits, 1);+ }+ sqlite3_finalize(stmt);+ if (wal_frames >= I1419_DEFAULT_AUTOCHECKPOINT) {+ (void)sqlite3_wal_checkpoint(db, db_name);+ }+ return SQLITE_OK;+}++static int i1419_trace(unsigned type, void *ctx, void *stmt, void *sql) {+ (void)type;+ (void)ctx;+ (void)sql;+ (void)sqlite3_wal_hook(sqlite3_db_handle((sqlite3_stmt *)stmt), i1419_wal_probe, NULL);+ return 0;+}++static int i1419_install_probe(sqlite3 *db, char **errmsg, const sqlite3_api_routines *api) {+ (void)errmsg;+ (void)api;+ return sqlite3_trace_v2(db, SQLITE_TRACE_STMT, i1419_trace, NULL);+}++static int i1419_index(const char *repo, const char *db_path) {+ cbm_pipeline_t *p = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL);+ if (!p) {+ return -1;+ }+ int rc = cbm_pipeline_run(p);+ cbm_pipeline_free(p);+ return rc;+}++TEST(pipeline_staged_publish_commits_below_full_sync_issue1419) {+ char tmp[256];+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_i1419_XXXXXX");+ if (!cbm_mkdtemp(tmp)) {+ FAIL("tmpdir");+ }+ /* Equal lengths + equal mtimes: the second run must route on content+ * bytes to the closure-repair (delta) publish, not a full rebuild. */+ static const char initial_source[] =+ "package syncprobe\n"+ "func alphaSyncTarget() {}\n"+ "func bravoSyncTarget() {}\n"+ "func syncProbeAccept(callback func()) {}\n"+ "func syncProbeSite() { syncProbeAccept(alphaSyncTarget) }\n";+ static const char replacement_source[] =+ "package syncprobe\n"+ "func alphaSyncTarget() {}\n"+ "func bravoSyncTarget() {}\n"+ "func syncProbeAccept(callback func()) {}\n"+ "func syncProbeSite() { syncProbeAccept(bravoSyncTarget) }\n";+ ASSERT_EQ(sizeof(initial_source), sizeof(replacement_source));+ write_temp_file(tmp, "probe.go", initial_source);+ char source_path[512];+ snprintf(source_path, sizeof(source_path), "%s/probe.go", tmp);+ const time_t fixed_mtime = 1700000000;+ ASSERT_EQ(pipeline_test_set_mtime(source_path, fixed_mtime, 123456789L), 0);+ char db_path[512];+ snprintf(db_path, sizeof(db_path), "%s/i1419.db", tmp);++ atomic_store(&g_i1419_wal_commits, 0);+ atomic_store(&g_i1419_full_sync_commits, 0);+ ASSERT_EQ(sqlite3_auto_extension((void (*)(void))i1419_install_probe), SQLITE_OK);+ /* Both publish routes run before any assertion, so the probe is always+ * unregistered again even when a run fails. */+ int full_rc = i1419_index(tmp, db_path);+ write_temp_file(tmp, "probe.go", replacement_source);+ int mtime_rc = pipeline_test_set_mtime(source_path, fixed_mtime, 123456789L);+ cbm_pipeline_incremental_test_reset_faults();+ int delta_rc = i1419_index(tmp, db_path);+ cbm_incremental_route_t delta_route = cbm_pipeline_incremental_test_last_route();+ (void)sqlite3_cancel_auto_extension((void (*)(void))i1419_install_probe);+ th_rmtree(tmp);++ ASSERT_EQ(full_rc, 0);+ ASSERT_EQ(mtime_rc, 0);+ ASSERT_EQ(delta_rc, 0);+ ASSERT_EQ(delta_route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);+ /* Non-vacuous: both publishes commit through the WAL. */+ ASSERT_GT(atomic_load(&g_i1419_wal_commits), 0);+ ASSERT_EQ(atomic_load(&g_i1419_full_sync_commits), 0);+ PASS();+}+/* SQL DDL becomes first-class Table/View nodes wired into FROM/JOIN lineage,* while the shared name registry must NOT leak those relations into other* languages' textual resolution: a Python call or identifier sharing the@@ -16556,6 +16665,7 @@ SUITE(pipeline) {SUITE(pipeline_semantic_manifest_repro) {RUN_TEST(incremental_downgrade_preserves_scope_and_artifact_across_change_noop_delete);RUN_TEST(pipeline_incremental_repoints_call_reference_without_stale_edge);+ RUN_TEST(pipeline_staged_publish_commits_below_full_sync_issue1419);RUN_TEST(pipeline_sql_lineage_and_relation_isolation);RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage);RUN_TEST(pipeline_dbt_jinja_lineage);diff --git a/tests/test_store_pragmas.c b/tests/test_store_pragmas.cindex 757d46363..a81d73cbd 100644--- a/tests/test_store_pragmas.c+++ b/tests/test_store_pragmas.c@@ -11,6 +11,8 @@#include "test_framework.h"#include "test_helpers.h"#include <store/store.h>+#include "sqlite3.h" /* vendored/sqlite3 — read pragmas back on the store's own handle */+#include <stdbool.h>#include <stdio.h>#include <stdlib.h>#include <string.h>@@ -98,8 +100,9 @@ TEST(journal_size_limit_bounds_wal_issue1083) {cbm_store_t *s = cbm_store_open_path(tmp_path);ASSERT(s != NULL);- /* 256 MiB — far above the healthy WAL (~4 MiB), so no truncate/regrow churn- * in normal operation; it only fires after abnormal (starved) growth. */+ /* 256 MiB — far above the healthy WAL (~64 MiB: 1000 autocheckpoint pages+ * of 64 KiB), so no truncate/regrow churn in normal operation; it only+ * fires after abnormal (starved) growth. */ASSERT(cbm_store_journal_size_limit(s) == (int64_t)268435456);cbm_store_close(s);@@ -113,6 +116,86 @@ TEST(journal_size_limit_bounds_wal_issue1083) {PASS();}+/* #1419: effective pragmas per store role, read on the handle itself —+ * synchronous is per-connection and never persisted, so an external sqlite3+ * shell reports its own default (FULL), not what the indexer runs with. */+static int role_pragma_int(cbm_store_t *s, const char *sql) {+ sqlite3_stmt *stmt = NULL;+ int value = -1;+ if (sqlite3_prepare_v2(cbm_store_get_db(s), sql, -1, &stmt, NULL) == SQLITE_OK &&+ sqlite3_step(stmt) == SQLITE_ROW) {+ value = sqlite3_column_int(stmt, 0);+ }+ sqlite3_finalize(stmt);+ return value;+}++static bool role_journal_mode_is(cbm_store_t *s, const char *want) {+ sqlite3_stmt *stmt = NULL;+ bool match = false;+ if (sqlite3_prepare_v2(cbm_store_get_db(s), "PRAGMA journal_mode;", -1, &stmt, NULL) ==+ SQLITE_OK &&+ sqlite3_step(stmt) == SQLITE_ROW) {+ const char *mode = (const char *)sqlite3_column_text(stmt, 0);+ match = mode && strcmp(mode, want) == 0;+ }+ sqlite3_finalize(stmt);+ return match;+}++TEST(store_role_pragmas_issue1419) {+ enum {+ SYNC_OFF = 0,+ SYNC_NORMAL = 1,+ SYNC_FULL = 2,+ /* 64 MiB = 1024 of the 64 KiB pages every index is written with. */+ WRITE_CACHE_KIB = -65536,+ SQLITE_DEFAULT_CACHE_KIB = -2000,+ };+ char tmp_path[256];+ snprintf(tmp_path, sizeof(tmp_path), "%s/cbm_test_roles_%d.db", cbm_tmpdir(), (int)getpid());+ unlink(tmp_path);++ /* Read-write role (live ADR writes, staging generations): WAL at NORMAL,+ * with a page cache that holds more than a few 64 KiB pages. */+ cbm_store_t *s = cbm_store_open_path(tmp_path);+ ASSERT(s != NULL);+ ASSERT_TRUE(role_journal_mode_is(s, "wal"));+ ASSERT_EQ(role_pragma_int(s, "PRAGMA synchronous;"), SYNC_NORMAL);+ ASSERT_EQ(role_pragma_int(s, "PRAGMA cache_size;"), WRITE_CACHE_KIB);+ /* Bulk role: sync off for the write burst, then back to the read-write+ * settings rather than SQLite's defaults. */+ ASSERT_EQ(cbm_store_begin_bulk(s), CBM_STORE_OK);+ ASSERT_EQ(role_pragma_int(s, "PRAGMA synchronous;"), SYNC_OFF);+ ASSERT_EQ(cbm_store_end_bulk(s), CBM_STORE_OK);+ ASSERT_EQ(role_pragma_int(s, "PRAGMA synchronous;"), SYNC_NORMAL);+ ASSERT_EQ(role_pragma_int(s, "PRAGMA cache_size;"), WRITE_CACHE_KIB);+ /* Seal role: the durable checkpoint before the atomic rename runs at FULL+ * and leaves a self-contained DELETE-mode file. */+ ASSERT_EQ(cbm_store_seal_for_atomic_publish(s), CBM_STORE_OK);+ ASSERT_EQ(role_pragma_int(s, "PRAGMA synchronous;"), SYNC_FULL);+ ASSERT_TRUE(role_journal_mode_is(s, "delete"));+ cbm_store_close(s);++ /* Query role: read-only, never switches a sealed file back to WAL, and+ * keeps SQLite's small default cache (one per request, so it stays cheap). */+ cbm_store_t *q = cbm_store_open_path_query(tmp_path);+ ASSERT(q != NULL);+ ASSERT_EQ(sqlite3_db_readonly(cbm_store_get_db(q), "main"), 1);+ ASSERT_TRUE(role_journal_mode_is(q, "delete"));+ ASSERT_EQ(role_pragma_int(q, "PRAGMA cache_size;"), SQLITE_DEFAULT_CACHE_KIB);+ cbm_store_close(q);++ unlink(tmp_path);+ char tmp_wal[300];+ char tmp_shm[300];+ snprintf(tmp_wal, sizeof(tmp_wal), "%s-wal", tmp_path);+ snprintf(tmp_shm, sizeof(tmp_shm), "%s-shm", tmp_path);+ unlink(tmp_wal);+ unlink(tmp_shm);+ PASS();+}+/* Pagination-cursor generation: minted per DB file, bumped per index run.* Same store + reads only -> stable; upsert_project (every index run's choke* point) -> changes; two distinct DB files can never share a generation even@@ -264,6 +347,7 @@ TEST(corrupt_page_scan_returns_error_not_truncation) {SUITE(store_pragmas) {RUN_TEST(journal_size_limit_bounds_wal_issue1083);+ RUN_TEST(store_role_pragmas_issue1419);RUN_TEST(store_generation_tracks_mutations);RUN_TEST(store_generation_rejects_malformed_metadata_atomically);RUN_TEST(corrupt_page_scan_returns_error_not_truncation);