Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(store): use 64-bit file offsets in the SQLite page writer
lixsp11104dc21b045d
diff --git a/internal/cbm/sqlite_writer.c b/internal/cbm/sqlite_writer.cindex f965da1ce..d684ff3a0 100644--- a/internal/cbm/sqlite_writer.c+++ b/internal/cbm/sqlite_writer.c@@ -35,13 +35,22 @@#include <process.h>#include <windows.h>#define cbm_writer_getpid _getpid+/* long is 32 bits on Windows, so fseek/ftell cannot address past 2 GiB there. */+#define cbm_writer_seek(fp, off, whence) _fseeki64((fp), (long long)(off), (whence))+#define cbm_writer_tell(fp) ((int64_t)_ftelli64(fp))#else#include <unistd.h>#define cbm_writer_getpid getpid+#define cbm_writer_seek(fp, off, whence) fseeko((fp), (off_t)(off), (whence))+#define cbm_writer_tell(fp) ((int64_t)ftello(fp))#endif#define CBM_PAGE_SIZE 65536+int64_t cbm_writer_page_offset(uint32_t page_num) {+ return (int64_t)(page_num - SKIP_ONE) * CBM_PAGE_SIZE;+}+/* SQLite reserves the page containing the 1 GiB file offset (the "pending byte"* used for file locking on Windows). This page MUST be skipped during allocation* otherwise integrity_check reports "2nd reference to page N" because it marks@@ -601,8 +610,8 @@ static void pb_flush_leaf(PageBuilder *pb) {// Write page to file. Skip the pending byte page (SQLite reserved).pb->next_page = cbm_skip_pending_byte(pb->next_page);uint32_t page_num = pb->next_page;- long offset = (long)(page_num - SKIP_ONE) * CBM_PAGE_SIZE;- (void)fseek(pb->fp, offset, SEEK_SET);+ int64_t offset = cbm_writer_page_offset(page_num);+ (void)cbm_writer_seek(pb->fp, offset, SEEK_SET);(void)fwrite(pb->page, SKIP_ONE, CBM_PAGE_SIZE, pb->fp);// Record this leaf for interior page building@@ -707,7 +716,7 @@ static int write_interior_page(PageBuilder *pb, uint8_t *page, int cell_count, ipage[HDR_FRAGBYTES_OFF] = 0;put_u32(page + HDR_RIGHTCHILD_OFF, right_child_page);- (void)fseek(pb->fp, (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE, SEEK_SET);+ (void)cbm_writer_seek(pb->fp, cbm_writer_page_offset(pnum), SEEK_SET);(void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, pb->fp);if (parent_count >= *parent_cap) {@@ -970,7 +979,7 @@ static uint32_t write_overflow_pages(FILE *fp, uint32_t *next_page, const uint8_int data_len) {int per_page = CBM_PAGE_SIZE - BTREE_PTR_SIZE;uint32_t first_page = 0;- long prev_next_ptr_offset = -SKIP_ONE;+ int64_t prev_next_ptr_offset = -SKIP_ONE;int offset = 0;while (offset < data_len) {@@ -983,7 +992,7 @@ static uint32_t write_overflow_pages(FILE *fp, uint32_t *next_page, const uint8_if (prev_next_ptr_offset >= 0) {uint8_t ptr[BTREE_PTR_SIZE];put_u32(ptr, pnum);- (void)fseek(fp, prev_next_ptr_offset, SEEK_SET);+ (void)cbm_writer_seek(fp, prev_next_ptr_offset, SEEK_SET);(void)fwrite(ptr, SKIP_ONE, BTREE_PTR_SIZE, fp);}@@ -997,9 +1006,9 @@ static uint32_t write_overflow_pages(FILE *fp, uint32_t *next_page, const uint8_put_u32(page, 0); // next-page pointer — 0 for now, backpatched on next iterationmemcpy(page + BTREE_PTR_SIZE, data + offset, chunk);- long page_offset = (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE;+ int64_t page_offset = cbm_writer_page_offset(pnum);prev_next_ptr_offset = page_offset;- (void)fseek(fp, page_offset, SEEK_SET);+ (void)cbm_writer_seek(fp, page_offset, SEEK_SET);(void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, fp);offset += chunk;@@ -1276,7 +1285,7 @@ static uint32_t write_table_btree(FILE *fp, uint32_t *next_page, const uint8_t *put_u16(page + hdr + HDR_CELLCOUNT_OFF, 0); // 0 cellsput_u16(page + hdr + HDR_CONTENT_OFF, (uint16_t)CBM_PAGE_SIZE); // content at end of pagepage[hdr + HDR_FRAGBYTES_OFF] = 0; // 0 fragmented bytes- (void)fseek(fp, (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE, SEEK_SET);+ (void)cbm_writer_seek(fp, cbm_writer_page_offset(pnum), SEEK_SET);(void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, fp);return pnum;}@@ -1326,7 +1335,7 @@ static uint32_t write_empty_index_leaf(FILE *fp, uint32_t *next_page) {put_u16(page + HDR_CELLCOUNT_OFF, 0);put_u16(page + HDR_CONTENT_OFF, (uint16_t)CBM_PAGE_SIZE);page[HDR_FRAGBYTES_OFF] = 0;- (void)fseek(fp, (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE, SEEK_SET);+ (void)cbm_writer_seek(fp, cbm_writer_page_offset(pnum), SEEK_SET);(void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, fp);return pnum;}@@ -2020,7 +2029,7 @@ static int write_master_page1(FILE *fp, MasterEntry *master, int master_count, uwrite_sqlite_file_header(page1, next_page - SKIP_ONE);- (void)fseek(fp, 0, SEEK_SET);+ (void)cbm_writer_seek(fp, 0, SEEK_SET);(void)fwrite(page1, SKIP_ONE, CBM_PAGE_SIZE, fp);for (int i = 0; i < master_count; i++) {@@ -2034,12 +2043,12 @@ static int write_master_page1(FILE *fp, MasterEntry *master, int master_count, u/* Pad file to exact page boundary. */static void pad_file_to_page_boundary(FILE *fp, uint32_t next_page) {- (void)fseek(fp, 0, SEEK_END);- long file_size = ftell(fp);- long expected_size = (long)(next_page - SKIP_ONE) * CBM_PAGE_SIZE;+ (void)cbm_writer_seek(fp, 0, SEEK_END);+ int64_t file_size = cbm_writer_tell(fp);+ int64_t expected_size = cbm_writer_page_offset(next_page);if (file_size < expected_size) {uint8_t zero = 0;- (void)fseek(fp, expected_size - SKIP_ONE, SEEK_SET);+ (void)cbm_writer_seek(fp, expected_size - SKIP_ONE, SEEK_SET);(void)fwrite(&zero, SKIP_ONE, SKIP_ONE, fp);}}diff --git a/internal/cbm/sqlite_writer.h b/internal/cbm/sqlite_writer.hindex 1142e7d57..b92a13719 100644--- a/internal/cbm/sqlite_writer.h+++ b/internal/cbm/sqlite_writer.h@@ -90,4 +90,8 @@ int cbm_writer_finalize(cbm_db_writer_t *w, const char *project, const char *rooCBMDumpEdge *edges, int edge_count, CBMDumpVector *vectors,int vector_count, CBMDumpTokenVec *token_vecs, int token_vec_count);+// Byte offset of 1-based page `page_num` in the output file. 64-bit on every+// platform, so databases larger than 2 GiB are addressed correctly on Windows.+int64_t cbm_writer_page_offset(uint32_t page_num);+#endif // CBM_SQLITE_WRITER_Hdiff --git a/tests/test_sqlite_writer.c b/tests/test_sqlite_writer.cindex bacf06a95..cbeb2dfc3 100644--- a/tests/test_sqlite_writer.c+++ b/tests/test_sqlite_writer.c@@ -996,6 +996,18 @@ TEST(sw_publish_preserves_live_reader) {/* ── Suite ─────────────────────────────────────────────────────── */+TEST(sw_page_offsets_do_not_wrap_past_4gib) {+ /* A 32-bit long offset wraps negative at 2 GiB and back to 0 at 4 GiB: on+ * Windows a staging DB stopped at exactly 4 GiB with its first pages+ * overwritten, and publish failed with persist_failed (#2399). */+ ASSERT_EQ(cbm_writer_page_offset(1), 0);+ ASSERT_EQ(cbm_writer_page_offset(2), 65536);+ ASSERT_EQ(cbm_writer_page_offset(32769), INT64_C(2147483648));+ ASSERT_EQ(cbm_writer_page_offset(65537), INT64_C(4294967296));+ ASSERT_EQ(cbm_writer_page_offset(UINT32_MAX), (int64_t)(UINT32_MAX - 1) * 65536);+ PASS();+}+TEST(sw_open_truncated_path_names_its_reason) {/* final_path is a fixed 4K buffer and make_writer_temp_path only fails by* truncation. Neither sets an errno of its own, so without naming one the@@ -1065,6 +1077,7 @@ SUITE(sqlite_writer) {RUN_TEST(sw_publish_failure_preserves_destination_sidecars);RUN_TEST(sw_publish_supports_non_ascii_path);RUN_TEST(sw_publish_preserves_live_reader);+ RUN_TEST(sw_page_offsets_do_not_wrap_past_4gib);RUN_TEST(sw_open_truncated_path_names_its_reason);RUN_TEST(sw_publish_failure_reports_the_rename_not_the_cleanup);}