Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(store): use 64-bit file offsets in the SQLite page writer

lixsp11104dc21b045d
diff --git a/internal/cbm/sqlite_writer.c b/internal/cbm/sqlite_writer.c
index f965da1ce..d684ff3a0 100644
--- a/internal/cbm/sqlite_writer.c
+++ b/internal/cbm/sqlite_writer.c
@@ -35,13 +35,22 @@
#include <process.h>
#include <windows.h>
#define cbm_writer_getpid _getpid
+/* long is 32 bits on Windows, so fseek/ftell cannot address past 2 GiB there. */
+#define cbm_writer_seek(fp, off, whence) _fseeki64((fp), (long long)(off), (whence))
+#define cbm_writer_tell(fp) ((int64_t)_ftelli64(fp))
#else
#include <unistd.h>
#define cbm_writer_getpid getpid
+#define cbm_writer_seek(fp, off, whence) fseeko((fp), (off_t)(off), (whence))
+#define cbm_writer_tell(fp) ((int64_t)ftello(fp))
#endif
#define CBM_PAGE_SIZE 65536
+int64_t cbm_writer_page_offset(uint32_t page_num) {
+ return (int64_t)(page_num - SKIP_ONE) * CBM_PAGE_SIZE;
+}
+
/* SQLite reserves the page containing the 1 GiB file offset (the "pending byte"
* used for file locking on Windows). This page MUST be skipped during allocation
* otherwise integrity_check reports "2nd reference to page N" because it marks
@@ -601,8 +610,8 @@ static void pb_flush_leaf(PageBuilder *pb) {
// Write page to file. Skip the pending byte page (SQLite reserved).
pb->next_page = cbm_skip_pending_byte(pb->next_page);
uint32_t page_num = pb->next_page;
- long offset = (long)(page_num - SKIP_ONE) * CBM_PAGE_SIZE;
- (void)fseek(pb->fp, offset, SEEK_SET);
+ int64_t offset = cbm_writer_page_offset(page_num);
+ (void)cbm_writer_seek(pb->fp, offset, SEEK_SET);
(void)fwrite(pb->page, SKIP_ONE, CBM_PAGE_SIZE, pb->fp);
// Record this leaf for interior page building
@@ -707,7 +716,7 @@ static int write_interior_page(PageBuilder *pb, uint8_t *page, int cell_count, i
page[HDR_FRAGBYTES_OFF] = 0;
put_u32(page + HDR_RIGHTCHILD_OFF, right_child_page);
- (void)fseek(pb->fp, (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE, SEEK_SET);
+ (void)cbm_writer_seek(pb->fp, cbm_writer_page_offset(pnum), SEEK_SET);
(void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, pb->fp);
if (parent_count >= *parent_cap) {
@@ -970,7 +979,7 @@ static uint32_t write_overflow_pages(FILE *fp, uint32_t *next_page, const uint8_
int data_len) {
int per_page = CBM_PAGE_SIZE - BTREE_PTR_SIZE;
uint32_t first_page = 0;
- long prev_next_ptr_offset = -SKIP_ONE;
+ int64_t prev_next_ptr_offset = -SKIP_ONE;
int offset = 0;
while (offset < data_len) {
@@ -983,7 +992,7 @@ static uint32_t write_overflow_pages(FILE *fp, uint32_t *next_page, const uint8_
if (prev_next_ptr_offset >= 0) {
uint8_t ptr[BTREE_PTR_SIZE];
put_u32(ptr, pnum);
- (void)fseek(fp, prev_next_ptr_offset, SEEK_SET);
+ (void)cbm_writer_seek(fp, prev_next_ptr_offset, SEEK_SET);
(void)fwrite(ptr, SKIP_ONE, BTREE_PTR_SIZE, fp);
}
@@ -997,9 +1006,9 @@ static uint32_t write_overflow_pages(FILE *fp, uint32_t *next_page, const uint8_
put_u32(page, 0); // next-page pointer — 0 for now, backpatched on next iteration
memcpy(page + BTREE_PTR_SIZE, data + offset, chunk);
- long page_offset = (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE;
+ int64_t page_offset = cbm_writer_page_offset(pnum);
prev_next_ptr_offset = page_offset;
- (void)fseek(fp, page_offset, SEEK_SET);
+ (void)cbm_writer_seek(fp, page_offset, SEEK_SET);
(void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, fp);
offset += chunk;
@@ -1276,7 +1285,7 @@ static uint32_t write_table_btree(FILE *fp, uint32_t *next_page, const uint8_t *
put_u16(page + hdr + HDR_CELLCOUNT_OFF, 0); // 0 cells
put_u16(page + hdr + HDR_CONTENT_OFF, (uint16_t)CBM_PAGE_SIZE); // content at end of page
page[hdr + HDR_FRAGBYTES_OFF] = 0; // 0 fragmented bytes
- (void)fseek(fp, (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE, SEEK_SET);
+ (void)cbm_writer_seek(fp, cbm_writer_page_offset(pnum), SEEK_SET);
(void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, fp);
return pnum;
}
@@ -1326,7 +1335,7 @@ static uint32_t write_empty_index_leaf(FILE *fp, uint32_t *next_page) {
put_u16(page + HDR_CELLCOUNT_OFF, 0);
put_u16(page + HDR_CONTENT_OFF, (uint16_t)CBM_PAGE_SIZE);
page[HDR_FRAGBYTES_OFF] = 0;
- (void)fseek(fp, (long)(pnum - SKIP_ONE) * CBM_PAGE_SIZE, SEEK_SET);
+ (void)cbm_writer_seek(fp, cbm_writer_page_offset(pnum), SEEK_SET);
(void)fwrite(page, SKIP_ONE, CBM_PAGE_SIZE, fp);
return pnum;
}
@@ -2020,7 +2029,7 @@ static int write_master_page1(FILE *fp, MasterEntry *master, int master_count, u
write_sqlite_file_header(page1, next_page - SKIP_ONE);
- (void)fseek(fp, 0, SEEK_SET);
+ (void)cbm_writer_seek(fp, 0, SEEK_SET);
(void)fwrite(page1, SKIP_ONE, CBM_PAGE_SIZE, fp);
for (int i = 0; i < master_count; i++) {
@@ -2034,12 +2043,12 @@ static int write_master_page1(FILE *fp, MasterEntry *master, int master_count, u
/* Pad file to exact page boundary. */
static void pad_file_to_page_boundary(FILE *fp, uint32_t next_page) {
- (void)fseek(fp, 0, SEEK_END);
- long file_size = ftell(fp);
- long expected_size = (long)(next_page - SKIP_ONE) * CBM_PAGE_SIZE;
+ (void)cbm_writer_seek(fp, 0, SEEK_END);
+ int64_t file_size = cbm_writer_tell(fp);
+ int64_t expected_size = cbm_writer_page_offset(next_page);
if (file_size < expected_size) {
uint8_t zero = 0;
- (void)fseek(fp, expected_size - SKIP_ONE, SEEK_SET);
+ (void)cbm_writer_seek(fp, expected_size - SKIP_ONE, SEEK_SET);
(void)fwrite(&zero, SKIP_ONE, SKIP_ONE, fp);
}
}
diff --git a/internal/cbm/sqlite_writer.h b/internal/cbm/sqlite_writer.h
index 1142e7d57..b92a13719 100644
--- a/internal/cbm/sqlite_writer.h
+++ b/internal/cbm/sqlite_writer.h
@@ -90,4 +90,8 @@ int cbm_writer_finalize(cbm_db_writer_t *w, const char *project, const char *roo
CBMDumpEdge *edges, int edge_count, CBMDumpVector *vectors,
int vector_count, CBMDumpTokenVec *token_vecs, int token_vec_count);
+// Byte offset of 1-based page `page_num` in the output file. 64-bit on every
+// platform, so databases larger than 2 GiB are addressed correctly on Windows.
+int64_t cbm_writer_page_offset(uint32_t page_num);
+
#endif // CBM_SQLITE_WRITER_H
diff --git a/tests/test_sqlite_writer.c b/tests/test_sqlite_writer.c
index bacf06a95..cbeb2dfc3 100644
--- a/tests/test_sqlite_writer.c
+++ b/tests/test_sqlite_writer.c
@@ -996,6 +996,18 @@ TEST(sw_publish_preserves_live_reader) {
/* ── Suite ─────────────────────────────────────────────────────── */
+TEST(sw_page_offsets_do_not_wrap_past_4gib) {
+ /* A 32-bit long offset wraps negative at 2 GiB and back to 0 at 4 GiB: on
+ * Windows a staging DB stopped at exactly 4 GiB with its first pages
+ * overwritten, and publish failed with persist_failed (#2399). */
+ ASSERT_EQ(cbm_writer_page_offset(1), 0);
+ ASSERT_EQ(cbm_writer_page_offset(2), 65536);
+ ASSERT_EQ(cbm_writer_page_offset(32769), INT64_C(2147483648));
+ ASSERT_EQ(cbm_writer_page_offset(65537), INT64_C(4294967296));
+ ASSERT_EQ(cbm_writer_page_offset(UINT32_MAX), (int64_t)(UINT32_MAX - 1) * 65536);
+ PASS();
+}
+
TEST(sw_open_truncated_path_names_its_reason) {
/* final_path is a fixed 4K buffer and make_writer_temp_path only fails by
* truncation. Neither sets an errno of its own, so without naming one the
@@ -1065,6 +1077,7 @@ SUITE(sqlite_writer) {
RUN_TEST(sw_publish_failure_preserves_destination_sidecars);
RUN_TEST(sw_publish_supports_non_ascii_path);
RUN_TEST(sw_publish_preserves_live_reader);
+ RUN_TEST(sw_page_offsets_do_not_wrap_past_4gib);
RUN_TEST(sw_open_truncated_path_names_its_reason);
RUN_TEST(sw_publish_failure_reports_the_rename_not_the_cleanup);
}