Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
pdf: a page's docstring keeps 256 KiB of its text and says so
Martin Vogelf403f921dd41
diff --git a/internal/cbm/doc_pdf.c b/internal/cbm/doc_pdf.cindex 586417b2e..b3ca4a4f3 100644--- a/internal/cbm/doc_pdf.c+++ b/internal/cbm/doc_pdf.c@@ -45,6 +45,7 @@ enum {PDF_PAGE_NAME = 32,PDF_LIST_MIN = 64,PDF_U8_MAX = 4,+ PDF_PAGE_DOC_MAX = 256 * 1024, /* bytes of a page's text kept as its docstring */};#define PDF_JOIN_SEP '\x1f' /* between a join's forms */@@ -1321,6 +1322,23 @@ static bool fixture_pdf(const char *rel_path) {}}+/* A page's docstring (its snippet): the page text, up to PDF_PAGE_DOC_MAX bytes+ * cut on a character boundary and then said so. A page holds a few kilobytes;+ * only a crafted text layer reaches the bound, and the mentions are still read+ * from the whole text. */+static const char *page_doc(CBMArena *a, const cbm_pdf_page_t *pg) {+ if (pg->len <= PDF_PAGE_DOC_MAX) {+ return pg->text;+ }+ size_t cut = PDF_PAGE_DOC_MAX;+ while (cut > 0 && ((unsigned char)pg->text[cut] & 0xC0U) == 0x80U) {+ cut--; /* not inside a UTF-8 sequence */+ }+ const char *doc = cbm_arena_sprintf(a, "%.*s\n[page text cut at %zu of %zu bytes]", (int)cut,+ pg->text, cut, pg->len);+ return doc ? doc : pg->text;+}+void cbm_pdf_extract_document(CBMExtractCtx *ctx) {CBMFileResult *result = ctx->result;CBMArena *a = ctx->arena;@@ -1364,7 +1382,7 @@ void cbm_pdf_extract_document(CBMExtractCtx *ctx) {def.start_line = page;def.end_line = page;def.is_exported = true;- def.docstring = pr.pages[i].len ? pr.pages[i].text : NULL;+ def.docstring = pr.pages[i].len ? page_doc(a, &pr.pages[i]) : NULL;def.extra_props = kv;cbm_defs_push(&result->defs, a, def);if (!fixture && pr.pages[i].len) {diff --git a/tests/test_doc_links_pdf.c b/tests/test_doc_links_pdf.cindex ee4e77996..ff96cec01 100644--- a/tests/test_doc_links_pdf.c+++ b/tests/test_doc_links_pdf.c@@ -1133,6 +1133,48 @@ TEST(pdf_links_pipeline) {PASS();}+/* A page's docstring keeps 256 KiB of its text and says where it was cut; a+ * page's mentions still come from the whole text. */+TEST(pdf_page_doc_bounded) {+ enum { WORDS = 45000 }; /* 315 KB of text */+ tp_buf_t c = {0};+ tp_puts(&c, "BT /F1 12 Tf 72 700 Td (");+ for (int i = 0; i < WORDS; i++) {+ tp_puts(&c, "filler ");+ }+ tp_puts(&c, "src/tail/last.py) Tj ET");+ tp_obj_t o[5] = {+ {TP_CATALOG, NULL, 0}, {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0},+ {TP_PAGE_F1_5, NULL, 0}, {TP_HELV, NULL, 0},+ {"", c.p, c.n},+ };+ size_t len;+ unsigned char *pdf = tp_pdf(o, 5, NULL, &len);+ CBMFileResult *r =+ cbm_extract_file((const char *)pdf, (int)len, CBM_LANG_PDF, "p", "doc.pdf", 0, NULL, NULL);+ ASSERT_NOT_NULL(r);+ const CBMDefinition *page = NULL;+ for (int i = 0; i < r->defs.count; i++) {+ if (strcmp(r->defs.items[i].label, "Section") == 0) {+ page = &r->defs.items[i];+ }+ }+ ASSERT_NOT_NULL(page);+ ASSERT_NOT_NULL(page->docstring);+ size_t dl = strlen(page->docstring);+ ASSERT_TRUE(dl > 256 * 1024 && dl < 256 * 1024 + 128);+ ASSERT_NOT_NULL(strstr(page->docstring, "[page text cut at 262144 of "));+ bool tail = false;+ for (int i = 0; i < r->doc_links.count; i++) {+ tail = tail || strcmp(r->doc_links.items[i].raw, "src/tail/last.py") == 0;+ }+ ASSERT_TRUE(tail);+ cbm_free_result(r);+ free(pdf);+ free(c.p);+ PASS();+}+TEST(pdf_snippet_is_page_text) {char tmp[256] = "/tmp/cbm_dlpdf_snip_XXXXXX";ASSERT_NOT_NULL(cbm_mkdtemp(tmp));@@ -1222,6 +1264,7 @@ SUITE(doc_links_pdf) {RUN_TEST(pdf_scan_mentions);RUN_TEST(pdf_scan_join_linear);RUN_TEST(pdf_links_pipeline);+ RUN_TEST(pdf_page_doc_bounded);RUN_TEST(pdf_snippet_is_page_text);RUN_TEST(pdf_links_incremental);}