Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

pdf: a page's docstring keeps 256 KiB of its text and says so

Martin Vogelf403f921dd41
diff --git a/internal/cbm/doc_pdf.c b/internal/cbm/doc_pdf.c
index 586417b2e..b3ca4a4f3 100644
--- a/internal/cbm/doc_pdf.c
+++ b/internal/cbm/doc_pdf.c
@@ -45,6 +45,7 @@ enum {
PDF_PAGE_NAME = 32,
PDF_LIST_MIN = 64,
PDF_U8_MAX = 4,
+ PDF_PAGE_DOC_MAX = 256 * 1024, /* bytes of a page's text kept as its docstring */
};
#define PDF_JOIN_SEP '\x1f' /* between a join's forms */
@@ -1321,6 +1322,23 @@ static bool fixture_pdf(const char *rel_path) {
}
}
+/* A page's docstring (its snippet): the page text, up to PDF_PAGE_DOC_MAX bytes
+ * cut on a character boundary and then said so. A page holds a few kilobytes;
+ * only a crafted text layer reaches the bound, and the mentions are still read
+ * from the whole text. */
+static const char *page_doc(CBMArena *a, const cbm_pdf_page_t *pg) {
+ if (pg->len <= PDF_PAGE_DOC_MAX) {
+ return pg->text;
+ }
+ size_t cut = PDF_PAGE_DOC_MAX;
+ while (cut > 0 && ((unsigned char)pg->text[cut] & 0xC0U) == 0x80U) {
+ cut--; /* not inside a UTF-8 sequence */
+ }
+ const char *doc = cbm_arena_sprintf(a, "%.*s\n[page text cut at %zu of %zu bytes]", (int)cut,
+ pg->text, cut, pg->len);
+ return doc ? doc : pg->text;
+}
+
void cbm_pdf_extract_document(CBMExtractCtx *ctx) {
CBMFileResult *result = ctx->result;
CBMArena *a = ctx->arena;
@@ -1364,7 +1382,7 @@ void cbm_pdf_extract_document(CBMExtractCtx *ctx) {
def.start_line = page;
def.end_line = page;
def.is_exported = true;
- def.docstring = pr.pages[i].len ? pr.pages[i].text : NULL;
+ def.docstring = pr.pages[i].len ? page_doc(a, &pr.pages[i]) : NULL;
def.extra_props = kv;
cbm_defs_push(&result->defs, a, def);
if (!fixture && pr.pages[i].len) {
diff --git a/tests/test_doc_links_pdf.c b/tests/test_doc_links_pdf.c
index ee4e77996..ff96cec01 100644
--- a/tests/test_doc_links_pdf.c
+++ b/tests/test_doc_links_pdf.c
@@ -1133,6 +1133,48 @@ TEST(pdf_links_pipeline) {
PASS();
}
+/* A page's docstring keeps 256 KiB of its text and says where it was cut; a
+ * page's mentions still come from the whole text. */
+TEST(pdf_page_doc_bounded) {
+ enum { WORDS = 45000 }; /* 315 KB of text */
+ tp_buf_t c = {0};
+ tp_puts(&c, "BT /F1 12 Tf 72 700 Td (");
+ for (int i = 0; i < WORDS; i++) {
+ tp_puts(&c, "filler ");
+ }
+ tp_puts(&c, "src/tail/last.py) Tj ET");
+ tp_obj_t o[5] = {
+ {TP_CATALOG, NULL, 0}, {"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, 0},
+ {TP_PAGE_F1_5, NULL, 0}, {TP_HELV, NULL, 0},
+ {"", c.p, c.n},
+ };
+ size_t len;
+ unsigned char *pdf = tp_pdf(o, 5, NULL, &len);
+ CBMFileResult *r =
+ cbm_extract_file((const char *)pdf, (int)len, CBM_LANG_PDF, "p", "doc.pdf", 0, NULL, NULL);
+ ASSERT_NOT_NULL(r);
+ const CBMDefinition *page = NULL;
+ for (int i = 0; i < r->defs.count; i++) {
+ if (strcmp(r->defs.items[i].label, "Section") == 0) {
+ page = &r->defs.items[i];
+ }
+ }
+ ASSERT_NOT_NULL(page);
+ ASSERT_NOT_NULL(page->docstring);
+ size_t dl = strlen(page->docstring);
+ ASSERT_TRUE(dl > 256 * 1024 && dl < 256 * 1024 + 128);
+ ASSERT_NOT_NULL(strstr(page->docstring, "[page text cut at 262144 of "));
+ bool tail = false;
+ for (int i = 0; i < r->doc_links.count; i++) {
+ tail = tail || strcmp(r->doc_links.items[i].raw, "src/tail/last.py") == 0;
+ }
+ ASSERT_TRUE(tail);
+ cbm_free_result(r);
+ free(pdf);
+ free(c.p);
+ PASS();
+}
+
TEST(pdf_snippet_is_page_text) {
char tmp[256] = "/tmp/cbm_dlpdf_snip_XXXXXX";
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
@@ -1222,6 +1264,7 @@ SUITE(doc_links_pdf) {
RUN_TEST(pdf_scan_mentions);
RUN_TEST(pdf_scan_join_linear);
RUN_TEST(pdf_links_pipeline);
+ RUN_TEST(pdf_page_doc_bounded);
RUN_TEST(pdf_snippet_is_page_text);
RUN_TEST(pdf_links_incremental);
}