Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
pdf: an object's value is read up to the next object
Martin Vogel2af58b0a3336
diff --git a/internal/cbm/pdf/pdf_obj.c b/internal/cbm/pdf/pdf_obj.cindex 229338676..25fcafaa8 100644--- a/internal/cbm/pdf/pdf_obj.c+++ b/internal/cbm/pdf/pdf_obj.c@@ -1072,6 +1072,30 @@ static bool endstream_follows(const pdf_doc_t *d, size_t q) {return false;}+static bool build_hdrs(pdf_doc_t *d);++/* Where the first object header at or after `at` starts (d->len when none):+ * an object's value is read up to there. The objects of a file do not overlap;+ * a token that ran on into the next ones (a string without its end) made every+ * object before it read the rest of the file again, n objects times the file.+ * The bytes of a stream are not bounded by this: its /Length places them. */+static size_t object_end(pdf_doc_t *d, size_t at) {+ if (!build_hdrs(d)) {+ return d->len;+ }+ int lo = 0;+ int hi = d->nhdrs;+ while (lo < hi) {+ int mid = lo + (hi - lo) / 2;+ if (d->hdrs[mid].off >= at) {+ hi = mid;+ } else {+ lo = mid + 1;+ }+ }+ return lo < d->nhdrs ? d->hdrs[lo].off : d->len;+}+/* The value of the object at off (or off + hdr), without its stream; *stream_at* is where a "stream" keyword after the value starts, 0 when none follows. NULL:* no matching header there. */@@ -1097,7 +1121,7 @@ static pdf_val_t *indirect_value_at(pdf_doc_t *d, int64_t off, bool has_expect,return NULL;}const unsigned char *s = d->data;- size_t len = d->len;+ size_t len = object_end(d, at);size_t p = at;pdf_tok_t t;pdf_lex(d, s, len, &p, &t);@@ -1343,8 +1367,40 @@ struct pdf_objstm {int64_t *ooff;int n;pdf_numidx_t *sorted; /* (onum, order), built on first need */+ int64_t *starts; /* first + each offset, ascending, built on first need */};+static int i64_cmp(const void *x, const void *y);++/* Where the member after the one at `at` starts (sm->len when none): members+ * do not overlap, as the objects of a file do not (object_end). */+static size_t member_end(pdf_doc_t *d, pdf_objstm_t *sm, size_t at) {+ if (!sm->starts) {+ sm->starts = (int64_t *)pdf_big(d, (size_t)(sm->n ? sm->n : 1) * sizeof(int64_t));+ if (!sm->starts) {+ return sm->len;+ }+ for (int i = 0; i < sm->n; i++) {+ sm->starts[i] = pdf_sat_add(sm->first, sm->ooff[i]);+ }+ qsort(sm->starts, (size_t)sm->n, sizeof(int64_t), i64_cmp);+ }+ int lo = 0;+ int hi = sm->n;+ while (lo < hi) {+ int mid = lo + (hi - lo) / 2;+ if (sm->starts[mid] > (int64_t)at) {+ hi = mid;+ } else {+ lo = mid + 1;+ }+ }+ if (lo < sm->n && (uint64_t)sm->starts[lo] < (uint64_t)sm->len) {+ return (size_t)sm->starts[lo];+ }+ return sm->len;+}+static int64_t objstm_find(pdf_doc_t *d, pdf_objstm_t *sm, int64_t num, int64_t idx) {if (idx >= 0 && idx < sm->n && sm->onum[idx] == num) {return sm->ooff[idx];@@ -1408,9 +1464,10 @@ pdf_val_t *pdf_get(pdf_doc_t *d, int64_t num) {int64_t at = off == INT64_MIN ? -1 : pdf_sat_add(sm->first, off);if (at >= 0 && (uint64_t)at <= (uint64_t)sm->len) {size_t p = (size_t)at;+ size_t end = member_end(d, sm, (size_t)at);pdf_tok_t t;- pdf_lex(d, sm->data, sm->len, &p, &t);- obj = pdf_parse_value(d, sm->data, sm->len, &p, &t, 0);+ pdf_lex(d, sm->data, end, &p, &t);+ obj = pdf_parse_value(d, sm->data, end, &p, &t, 0);}}}@@ -1987,8 +2044,11 @@ static void reconstruct(pdf_doc_t *d) {}size_t p = q;pdf_tok_t t;- pdf_lex(d, s, len, &p, &t);- pdf_val_t *v = pdf_parse_value(d, s, len, &p, &t, 0);+ /* read up to the next "trailer": scanned trailers do not overlap either */+ const unsigned char *nx = cbm_memmem(s + q, len - q, "trailer", 7);+ size_t tlen = nx ? (size_t)(nx - s) : len;+ pdf_lex(d, s, tlen, &p, &t);+ pdf_val_t *v = pdf_parse_value(d, s, tlen, &p, &t, 0);if (v && v->kind == PV_DICT) {for (int k = 0; k < v->nkv; k++) {kl_push(&trail, v->kv[k].key, v->kv[k].klen, v->kv[k].val);diff --git a/tests/test_doc_links_pdf.c b/tests/test_doc_links_pdf.cindex bdaf49e1f..bef150442 100644--- a/tests/test_doc_links_pdf.c+++ b/tests/test_doc_links_pdf.c@@ -707,6 +707,63 @@ static unsigned char *tp_xrefstm_sections(int sections, size_t *out_len) {return b.p;}+/* tp_simple's page, then `n` objects that each open a string and never close+ * it, and no cross-reference: the reader rebuilds it from the object headers. */+static unsigned char *tp_open_strings(int n, size_t *out_len) {+ tp_buf_t b = {0};+ tp_puts(&b, "%PDF-1.7\n");+ const char *content = "BT /F1 12 Tf 72 700 Td (open strings) Tj ET";+ const char *body[4] = {TP_CATALOG, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", TP_PAGE_F1_5,+ TP_HELV};+ char line[128];+ for (int i = 1; i <= 4; i++) {+ snprintf(line, sizeof(line), "%d 0 obj\n", i);+ tp_puts(&b, line);+ tp_puts(&b, body[i - 1]);+ tp_puts(&b, "\nendobj\n");+ }+ snprintf(line, sizeof(line), "5 0 obj\n<< /Length %zu >>\nstream\n", strlen(content));+ tp_puts(&b, line);+ tp_puts(&b, content);+ tp_puts(&b, "\nendstream\nendobj\n");+ for (int i = 0; i < n; i++) {+ snprintf(line, sizeof(line), "%d 0 obj\n(", 6 + i);+ tp_puts(&b, line);+ for (int k = 0; k < 100; k++) {+ tp_puts(&b, "a");+ }+ tp_puts(&b, "\n");+ }+ *out_len = b.n;+ return b.p;+}++/* Objects whose strings never end cost what the file costs: each object's+ * value is read up to the next object, not to the end of the file again (a+ * thousand objects asked for about 60 MB). */+TEST(pdf_extract_open_strings_linear) {+ size_t len;+ tp_text_t t;+ unsigned char *pdf = tp_open_strings(1000, &len);+ cbm_mem_class_reset_peaks();+ size_t arena0 = cbm_mem_class_peak_bytes(CBM_MEM_CLASS_ARENA);+ size_t extract0 = cbm_mem_class_peak_bytes(CBM_MEM_CLASS_EXTRACT);+ ASSERT_TRUE(tp_extract(pdf, len, &t));+ size_t grew = (cbm_mem_class_peak_bytes(CBM_MEM_CLASS_ARENA) - arena0) ++ (cbm_mem_class_peak_bytes(CBM_MEM_CLASS_EXTRACT) - extract0);+ ASSERT_EQ(t.r.status, CBM_PDF_OK);+ ASSERT_EQ(t.r.counts.xref_reconstructed, 1);+ ASSERT_EQ(t.r.npages, 1);+ ASSERT_STR_EQ(t.r.pages[0].text, "open strings");+ if (grew > 16 * len + (1U << 20)) {+ fprintf(stderr, "open strings: %zu bytes for a %zu-byte file\n", grew, len);+ }+ ASSERT_TRUE(grew <= 16 * len + (1U << 20));+ cbm_arena_destroy(&t.arena);+ free(pdf);+ PASS();+}+/* A thousand sections naming one /XRefStm decode it once: a later (older)* section adds nothing, and every copy used to stay until the document closed. */TEST(pdf_extract_xrefstm_once) {@@ -1059,6 +1116,7 @@ SUITE(doc_links_pdf) {RUN_TEST(pdf_extract_filters_and_hostile);RUN_TEST(pdf_extract_object_chains);RUN_TEST(pdf_extract_xrefstm_once);+ RUN_TEST(pdf_extract_open_strings_linear);RUN_TEST(pdf_scan_mentions);RUN_TEST(pdf_links_pipeline);RUN_TEST(pdf_snippet_is_page_text);