Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
pdf: a cross-reference stream named by many sections is decoded once
Martin Vogela21fa8af8836
diff --git a/internal/cbm/pdf/pdf.h b/internal/cbm/pdf/pdf.hindex 98f1e834b..4cfe1b4da 100644--- a/internal/cbm/pdf/pdf.h+++ b/internal/cbm/pdf/pdf.h@@ -39,6 +39,7 @@ typedef struct {int xref_offset_fixed; /* an xref section found near a wrong offset */int stream_length_recovered; /* a stream's /Length was wrong */int objstm_nested; /* opening an object stream needed an object of one */+ int xref_streams; /* cross-reference streams decoded */int flate_recovered; /* a damaged Flate stream decoded in part */int stream_guard_hits; /* a decoded stream cut at 512 MiB */int filters_unsupported; /* a stream with a filter this reader does not decode */diff --git a/internal/cbm/pdf/pdf_obj.c b/internal/cbm/pdf/pdf_obj.cindex 22ab40491..229338676 100644--- a/internal/cbm/pdf/pdf_obj.c+++ b/internal/cbm/pdf/pdf_obj.c@@ -1643,6 +1643,58 @@ static bool num_to_int(const pdf_val_t *v, int64_t *out) {return true;}+static const unsigned char *decode_filters(pdf_doc_t *d, const pdf_stream_t *st, size_t *len,+ pdf_buf_t *held_out);++/* The rows of a cross-reference stream's decoded bytes into sec. */+static bool xref_rows(pdf_doc_t *d, pdf_xref_t *sec, const unsigned char *raw, size_t rlen,+ const int64_t w[3], pdf_val_t **items, int nitems, const int64_t pairs[2]) {+ int64_t rec = w[0] + w[1] + w[2];+ if (rec == 0) {+ return false;+ }+ size_t pos = 0;+ for (int j = 0; j + 1 < nitems; j += 2) {+ int64_t start;+ int64_t count;+ if (items) {+ if (!num_to_int(items[j], &start) || !num_to_int(items[j + 1], &count)) {+ return false;+ }+ } else {+ start = pairs[0];+ count = pairs[1];+ }+ for (int64_t i = 0; i < count; i++) {+ if (pos + (size_t)rec > rlen) {+ break;+ }+ const unsigned char *r = raw + pos;+ int64_t t = w[0] ? be_field(r, (int)w[0]) : 1;+ int64_t f2 = be_field(r + w[0], (int)w[1]);+ int64_t f3 = w[2] ? be_field(r + w[0] + w[1], (int)w[2]) : 0;+ pos += (size_t)rec;+ bool fresh;+ pdf_xent_t *e = xref_slot(d, sec, pdf_sat_add(start, i), &fresh);+ if (!e) {+ return false;+ }+ if (!fresh) {+ continue;+ }+ if (t == 1 || t == 2) {+ e->type = (uint8_t)t;+ e->a = f2;+ e->b = f3;+ }+ }+ }+ return true;+}++/* A cross-reference stream at off into sec. Its decoded bytes are freed once+ * read: a stream parsed here is no cached object, and keeping them made every+ * section that names one stream add a copy until the document closed. */static bool xref_stream_at(pdf_doc_t *d, int64_t off, pdf_xref_t *sec, pdf_val_t **tr) {bool ok;pdf_val_t *v = parse_indirect_at(d, off, false, 0, &ok);@@ -1682,50 +1734,17 @@ static bool xref_stream_at(pdf_doc_t *d, int64_t off, pdf_xref_t *sec, pdf_val_treturn false;}size_t rlen = 0;- const unsigned char *raw = pdf_stream_decode(d, v->stream, &rlen);+ pdf_buf_t held = {0};+ const unsigned char *raw = decode_filters(d, v->stream, &rlen, &held);if (!raw) {return false;}- int64_t rec = w[0] + w[1] + w[2];- if (rec == 0) {+ d->counts->xref_streams++;+ bool rows_ok = xref_rows(d, sec, raw, rlen, w, items, nitems, pairs);+ buf_free(&held);+ if (!rows_ok) {return false;}- size_t pos = 0;- for (int j = 0; j + 1 < nitems; j += 2) {- int64_t start;- int64_t count;- if (items) {- if (!num_to_int(items[j], &start) || !num_to_int(items[j + 1], &count)) {- return false;- }- } else {- start = pairs[0];- count = pairs[1];- }- for (int64_t i = 0; i < count; i++) {- if (pos + (size_t)rec > rlen) {- break;- }- const unsigned char *r = raw + pos;- int64_t t = w[0] ? be_field(r, (int)w[0]) : 1;- int64_t f2 = be_field(r + w[0], (int)w[1]);- int64_t f3 = w[2] ? be_field(r + w[0] + w[1], (int)w[2]) : 0;- pos += (size_t)rec;- bool fresh;- pdf_xent_t *e = xref_slot(d, sec, pdf_sat_add(start, i), &fresh);- if (!e) {- return false;- }- if (!fresh) {- continue;- }- if (t == 1 || t == 2) {- e->type = (uint8_t)t;- e->a = f2;- e->b = f3;- }- }- }*tr = dict;return true;}@@ -1811,6 +1830,9 @@ static bool load_xref(pdf_doc_t *d) {return false;}pdf_map_t seen = {0};+ /* /XRefStm targets already merged: a later (older) section naming one again+ * adds nothing, since every number it holds is in d->xref already */+ pdf_map_t stm_seen = {0};kv_list_t trail = {0};bool first = true;bool ok = true;@@ -1842,7 +1864,14 @@ static bool load_xref(pdf_doc_t *d) {break;}pdf_val_t *xs = pdf_dget_raw(tr, "XRefStm");- if (xs && xs->kind == PV_NUM && xs->is_int) {+ if (xs && xs->kind == PV_NUM && xs->is_int &&+ pdf_map_get(&stm_seen, (uint64_t)xs->inum) < 0) {+ if (!pdf_map_put(&stm_seen, (uint64_t)xs->inum, 1)) {+ d->nomem = true;+ xref_free(&sec);+ ok = false;+ break;+ }pdf_xref_t sec2 = {0};pdf_val_t *tr2 = NULL;if (xref_stream_at(d, xs->inum, &sec2, &tr2)) {@@ -1882,6 +1911,7 @@ static bool load_xref(pdf_doc_t *d) {off = pdf_dget_raw(tr, "Prev");}pdf_map_free(&seen);+ pdf_map_free(&stm_seen);d->trailer = kl_finish_dict(d, &trail, false);if (!ok || d->nomem) {return false;@@ -2557,14 +2587,10 @@ static pdf_filter_t filter_kind(const pdf_val_t *f) {return FL_UNKNOWN;}-const unsigned char *pdf_stream_decode(pdf_doc_t *d, pdf_stream_t *st, size_t *len) {- if (st->done) {- *len = st->dec_len;- return st->decoded;- }- st->done = true;- st->decoded = NULL;- st->dec_len = 0;+/* The stream's bytes through its filters: the raw bytes themselves, or a+ * buffer in *held that the caller owns. NULL: a filter failed or is not read. */+static const unsigned char *decode_filters(pdf_doc_t *d, const pdf_stream_t *st, size_t *len,+ pdf_buf_t *held_out) {*len = 0;pdf_val_t *filters = pdf_resolve(d, pdf_dget_raw(st->dict, "Filter"));pdf_val_t *parms = pdf_resolve(d, pdf_dget_raw(st->dict, "DecodeParms"));@@ -2649,6 +2675,26 @@ const unsigned char *pdf_stream_decode(pdf_doc_t *d, pdf_stream_t *st, size_t *lcur = held.p ? held.p : (const unsigned char *)"";cur_len = held.n;}+ *held_out = held;+ *len = cur_len;+ return cur;+}++const unsigned char *pdf_stream_decode(pdf_doc_t *d, pdf_stream_t *st, size_t *len) {+ if (st->done) {+ *len = st->dec_len;+ return st->decoded;+ }+ st->done = true;+ st->decoded = NULL;+ st->dec_len = 0;+ *len = 0;+ pdf_buf_t held = {0};+ size_t cur_len;+ const unsigned char *cur = decode_filters(d, st, &cur_len, &held);+ if (!cur) {+ return NULL;+ }if (held.p) {unsigned char *fit =(unsigned char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, held.p, held.n ? held.n : 1);diff --git a/tests/test_doc_links_pdf.c b/tests/test_doc_links_pdf.cindex a8c228a18..bdaf49e1f 100644--- a/tests/test_doc_links_pdf.c+++ b/tests/test_doc_links_pdf.c@@ -650,6 +650,79 @@ TEST(pdf_extract_filters_and_hostile) {PASS();}+/* Objects 1-5 as tp_simple placed by one cross-reference stream (object 6),+ * then `sections` classic sections, each naming that stream as its /XRefStm,+ * chained by /Prev. */+static unsigned char *tp_xrefstm_sections(int sections, size_t *out_len) {+ size_t off[7] = {0};+ tp_buf_t b = {0};+ tp_puts(&b, "%PDF-1.7\n");+ const char *content = "BT /F1 12 Tf 72 700 Td (xrefstm) Tj ET";+ const char *body[5] = {TP_CATALOG, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", TP_PAGE_F1_5,+ TP_HELV, NULL};+ char line[160];+ for (int i = 1; i <= 5; i++) {+ tp_obj_head(&b, off, i);+ if (i == 5) {+ snprintf(line, sizeof(line), "<< /Length %zu >>\nstream\n", strlen(content));+ tp_puts(&b, line);+ tp_puts(&b, content);+ tp_puts(&b, "\nendstream");+ } else {+ tp_puts(&b, body[i - 1]);+ }+ tp_puts(&b, "\nendobj\n");+ }+ off[6] = b.n;+ unsigned char rows[7][7] = {{0}};+ for (int i = 1; i <= 6; i++) {+ uint32_t f2 = (uint32_t)off[i];+ rows[i][0] = 1;+ rows[i][1] = (unsigned char)(f2 >> 24);+ rows[i][2] = (unsigned char)(f2 >> 16);+ rows[i][3] = (unsigned char)(f2 >> 8);+ rows[i][4] = (unsigned char)f2;+ }+ snprintf(line, sizeof(line),+ "6 0 obj\n<< /Type /XRef /Size 7 /W [1 4 2] /Length %zu >>\nstream\n", sizeof(rows));+ tp_puts(&b, line);+ tp_put(&b, rows, sizeof(rows));+ tp_puts(&b, "\nendstream\nendobj\n");+ size_t prev = 0;+ for (int s = 0; s < sections; s++) {+ size_t at = b.n;+ tp_puts(&b, "xref\n0 1\n0000000000 65535 f \ntrailer\n");+ if (s == 0) {+ snprintf(line, sizeof(line), "<< /Size 7 /Root 1 0 R /XRefStm %zu >>\n", off[6]);+ } else {+ snprintf(line, sizeof(line), "<< /Size 7 /Root 1 0 R /XRefStm %zu /Prev %zu >>\n",+ off[6], prev);+ }+ tp_puts(&b, line);+ prev = at;+ }+ snprintf(line, sizeof(line), "startxref\n%zu\n%%%%EOF\n", prev);+ tp_puts(&b, line);+ *out_len = b.n;+ return b.p;+}++/* A thousand sections naming one /XRefStm decode it once: a later (older)+ * section adds nothing, and every copy used to stay until the document closed. */+TEST(pdf_extract_xrefstm_once) {+ size_t len;+ tp_text_t t;+ unsigned char *pdf = tp_xrefstm_sections(1000, &len);+ ASSERT_TRUE(tp_extract(pdf, len, &t));+ ASSERT_EQ(t.r.status, CBM_PDF_OK);+ ASSERT_EQ(t.r.npages, 1);+ ASSERT_STR_EQ(t.r.pages[0].text, "xrefstm");+ ASSERT_EQ(t.r.counts.xref_streams, 1);+ cbm_arena_destroy(&t.arena);+ free(pdf);+ PASS();+}+/* A chain of objects, each needed to load the one before, costs no stack per* object: a stream's /Length is read as a value (a stream there is no length),* and the objects that open an object stream are no objects of one. Run on the@@ -985,6 +1058,7 @@ SUITE(doc_links_pdf) {RUN_TEST(pdf_extract_structure);RUN_TEST(pdf_extract_filters_and_hostile);RUN_TEST(pdf_extract_object_chains);+ RUN_TEST(pdf_extract_xrefstm_once);RUN_TEST(pdf_scan_mentions);RUN_TEST(pdf_links_pipeline);RUN_TEST(pdf_snippet_is_page_text);