Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

pdf: a cross-reference stream named by many sections is decoded once

Martin Vogela21fa8af8836
diff --git a/internal/cbm/pdf/pdf.h b/internal/cbm/pdf/pdf.h
index 98f1e834b..4cfe1b4da 100644
--- a/internal/cbm/pdf/pdf.h
+++ b/internal/cbm/pdf/pdf.h
@@ -39,6 +39,7 @@ typedef struct {
int xref_offset_fixed; /* an xref section found near a wrong offset */
int stream_length_recovered; /* a stream's /Length was wrong */
int objstm_nested; /* opening an object stream needed an object of one */
+ int xref_streams; /* cross-reference streams decoded */
int flate_recovered; /* a damaged Flate stream decoded in part */
int stream_guard_hits; /* a decoded stream cut at 512 MiB */
int filters_unsupported; /* a stream with a filter this reader does not decode */
diff --git a/internal/cbm/pdf/pdf_obj.c b/internal/cbm/pdf/pdf_obj.c
index 22ab40491..229338676 100644
--- a/internal/cbm/pdf/pdf_obj.c
+++ b/internal/cbm/pdf/pdf_obj.c
@@ -1643,6 +1643,58 @@ static bool num_to_int(const pdf_val_t *v, int64_t *out) {
return true;
}
+static const unsigned char *decode_filters(pdf_doc_t *d, const pdf_stream_t *st, size_t *len,
+ pdf_buf_t *held_out);
+
+/* The rows of a cross-reference stream's decoded bytes into sec. */
+static bool xref_rows(pdf_doc_t *d, pdf_xref_t *sec, const unsigned char *raw, size_t rlen,
+ const int64_t w[3], pdf_val_t **items, int nitems, const int64_t pairs[2]) {
+ int64_t rec = w[0] + w[1] + w[2];
+ if (rec == 0) {
+ return false;
+ }
+ size_t pos = 0;
+ for (int j = 0; j + 1 < nitems; j += 2) {
+ int64_t start;
+ int64_t count;
+ if (items) {
+ if (!num_to_int(items[j], &start) || !num_to_int(items[j + 1], &count)) {
+ return false;
+ }
+ } else {
+ start = pairs[0];
+ count = pairs[1];
+ }
+ for (int64_t i = 0; i < count; i++) {
+ if (pos + (size_t)rec > rlen) {
+ break;
+ }
+ const unsigned char *r = raw + pos;
+ int64_t t = w[0] ? be_field(r, (int)w[0]) : 1;
+ int64_t f2 = be_field(r + w[0], (int)w[1]);
+ int64_t f3 = w[2] ? be_field(r + w[0] + w[1], (int)w[2]) : 0;
+ pos += (size_t)rec;
+ bool fresh;
+ pdf_xent_t *e = xref_slot(d, sec, pdf_sat_add(start, i), &fresh);
+ if (!e) {
+ return false;
+ }
+ if (!fresh) {
+ continue;
+ }
+ if (t == 1 || t == 2) {
+ e->type = (uint8_t)t;
+ e->a = f2;
+ e->b = f3;
+ }
+ }
+ }
+ return true;
+}
+
+/* A cross-reference stream at off into sec. Its decoded bytes are freed once
+ * read: a stream parsed here is no cached object, and keeping them made every
+ * section that names one stream add a copy until the document closed. */
static bool xref_stream_at(pdf_doc_t *d, int64_t off, pdf_xref_t *sec, pdf_val_t **tr) {
bool ok;
pdf_val_t *v = parse_indirect_at(d, off, false, 0, &ok);
@@ -1682,50 +1734,17 @@ static bool xref_stream_at(pdf_doc_t *d, int64_t off, pdf_xref_t *sec, pdf_val_t
return false;
}
size_t rlen = 0;
- const unsigned char *raw = pdf_stream_decode(d, v->stream, &rlen);
+ pdf_buf_t held = {0};
+ const unsigned char *raw = decode_filters(d, v->stream, &rlen, &held);
if (!raw) {
return false;
}
- int64_t rec = w[0] + w[1] + w[2];
- if (rec == 0) {
+ d->counts->xref_streams++;
+ bool rows_ok = xref_rows(d, sec, raw, rlen, w, items, nitems, pairs);
+ buf_free(&held);
+ if (!rows_ok) {
return false;
}
- size_t pos = 0;
- for (int j = 0; j + 1 < nitems; j += 2) {
- int64_t start;
- int64_t count;
- if (items) {
- if (!num_to_int(items[j], &start) || !num_to_int(items[j + 1], &count)) {
- return false;
- }
- } else {
- start = pairs[0];
- count = pairs[1];
- }
- for (int64_t i = 0; i < count; i++) {
- if (pos + (size_t)rec > rlen) {
- break;
- }
- const unsigned char *r = raw + pos;
- int64_t t = w[0] ? be_field(r, (int)w[0]) : 1;
- int64_t f2 = be_field(r + w[0], (int)w[1]);
- int64_t f3 = w[2] ? be_field(r + w[0] + w[1], (int)w[2]) : 0;
- pos += (size_t)rec;
- bool fresh;
- pdf_xent_t *e = xref_slot(d, sec, pdf_sat_add(start, i), &fresh);
- if (!e) {
- return false;
- }
- if (!fresh) {
- continue;
- }
- if (t == 1 || t == 2) {
- e->type = (uint8_t)t;
- e->a = f2;
- e->b = f3;
- }
- }
- }
*tr = dict;
return true;
}
@@ -1811,6 +1830,9 @@ static bool load_xref(pdf_doc_t *d) {
return false;
}
pdf_map_t seen = {0};
+ /* /XRefStm targets already merged: a later (older) section naming one again
+ * adds nothing, since every number it holds is in d->xref already */
+ pdf_map_t stm_seen = {0};
kv_list_t trail = {0};
bool first = true;
bool ok = true;
@@ -1842,7 +1864,14 @@ static bool load_xref(pdf_doc_t *d) {
break;
}
pdf_val_t *xs = pdf_dget_raw(tr, "XRefStm");
- if (xs && xs->kind == PV_NUM && xs->is_int) {
+ if (xs && xs->kind == PV_NUM && xs->is_int &&
+ pdf_map_get(&stm_seen, (uint64_t)xs->inum) < 0) {
+ if (!pdf_map_put(&stm_seen, (uint64_t)xs->inum, 1)) {
+ d->nomem = true;
+ xref_free(&sec);
+ ok = false;
+ break;
+ }
pdf_xref_t sec2 = {0};
pdf_val_t *tr2 = NULL;
if (xref_stream_at(d, xs->inum, &sec2, &tr2)) {
@@ -1882,6 +1911,7 @@ static bool load_xref(pdf_doc_t *d) {
off = pdf_dget_raw(tr, "Prev");
}
pdf_map_free(&seen);
+ pdf_map_free(&stm_seen);
d->trailer = kl_finish_dict(d, &trail, false);
if (!ok || d->nomem) {
return false;
@@ -2557,14 +2587,10 @@ static pdf_filter_t filter_kind(const pdf_val_t *f) {
return FL_UNKNOWN;
}
-const unsigned char *pdf_stream_decode(pdf_doc_t *d, pdf_stream_t *st, size_t *len) {
- if (st->done) {
- *len = st->dec_len;
- return st->decoded;
- }
- st->done = true;
- st->decoded = NULL;
- st->dec_len = 0;
+/* The stream's bytes through its filters: the raw bytes themselves, or a
+ * buffer in *held that the caller owns. NULL: a filter failed or is not read. */
+static const unsigned char *decode_filters(pdf_doc_t *d, const pdf_stream_t *st, size_t *len,
+ pdf_buf_t *held_out) {
*len = 0;
pdf_val_t *filters = pdf_resolve(d, pdf_dget_raw(st->dict, "Filter"));
pdf_val_t *parms = pdf_resolve(d, pdf_dget_raw(st->dict, "DecodeParms"));
@@ -2649,6 +2675,26 @@ const unsigned char *pdf_stream_decode(pdf_doc_t *d, pdf_stream_t *st, size_t *l
cur = held.p ? held.p : (const unsigned char *)"";
cur_len = held.n;
}
+ *held_out = held;
+ *len = cur_len;
+ return cur;
+}
+
+const unsigned char *pdf_stream_decode(pdf_doc_t *d, pdf_stream_t *st, size_t *len) {
+ if (st->done) {
+ *len = st->dec_len;
+ return st->decoded;
+ }
+ st->done = true;
+ st->decoded = NULL;
+ st->dec_len = 0;
+ *len = 0;
+ pdf_buf_t held = {0};
+ size_t cur_len;
+ const unsigned char *cur = decode_filters(d, st, &cur_len, &held);
+ if (!cur) {
+ return NULL;
+ }
if (held.p) {
unsigned char *fit =
(unsigned char *)cbm_realloc(CBM_MEM_CLASS_EXTRACT, held.p, held.n ? held.n : 1);
diff --git a/tests/test_doc_links_pdf.c b/tests/test_doc_links_pdf.c
index a8c228a18..bdaf49e1f 100644
--- a/tests/test_doc_links_pdf.c
+++ b/tests/test_doc_links_pdf.c
@@ -650,6 +650,79 @@ TEST(pdf_extract_filters_and_hostile) {
PASS();
}
+/* Objects 1-5 as tp_simple placed by one cross-reference stream (object 6),
+ * then `sections` classic sections, each naming that stream as its /XRefStm,
+ * chained by /Prev. */
+static unsigned char *tp_xrefstm_sections(int sections, size_t *out_len) {
+ size_t off[7] = {0};
+ tp_buf_t b = {0};
+ tp_puts(&b, "%PDF-1.7\n");
+ const char *content = "BT /F1 12 Tf 72 700 Td (xrefstm) Tj ET";
+ const char *body[5] = {TP_CATALOG, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", TP_PAGE_F1_5,
+ TP_HELV, NULL};
+ char line[160];
+ for (int i = 1; i <= 5; i++) {
+ tp_obj_head(&b, off, i);
+ if (i == 5) {
+ snprintf(line, sizeof(line), "<< /Length %zu >>\nstream\n", strlen(content));
+ tp_puts(&b, line);
+ tp_puts(&b, content);
+ tp_puts(&b, "\nendstream");
+ } else {
+ tp_puts(&b, body[i - 1]);
+ }
+ tp_puts(&b, "\nendobj\n");
+ }
+ off[6] = b.n;
+ unsigned char rows[7][7] = {{0}};
+ for (int i = 1; i <= 6; i++) {
+ uint32_t f2 = (uint32_t)off[i];
+ rows[i][0] = 1;
+ rows[i][1] = (unsigned char)(f2 >> 24);
+ rows[i][2] = (unsigned char)(f2 >> 16);
+ rows[i][3] = (unsigned char)(f2 >> 8);
+ rows[i][4] = (unsigned char)f2;
+ }
+ snprintf(line, sizeof(line),
+ "6 0 obj\n<< /Type /XRef /Size 7 /W [1 4 2] /Length %zu >>\nstream\n", sizeof(rows));
+ tp_puts(&b, line);
+ tp_put(&b, rows, sizeof(rows));
+ tp_puts(&b, "\nendstream\nendobj\n");
+ size_t prev = 0;
+ for (int s = 0; s < sections; s++) {
+ size_t at = b.n;
+ tp_puts(&b, "xref\n0 1\n0000000000 65535 f \ntrailer\n");
+ if (s == 0) {
+ snprintf(line, sizeof(line), "<< /Size 7 /Root 1 0 R /XRefStm %zu >>\n", off[6]);
+ } else {
+ snprintf(line, sizeof(line), "<< /Size 7 /Root 1 0 R /XRefStm %zu /Prev %zu >>\n",
+ off[6], prev);
+ }
+ tp_puts(&b, line);
+ prev = at;
+ }
+ snprintf(line, sizeof(line), "startxref\n%zu\n%%%%EOF\n", prev);
+ tp_puts(&b, line);
+ *out_len = b.n;
+ return b.p;
+}
+
+/* A thousand sections naming one /XRefStm decode it once: a later (older)
+ * section adds nothing, and every copy used to stay until the document closed. */
+TEST(pdf_extract_xrefstm_once) {
+ size_t len;
+ tp_text_t t;
+ unsigned char *pdf = tp_xrefstm_sections(1000, &len);
+ ASSERT_TRUE(tp_extract(pdf, len, &t));
+ ASSERT_EQ(t.r.status, CBM_PDF_OK);
+ ASSERT_EQ(t.r.npages, 1);
+ ASSERT_STR_EQ(t.r.pages[0].text, "xrefstm");
+ ASSERT_EQ(t.r.counts.xref_streams, 1);
+ cbm_arena_destroy(&t.arena);
+ free(pdf);
+ PASS();
+}
+
/* A chain of objects, each needed to load the one before, costs no stack per
* object: a stream's /Length is read as a value (a stream there is no length),
* and the objects that open an object stream are no objects of one. Run on the
@@ -985,6 +1058,7 @@ SUITE(doc_links_pdf) {
RUN_TEST(pdf_extract_structure);
RUN_TEST(pdf_extract_filters_and_hostile);
RUN_TEST(pdf_extract_object_chains);
+ RUN_TEST(pdf_extract_xrefstm_once);
RUN_TEST(pdf_scan_mentions);
RUN_TEST(pdf_links_pipeline);
RUN_TEST(pdf_snippet_is_page_text);