Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
pdf: a chain of objects costs no stack per object
Martin Vogele3023674b142
diff --git a/internal/cbm/pdf/pdf.h b/internal/cbm/pdf/pdf.hindex 508b2158b..98f1e834b 100644--- a/internal/cbm/pdf/pdf.h+++ b/internal/cbm/pdf/pdf.h@@ -38,6 +38,7 @@ typedef struct {int xref_reconstructed; /* cross-reference rebuilt from a scan of the file */int xref_offset_fixed; /* an xref section found near a wrong offset */int stream_length_recovered; /* a stream's /Length was wrong */+ int objstm_nested; /* opening an object stream needed an object of one */int flate_recovered; /* a damaged Flate stream decoded in part */int stream_guard_hits; /* a decoded stream cut at 512 MiB */int filters_unsupported; /* a stream with a filter this reader does not decode */diff --git a/internal/cbm/pdf/pdf_internal.h b/internal/cbm/pdf/pdf_internal.hindex d97881e50..0b50b929b 100644--- a/internal/cbm/pdf/pdf_internal.h+++ b/internal/cbm/pdf/pdf_internal.h@@ -208,6 +208,7 @@ typedef struct {pdf_val_t *obj;bool cached;bool loading;+ bool not_length; /* read as a stream's /Length, it was a stream: no length */pdf_objstm_t *stm; /* this object as a decoded object stream */} pdf_xent_t;@@ -248,6 +249,7 @@ struct pdf_doc {pdf_xref_t xref;pdf_val_t *trailer; /* a dict */bool reconstructed;+ int objstm_opening; /* object streams being opened, one inside the other *//* every "N G obj" in file order, built on first need */pdf_hdr_t *hdrs;int nhdrs;diff --git a/internal/cbm/pdf/pdf_obj.c b/internal/cbm/pdf/pdf_obj.cindex c60c2bb8d..22ab40491 100644--- a/internal/cbm/pdf/pdf_obj.c+++ b/internal/cbm/pdf/pdf_obj.c@@ -1072,10 +1072,13 @@ static bool endstream_follows(const pdf_doc_t *d, size_t q) {return false;}-/* The object at off (or off + hdr). NULL: no matching header there. */-static pdf_val_t *parse_indirect_at(pdf_doc_t *d, int64_t off, bool has_expect, int64_t expect,- bool *ok) {+/* The value of the object at off (or off + hdr), without its stream; *stream_at+ * is where a "stream" keyword after the value starts, 0 when none follows. NULL:+ * no matching header there. */+static pdf_val_t *indirect_value_at(pdf_doc_t *d, int64_t off, bool has_expect, int64_t expect,+ bool *ok, size_t *stream_at) {*ok = false;+ *stream_at = 0;size_t at = 0;bool found = false;int64_t num = 0;@@ -1106,20 +1109,75 @@ static pdf_val_t *parse_indirect_at(pdf_doc_t *d, int64_t off, bool has_expect,size_t p2 = p;pdf_tok_t t2;lex_peek(d, s, len, &p2, &t2);- if (!(pdf_kw_is(&t2, "stream") && val->kind == PV_DICT)) {+ if (pdf_kw_is(&t2, "stream") && val->kind == PV_DICT) {+ *stream_at = p2;+ }+ return val;+}++static int64_t hdr_first_exact(pdf_doc_t *d, int64_t num, int64_t gen);++/* An object a stream's /Length names, read as a value only and cached as+ * pdf_get caches it. An object that is a stream there gives no length and is+ * marked: loading it would read ITS /Length first, one nested call per stream+ * of a chain of such streams. */+static pdf_val_t *length_object(pdf_doc_t *d, int64_t num) {+ pdf_xent_t *e = xent(d, num);+ bool ok;+ size_t stream_at;+ pdf_val_t *v = indirect_value_at(d, e->a, true, num, &ok, &stream_at);+ if (!ok && !d->reconstructed) {+ int64_t off = hdr_first_exact(d, num, e->b);+ if (off >= 0) {+ v = indirect_value_at(d, off, true, num, &ok, &stream_at);+ }+ }+ e = xent(d, num);+ if (ok && stream_at) {+ e->not_length = true;+ return NULL;+ }+ e->cached = true;+ e->obj = ok ? v : NULL;+ return e->obj;+}++/* A stream's /Length through its references (at most PDF_MAX_RESOLVE, as+ * pdf_resolve). */+static pdf_val_t *length_value(pdf_doc_t *d, pdf_val_t *v) {+ for (int depth = 0; v && v->kind == PV_REF; depth++) {+ pdf_xent_t *e = xent(d, v->ref_num);+ if (depth >= PDF_MAX_RESOLVE || !e || e->loading || e->not_length) {+ return NULL;+ }+ if (e->cached) {+ v = e->obj;+ } else if (e->type == 1) {+ v = length_object(d, v->ref_num);+ } else {+ v = pdf_get(d, v->ref_num);+ }+ }+ return v;+}++/* The object at off (or off + hdr), a stream with its raw bytes when one+ * follows. NULL: no matching header there. */+static pdf_val_t *parse_indirect_at(pdf_doc_t *d, int64_t off, bool has_expect, int64_t expect,+ bool *ok) {+ size_t st;+ pdf_val_t *val = indirect_value_at(d, off, has_expect, expect, ok, &st);+ if (!val || !st) {return val;}- size_t st = p2;+ const unsigned char *s = d->data;+ size_t len = d->len;if (st + 1 < len && s[st] == '\r' && s[st + 1] == '\n') {st += 2;} else if (st < len && (s[st] == '\n' || s[st] == '\r')) {st += 1;}- pdf_val_t *ln = pdf_dget_raw(val, "Length");- if (ln && ln->kind == PV_REF) {- pdf_xent_t *e = xent(d, ln->ref_num);- ln = (e && e->loading) ? NULL : pdf_resolve(d, ln);- }+ pdf_val_t *ln = length_value(d, pdf_dget_raw(val, "Length"));const unsigned char *raw = NULL;size_t raw_len = 0;if (ln && ln->kind == PV_NUM && ln->is_int && ln->inum >= 0 &&@@ -1318,6 +1376,14 @@ pdf_val_t *pdf_get(pdf_doc_t *d, int64_t num) {if (e->loading) {return NULL;}+ if (e->type == 2 && d->objstm_opening > 0) {+ /* The objects that open an object stream (its /Length, its filters) are+ * no objects of an object stream (ISO 32000-1, 7.5.7): one that is would+ * open the next stream inside this one, one nested call per stream of a+ * chain. */+ d->counts->objstm_nested++;+ return NULL;+ }e->loading = true;uint8_t type = e->type;int64_t a = e->a;@@ -1366,7 +1432,7 @@ pdf_val_t *pdf_resolve(pdf_doc_t *d, pdf_val_t *v) {return v;}-static pdf_objstm_t *objstm(pdf_doc_t *d, int64_t stmnum) {+static pdf_objstm_t *objstm_open(pdf_doc_t *d, int64_t stmnum) {pdf_xent_t *e = xent(d, stmnum);if (e && e->stm) {return e->stm;@@ -1421,6 +1487,13 @@ static pdf_objstm_t *objstm(pdf_doc_t *d, int64_t stmnum) {return sm;}+static pdf_objstm_t *objstm(pdf_doc_t *d, int64_t stmnum) {+ d->objstm_opening++;+ pdf_objstm_t *sm = objstm_open(d, stmnum);+ d->objstm_opening--;+ return sm;+}+/* ── Cross-reference ─────────────────────────────────────────────── */static void xref_free(pdf_xref_t *x) {diff --git a/tests/test_doc_links_pdf.c b/tests/test_doc_links_pdf.cindex 0ce880d59..a8c228a18 100644--- a/tests/test_doc_links_pdf.c+++ b/tests/test_doc_links_pdf.c@@ -16,6 +16,7 @@#include "cbm.h"#include "doclink.h"+#include "foundation/compat_thread.h"#include "mcp/mcp.h"#include "pdf/pdf.h"#include "pipeline/pipeline.h"@@ -134,6 +135,177 @@ static bool tp_extract(const unsigned char *pdf, size_t len, tp_text_t *t) {return cbm_pdf_extract(pdf, len, &t->arena, &t->r);}+/* The smallest stack a thread of this program runs on (the daemon's). */+#define TP_SMALL_STACK (256 * 1024)++typedef struct {+ const unsigned char *pdf;+ size_t len;+ tp_text_t *t;+ bool ok;+} tp_job_t;++static void *tp_extract_job(void *arg) {+ tp_job_t *j = (tp_job_t *)arg;+ j->ok = tp_extract(j->pdf, j->len, j->t);+ return NULL;+}++/* tp_extract on a thread with the smallest stack: an input that nests calls+ * per object crashes here long before it would on a worker's 8 MiB. */+static bool tp_extract_small_stack(const unsigned char *pdf, size_t len, tp_text_t *t) {+ tp_job_t j = {pdf, len, t, false};+ cbm_thread_t th;+ if (cbm_thread_create(&th, TP_SMALL_STACK, tp_extract_job, &j) != 0) {+ return false;+ }+ cbm_thread_join(&th);+ return j.ok;+}++/* "N 0 obj\n" for object n at the end of b; records its offset. */+static void tp_obj_head(tp_buf_t *b, size_t *off, int n) {+ char line[64];+ off[n] = b->n;+ snprintf(line, sizeof(line), "%d 0 obj\n", n);+ tp_puts(b, line);+}++/* A classic xref table for objects 1..n at off[] and the trailer. */+static void tp_classic_tail(tp_buf_t *b, const size_t *off, int n) {+ char line[128];+ size_t xref = b->n;+ snprintf(line, sizeof(line), "xref\n0 %d\n0000000000 65535 f \n", n + 1);+ tp_puts(b, line);+ for (int i = 1; i <= n; i++) {+ snprintf(line, sizeof(line), "%010zu 00000 n \n", off[i]);+ tp_puts(b, line);+ }+ snprintf(line, sizeof(line), "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%zu\n%%%%EOF\n",+ n + 1, xref);+ tp_puts(b, line);+}++#define TP_PAGE_F1_5 \+ "<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>"++/* A page whose content stream's /Length names a stream whose /Length names+ * the next one, `depth` streams deep (the last one's /Length is a number). */+static unsigned char *tp_length_chain(int depth, size_t *out_len) {+ int n = 4 + depth;+ size_t *off = (size_t *)calloc((size_t)n + 1, sizeof(size_t));+ tp_buf_t b = {0};+ tp_puts(&b, "%PDF-1.7\n");+ const char *head[4] = {TP_CATALOG, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", TP_PAGE_F1_5,+ TP_HELV};+ for (int i = 1; i <= 4; i++) {+ tp_obj_head(&b, off, i);+ tp_puts(&b, head[i - 1]);+ tp_puts(&b, "\nendobj\n");+ }+ char line[96];+ for (int i = 5; i <= n; i++) {+ tp_obj_head(&b, off, i);+ if (i < n) {+ snprintf(line, sizeof(line), "<< /Length %d 0 R >>\nstream\n", i + 1);+ } else {+ snprintf(line, sizeof(line), "<< /Length 1 >>\nstream\n");+ }+ tp_puts(&b, line);+ tp_puts(&b, i == 5 ? "BT /F1 12 Tf 72 700 Td (chain) Tj ET" : "x");+ tp_puts(&b, "\nendstream\nendobj\n");+ }+ tp_classic_tail(&b, off, n);+ free(off);+ *out_len = b.n;+ return b.p;+}++/* The page object lies in object stream C1, whose /Length lies in object+ * stream C2, whose /Length lies in C3, ... `depth` streams deep; a+ * cross-reference stream places them. Objects: 1-5 as tp_simple (3 in C1),+ * C_i = 5 + i, L_i (C_i's length, in C_{i+1}) = 5 + depth + i, the xref stream+ * last. */+static unsigned char *tp_objstm_chain(int depth, size_t *out_len) {+ int x = 5 + 2 * depth; /* the xref stream */+ size_t *off = (size_t *)calloc((size_t)x + 1, sizeof(size_t));+ size_t *data_len = (size_t *)calloc((size_t)depth + 1, sizeof(size_t));+ tp_buf_t b = {0};+ tp_puts(&b, "%PDF-1.7\n");+ const char *plain[5] = {TP_CATALOG, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", NULL, TP_HELV,+ NULL};+ char line[160];+ for (int i = 1; i <= 5; i++) {+ if (i == 3) {+ continue;+ }+ tp_obj_head(&b, off, i);+ if (i == 5) {+ const char *content = "BT /F1 12 Tf 72 700 Td (objstm chain) Tj ET";+ snprintf(line, sizeof(line), "<< /Length %zu >>\nstream\n", strlen(content));+ tp_puts(&b, line);+ tp_puts(&b, content);+ tp_puts(&b, "\nendstream");+ } else {+ tp_puts(&b, plain[i - 1]);+ }+ tp_puts(&b, "\nendobj\n");+ }+ for (int i = 1; i <= depth; i++) {+ char hdr[48];+ char body[160];+ if (i == 1) {+ snprintf(hdr, sizeof(hdr), "3 0 ");+ snprintf(body, sizeof(body), "%s", TP_PAGE_F1_5);+ } else {+ snprintf(hdr, sizeof(hdr), "%d 0 ", 5 + depth + i - 1);+ snprintf(body, sizeof(body), "%zu", data_len[i - 1]);+ }+ data_len[i] = strlen(hdr) + strlen(body);+ tp_obj_head(&b, off, 5 + i);+ if (i < depth) {+ snprintf(line, sizeof(line),+ "<< /Type /ObjStm /N 1 /First %zu /Length %d 0 R >>\nstream\n", strlen(hdr),+ 5 + depth + i);+ } else {+ snprintf(line, sizeof(line),+ "<< /Type /ObjStm /N 1 /First %zu /Length %zu >>\nstream\n", strlen(hdr),+ data_len[i]);+ }+ tp_puts(&b, line);+ tp_puts(&b, hdr);+ tp_puts(&b, body);+ tp_puts(&b, "\nendstream\nendobj\n");+ }+ off[x] = b.n;+ /* rows W [1 4 2]: type, offset or containing stream, generation or index */+ size_t nrows = (size_t)x + 1;+ unsigned char *rows = (unsigned char *)calloc(nrows, 7);+ for (int i = 1; i <= x; i++) {+ unsigned char *r = rows + (size_t)i * 7;+ bool in_stm = i == 3 || (i > 5 + depth && i < x);+ uint32_t f2 = in_stm ? (uint32_t)(i == 3 ? 6 : i - depth + 1) : (uint32_t)off[i];+ r[0] = in_stm ? 2 : 1;+ r[1] = (unsigned char)(f2 >> 24);+ r[2] = (unsigned char)(f2 >> 16);+ r[3] = (unsigned char)(f2 >> 8);+ r[4] = (unsigned char)f2;+ }+ snprintf(line, sizeof(line),+ "%d 0 obj\n<< /Type /XRef /Size %d /W [1 4 2] /Root 1 0 R /Length %zu >>\nstream\n", x,+ x + 1, nrows * 7);+ tp_puts(&b, line);+ tp_put(&b, rows, nrows * 7);+ tp_puts(&b, "\nendstream\nendobj\n");+ snprintf(line, sizeof(line), "startxref\n%zu\n%%%%EOF\n", off[x]);+ tp_puts(&b, line);+ free(rows);+ free(data_len);+ free(off);+ *out_len = b.n;+ return b.p;+}+/* ── extractor ───────────────────────────────────────────────────── */TEST(pdf_extract_layout) {@@ -478,6 +650,32 @@ TEST(pdf_extract_filters_and_hostile) {PASS();}+/* A chain of objects, each needed to load the one before, costs no stack per+ * object: a stream's /Length is read as a value (a stream there is no length),+ * and the objects that open an object stream are no objects of one. Run on the+ * smallest stack in the program. */+TEST(pdf_extract_object_chains) {+ size_t len;+ tp_text_t t;+ unsigned char *pdf = tp_length_chain(20000, &len);+ ASSERT_TRUE(tp_extract_small_stack(pdf, len, &t));+ ASSERT_EQ(t.r.status, CBM_PDF_OK);+ ASSERT_EQ(t.r.npages, 1);+ ASSERT_STR_EQ(t.r.pages[0].text, "chain");+ ASSERT_TRUE(t.r.counts.stream_length_recovered >= 1);+ cbm_arena_destroy(&t.arena);+ free(pdf);+ pdf = tp_objstm_chain(3000, &len);+ ASSERT_TRUE(tp_extract_small_stack(pdf, len, &t));+ ASSERT_EQ(t.r.status, CBM_PDF_OK);+ ASSERT_EQ(t.r.npages, 1);+ ASSERT_STR_EQ(t.r.pages[0].text, "objstm chain");+ ASSERT_TRUE(t.r.counts.objstm_nested >= 1);+ cbm_arena_destroy(&t.arena);+ free(pdf);+ PASS();+}+/* ── scanner (expected: the field-test scanner on the same text) ─── */TEST(pdf_scan_mentions) {@@ -786,6 +984,7 @@ SUITE(doc_links_pdf) {RUN_TEST(pdf_extract_fonts);RUN_TEST(pdf_extract_structure);RUN_TEST(pdf_extract_filters_and_hostile);+ RUN_TEST(pdf_extract_object_chains);RUN_TEST(pdf_scan_mentions);RUN_TEST(pdf_links_pipeline);RUN_TEST(pdf_snippet_is_page_text);