Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

pdf: a join compares only the candidates of its two lines

Martin Vogel143277944597
diff --git a/internal/cbm/doc_pdf.c b/internal/cbm/doc_pdf.c
index b7a59a7a5..586417b2e 100644
--- a/internal/cbm/doc_pdf.c
+++ b/internal/cbm/doc_pdf.c
@@ -30,10 +30,16 @@
#include "pdf/pdf.h"
#include "pdf/pdf_unicode.h"
+#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+/* candidates a join compared, for the cost test */
+static atomic_size_t g_join_steps;
+#endif
+
enum {
PDF_R3_WINDOW = 400, /* the field test looked this far back for "(" or "," */
PDF_PAGE_NAME = 32,
@@ -1120,6 +1126,46 @@ static void scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t
}
tok[i] = emit(&e, kind_syntax(c->kind), sb.p ? sb.p : "", line_of(&e, c->start), 0);
}
+ /* The emitted candidates of each line, ascending (CSR: lc[lc_off[l] ..
+ * lc_off[l + 1])). A join's span covers the end of one line and the start of
+ * the next, so only their candidates can overlap it; reading every candidate
+ * of the page per join cost (joins x candidates), quadratic in the text. */
+ int *lc_off = (int *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, (size_t)(nlines + 2) * sizeof(int));
+ int *lc = NULL;
+ if (!lc_off) {
+ e.failed = true;
+ } else {
+ for (int f = 0; f < cv.n; f++) {
+ if (tok[f] >= 0) {
+ int end = cv.v[f].end > cv.v[f].start ? cv.v[f].end - 1 : cv.v[f].start;
+ for (uint32_t l = line_of(&e, cv.v[f].start); l <= line_of(&e, end); l++) {
+ lc_off[l]++; /* line_of is 1-based: line l - 1 counts at l */
+ }
+ }
+ }
+ for (int l = 0; l < nlines; l++) {
+ lc_off[l + 1] += lc_off[l];
+ }
+ lc = (int *)cbm_alloc(CBM_MEM_CLASS_EXTRACT,
+ (size_t)(lc_off[nlines] ? lc_off[nlines] : 1) * sizeof(int));
+ if (!lc) {
+ e.failed = true;
+ } else {
+ /* lc_off[l] runs as line l's cursor, then shifts back to its start */
+ for (int f = 0; f < cv.n; f++) {
+ if (tok[f] >= 0) {
+ int end = cv.v[f].end > cv.v[f].start ? cv.v[f].end - 1 : cv.v[f].start;
+ for (uint32_t l = line_of(&e, cv.v[f].start); l <= line_of(&e, end); l++) {
+ lc[lc_off[l - 1]++] = f;
+ }
+ }
+ }
+ for (int l = nlines; l > 0; l--) {
+ lc_off[l] = lc_off[l - 1];
+ }
+ lc_off[0] = 0;
+ }
+ }
/* joins */
for (int li = 0; li + 1 < nlines && !sb.fail && !e.failed; li++) {
const uint32_t *prev = flat.v + starts[li];
@@ -1187,9 +1233,25 @@ static void scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t
char fs = PDF_JOIN_FRAG;
sb_put(&sb, &fs, 1);
bool first = true;
- for (int f = 0; f < cv.n; f++) {
+ int ia = lc_off[li];
+ int ea = lc_off[li + 1];
+ int ib = lc_off[li + 1];
+ int eb = lc_off[li + 2];
+ while (ia < ea || ib < eb) {
+ int f;
+ if (ib >= eb || (ia < ea && lc[ia] < lc[ib])) {
+ f = lc[ia++];
+ } else if (ia >= ea || lc[ib] < lc[ia]) {
+ f = lc[ib++];
+ } else {
+ f = lc[ia++]; /* on both lines */
+ ib++;
+ }
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+ atomic_fetch_add_explicit(&g_join_steps, 1, memory_order_relaxed);
+#endif
const cand_t *x = &cv.v[f];
- if (x->start < e_flat && x->end > s_flat && tok[f] >= 0) {
+ if (x->start < e_flat && x->end > s_flat) {
char num[16];
int nlen = snprintf(num, sizeof(num), first ? "%d" : ",%d", tok[f]);
sb_put(&sb, num, (size_t)nlen);
@@ -1209,6 +1271,8 @@ static void scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t
}
cbm_free(CBM_MEM_CLASS_EXTRACT, sb.p);
cbm_free(CBM_MEM_CLASS_EXTRACT, tok);
+ cbm_free(CBM_MEM_CLASS_EXTRACT, lc_off);
+ cbm_free(CBM_MEM_CLASS_EXTRACT, lc);
done:
candv_free(&cv);
u32_free(&flat);
@@ -1221,6 +1285,10 @@ void cbm_pdf_test_scan_page(CBMExtractCtx *ctx, const char *text, size_t len, ui
const char *page_qn) {
scan_page(ctx, text, len, page, page_qn);
}
+
+size_t cbm_pdf_test_join_steps(void) {
+ return atomic_load_explicit(&g_join_steps, memory_order_relaxed);
+}
#endif
/* ── The document ────────────────────────────────────────────────── */
diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.h
index e97aaf488..05d3283c4 100644
--- a/internal/cbm/doclink.h
+++ b/internal/cbm/doclink.h
@@ -363,6 +363,8 @@ void cbm_pdf_extract_document(CBMExtractCtx *ctx);
* into ctx->result->doc_links), without a PDF around it. */
void cbm_pdf_test_scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t page,
const char *page_qn);
+/* candidates the scanner's joins have compared so far (a cost counter) */
+size_t cbm_pdf_test_join_steps(void);
#endif
#endif /* CBM_DOCLINK_H */
diff --git a/tests/test_doc_links_pdf.c b/tests/test_doc_links_pdf.c
index 66a6c47c1..ee4e77996 100644
--- a/tests/test_doc_links_pdf.c
+++ b/tests/test_doc_links_pdf.c
@@ -927,6 +927,41 @@ TEST(pdf_scan_mentions) {
PASS();
}
+/* Join work on a page of `n` lines that all join, eight names a line. */
+static size_t tp_join_work(int n) {
+ tp_buf_t b = {0};
+ for (int i = 0; i < n; i++) {
+ tp_puts(&b, "pkg.mod.Alpha pkg.mod.Beta dir/sub/f1.py dir/sub/f2.py pkg.mod.Gamma "
+ "pkg.mod.Delta dir/sub/f3.py dir/sub/f4.py end_\n");
+ }
+ CBMFileResult *r = (CBMFileResult *)calloc(1, sizeof(*r));
+ cbm_arena_init(&r->arena);
+ CBMExtractCtx ctx;
+ memset(&ctx, 0, sizeof(ctx));
+ ctx.arena = &r->arena;
+ ctx.result = r;
+ size_t before = cbm_pdf_test_join_steps();
+ cbm_pdf_test_scan_page(&ctx, (const char *)b.p, b.n, 1, "p.doc.page_1");
+ size_t work = cbm_pdf_test_join_steps() - before;
+ bool ok = !r->doc_links.failed && r->doc_links.count > n * 8;
+ cbm_free_result(r);
+ free(b.p);
+ return ok ? work : 0;
+}
+
+/* A join compares only the candidates of its two lines: twice the page, about
+ * twice the work (every candidate of the page per join was four times). */
+TEST(pdf_scan_join_linear) {
+ size_t small = tp_join_work(300);
+ size_t large = tp_join_work(600);
+ ASSERT_TRUE(small > 0);
+ if (large > small * 5 / 2) {
+ fprintf(stderr, "join work: %zu for 300 lines, %zu for 600\n", small, large);
+ }
+ ASSERT_TRUE(large <= small * 5 / 2);
+ PASS();
+}
+
/* ── the pipeline ────────────────────────────────────────────────── */
/* A PDF whose pages each show the given lines (Helvetica, one Tj a line). */
@@ -1185,6 +1220,7 @@ SUITE(doc_links_pdf) {
RUN_TEST(pdf_extract_open_strings_linear);
RUN_TEST(pdf_extract_form_ladder_bounded);
RUN_TEST(pdf_scan_mentions);
+ RUN_TEST(pdf_scan_join_linear);
RUN_TEST(pdf_links_pipeline);
RUN_TEST(pdf_snippet_is_page_text);
RUN_TEST(pdf_links_incremental);