Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
pdf: a join compares only the candidates of its two lines
Martin Vogel143277944597
diff --git a/internal/cbm/doc_pdf.c b/internal/cbm/doc_pdf.cindex b7a59a7a5..586417b2e 100644--- a/internal/cbm/doc_pdf.c+++ b/internal/cbm/doc_pdf.c@@ -30,10 +30,16 @@#include "pdf/pdf.h"#include "pdf/pdf_unicode.h"+#include <stdatomic.h>#include <stdio.h>#include <stdlib.h>#include <string.h>+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS+/* candidates a join compared, for the cost test */+static atomic_size_t g_join_steps;+#endif+enum {PDF_R3_WINDOW = 400, /* the field test looked this far back for "(" or "," */PDF_PAGE_NAME = 32,@@ -1120,6 +1126,46 @@ static void scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t}tok[i] = emit(&e, kind_syntax(c->kind), sb.p ? sb.p : "", line_of(&e, c->start), 0);}+ /* The emitted candidates of each line, ascending (CSR: lc[lc_off[l] ..+ * lc_off[l + 1])). A join's span covers the end of one line and the start of+ * the next, so only their candidates can overlap it; reading every candidate+ * of the page per join cost (joins x candidates), quadratic in the text. */+ int *lc_off = (int *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, (size_t)(nlines + 2) * sizeof(int));+ int *lc = NULL;+ if (!lc_off) {+ e.failed = true;+ } else {+ for (int f = 0; f < cv.n; f++) {+ if (tok[f] >= 0) {+ int end = cv.v[f].end > cv.v[f].start ? cv.v[f].end - 1 : cv.v[f].start;+ for (uint32_t l = line_of(&e, cv.v[f].start); l <= line_of(&e, end); l++) {+ lc_off[l]++; /* line_of is 1-based: line l - 1 counts at l */+ }+ }+ }+ for (int l = 0; l < nlines; l++) {+ lc_off[l + 1] += lc_off[l];+ }+ lc = (int *)cbm_alloc(CBM_MEM_CLASS_EXTRACT,+ (size_t)(lc_off[nlines] ? lc_off[nlines] : 1) * sizeof(int));+ if (!lc) {+ e.failed = true;+ } else {+ /* lc_off[l] runs as line l's cursor, then shifts back to its start */+ for (int f = 0; f < cv.n; f++) {+ if (tok[f] >= 0) {+ int end = cv.v[f].end > cv.v[f].start ? cv.v[f].end - 1 : cv.v[f].start;+ for (uint32_t l = line_of(&e, cv.v[f].start); l <= line_of(&e, end); l++) {+ lc[lc_off[l - 1]++] = f;+ }+ }+ }+ for (int l = nlines; l > 0; l--) {+ lc_off[l] = lc_off[l - 1];+ }+ lc_off[0] = 0;+ }+ }/* joins */for (int li = 0; li + 1 < nlines && !sb.fail && !e.failed; li++) {const uint32_t *prev = flat.v + starts[li];@@ -1187,9 +1233,25 @@ static void scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_tchar fs = PDF_JOIN_FRAG;sb_put(&sb, &fs, 1);bool first = true;- for (int f = 0; f < cv.n; f++) {+ int ia = lc_off[li];+ int ea = lc_off[li + 1];+ int ib = lc_off[li + 1];+ int eb = lc_off[li + 2];+ while (ia < ea || ib < eb) {+ int f;+ if (ib >= eb || (ia < ea && lc[ia] < lc[ib])) {+ f = lc[ia++];+ } else if (ia >= ea || lc[ib] < lc[ia]) {+ f = lc[ib++];+ } else {+ f = lc[ia++]; /* on both lines */+ ib++;+ }+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS+ atomic_fetch_add_explicit(&g_join_steps, 1, memory_order_relaxed);+#endifconst cand_t *x = &cv.v[f];- if (x->start < e_flat && x->end > s_flat && tok[f] >= 0) {+ if (x->start < e_flat && x->end > s_flat) {char num[16];int nlen = snprintf(num, sizeof(num), first ? "%d" : ",%d", tok[f]);sb_put(&sb, num, (size_t)nlen);@@ -1209,6 +1271,8 @@ static void scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t}cbm_free(CBM_MEM_CLASS_EXTRACT, sb.p);cbm_free(CBM_MEM_CLASS_EXTRACT, tok);+ cbm_free(CBM_MEM_CLASS_EXTRACT, lc_off);+ cbm_free(CBM_MEM_CLASS_EXTRACT, lc);done:candv_free(&cv);u32_free(&flat);@@ -1221,6 +1285,10 @@ void cbm_pdf_test_scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uiconst char *page_qn) {scan_page(ctx, text, len, page, page_qn);}++size_t cbm_pdf_test_join_steps(void) {+ return atomic_load_explicit(&g_join_steps, memory_order_relaxed);+}#endif/* ── The document ────────────────────────────────────────────────── */diff --git a/internal/cbm/doclink.h b/internal/cbm/doclink.hindex e97aaf488..05d3283c4 100644--- a/internal/cbm/doclink.h+++ b/internal/cbm/doclink.h@@ -363,6 +363,8 @@ void cbm_pdf_extract_document(CBMExtractCtx *ctx);* into ctx->result->doc_links), without a PDF around it. */void cbm_pdf_test_scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t page,const char *page_qn);+/* candidates the scanner's joins have compared so far (a cost counter) */+size_t cbm_pdf_test_join_steps(void);#endif#endif /* CBM_DOCLINK_H */diff --git a/tests/test_doc_links_pdf.c b/tests/test_doc_links_pdf.cindex 66a6c47c1..ee4e77996 100644--- a/tests/test_doc_links_pdf.c+++ b/tests/test_doc_links_pdf.c@@ -927,6 +927,41 @@ TEST(pdf_scan_mentions) {PASS();}+/* Join work on a page of `n` lines that all join, eight names a line. */+static size_t tp_join_work(int n) {+ tp_buf_t b = {0};+ for (int i = 0; i < n; i++) {+ tp_puts(&b, "pkg.mod.Alpha pkg.mod.Beta dir/sub/f1.py dir/sub/f2.py pkg.mod.Gamma "+ "pkg.mod.Delta dir/sub/f3.py dir/sub/f4.py end_\n");+ }+ CBMFileResult *r = (CBMFileResult *)calloc(1, sizeof(*r));+ cbm_arena_init(&r->arena);+ CBMExtractCtx ctx;+ memset(&ctx, 0, sizeof(ctx));+ ctx.arena = &r->arena;+ ctx.result = r;+ size_t before = cbm_pdf_test_join_steps();+ cbm_pdf_test_scan_page(&ctx, (const char *)b.p, b.n, 1, "p.doc.page_1");+ size_t work = cbm_pdf_test_join_steps() - before;+ bool ok = !r->doc_links.failed && r->doc_links.count > n * 8;+ cbm_free_result(r);+ free(b.p);+ return ok ? work : 0;+}++/* A join compares only the candidates of its two lines: twice the page, about+ * twice the work (every candidate of the page per join was four times). */+TEST(pdf_scan_join_linear) {+ size_t small = tp_join_work(300);+ size_t large = tp_join_work(600);+ ASSERT_TRUE(small > 0);+ if (large > small * 5 / 2) {+ fprintf(stderr, "join work: %zu for 300 lines, %zu for 600\n", small, large);+ }+ ASSERT_TRUE(large <= small * 5 / 2);+ PASS();+}+/* ── the pipeline ────────────────────────────────────────────────── *//* A PDF whose pages each show the given lines (Helvetica, one Tj a line). */@@ -1185,6 +1220,7 @@ SUITE(doc_links_pdf) {RUN_TEST(pdf_extract_open_strings_linear);RUN_TEST(pdf_extract_form_ladder_bounded);RUN_TEST(pdf_scan_mentions);+ RUN_TEST(pdf_scan_join_linear);RUN_TEST(pdf_links_pipeline);RUN_TEST(pdf_snippet_is_page_text);RUN_TEST(pdf_links_incremental);