Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(extract): capture complete doc comments across languages
Martin Vogeldab9187ca5d4
diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.hindex fb5e446a5..c0bba7792 100644--- a/internal/cbm/cbm.h+++ b/internal/cbm/cbm.h@@ -606,6 +606,11 @@ typedef struct CBMFileResult {// by cbm_free_result(); ordinary single-file results leave these zeroed.struct CBMFileResult **owned_results;int owned_result_count;++ /* The file's own doc, set on its File node: the Go package comment or+ * the Rust inner docs (//!). NULL for other languages and undocumented+ * files. */+ const char *module_doc;} CBMFileResult;// --- Enclosing function cache ---@@ -692,6 +697,11 @@ typedef struct {/* How many nodes the unified walk actually visited (whether or not it ran* out of budget) — the measurement the budget has to be expressed in. */uint32_t walk_nodes_visited;+ /* Doc-comment lookup state (extract_defs.c), NULL until first used and+ * allocated in `scratch`: the memo of parents' child arrays and the Perl+ * POD section index. */+ void *doc_memo;+ void *doc_pod_index;} CBMExtractCtx;// --- Public API ---diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.cindex bb56e2870..b3b6e6acc 100644--- a/internal/cbm/extract_defs.c+++ b/internal/cbm/extract_defs.c@@ -1265,36 +1265,6 @@ static bool is_comment_node(const char *kind) {strcmp(kind, "line_comment") == 0 || strcmp(kind, "multiline_comment") == 0);}-// Extract comment text, truncating to MAX_COMMENT_LEN.-// #1017: snap the cut point back to a complete UTF-8 codepoint boundary.-static char *extract_comment_text(CBMArena *a, TSNode node, const char *source) {- char *text = cbm_node_text(a, node, source);- if (text && strlen(text) > MAX_COMMENT_LEN) {- size_t cut = MAX_COMMENT_LEN;- while (cut > 0 && ((unsigned char)text[cut] & 0xC0) == 0x80)- cut--;- text[cut] = '\0';- }- return text;-}--// Go-specific: type_spec/type_alias comment is before the parent type_declaration.-static const char *extract_go_type_docstring(CBMArena *a, TSNode node, const char *source) {- const char *kind = ts_node_type(node);- if (strcmp(kind, "type_spec") != 0 && strcmp(kind, "type_alias") != 0) {- return NULL;- }- TSNode parent = ts_node_parent(node);- if (ts_node_is_null(parent) || strcmp(ts_node_type(parent), "type_declaration") != 0) {- return NULL;- }- TSNode pprev = ts_node_prev_sibling(parent);- if (!ts_node_is_null(pprev) && is_comment_node(ts_node_type(pprev))) {- return extract_comment_text(a, pprev, source);- }- return NULL;-}-// Python-specific: docstring as first expression_statement -> string in function body.static const char *extract_python_docstring(CBMArena *a, TSNode node, const char *source) {TSNode body = ts_node_child_by_field_name(node, TS_FIELD("body"));@@ -1314,30 +1284,1423 @@ static const char *extract_python_docstring(CBMArena *a, TSNode node, const char}const char *sk = ts_node_type(str);if (strcmp(sk, "string") == 0 || strcmp(sk, "concatenated_string") == 0) {- return extract_comment_text(a, str, source);+ return cbm_node_text(a, str, source);}return NULL;}-// Extract docstring from the node's leading comment.-static const char *extract_docstring(CBMArena *a, TSNode node, const char *source,- CBMLanguage lang) {- if (lang == CBM_LANG_GO) {- const char *doc = extract_go_type_docstring(a, node, source);- if (doc) {- return doc;+/* ── Doc comments ─────────────────────────────────────────────────────────+ * A definition's doc is the comment RUN directly above it, not only the one+ * comment node before it:+ * - a run is contiguous (no blank line) and of one marker style; a Go comment+ * group may mix styles and drops go/ast directives (//go:generate, //line);+ * - it sits above the item's attributes (Rust), its declaration or export+ * wrapper (JS/TS, C#, Go var/const) or its typedef (C);+ * - a comment on the previous code's last line is that code's trailing+ * comment, and a Rust inner doc documents the module, never the item;+ * - a nearer plain comment does not hide a doc-style comment directly above;+ * - a run a blank line away is detached (Go: always; elsewhere: unless it is+ * doc-style), and a run with no letter or digit (a banner) is no doc;+ * - the whole run is kept: the node's properties buffer grows with the doc.+ * Fields, variables, enum members and macros of code languages get the same+ * doc, a Perl sub the POD section that names it, and a Go or Rust file its+ * package comment or inner docs (on the File node).+ *+ * Leading trivia is read from the parent's child array, kept in a small+ * per-file memo: tree-sitter has no parent pointers, so ts_node_parent and+ * ts_node_prev_sibling are O(position), and one lookup per member of an+ * N-member class body cost O(N^2) (65,536 fields in one dotnet test class). */++enum {+ DOC_SPAN_COMMENT = 1,+ DOC_SPAN_ATTR = 2,+ DOC_SPAN_INIT_CAP = 8,+ DOC_SPAN_GROW = 2,+ DOC_MEMO_SLOTS = 8, /* parents whose child arrays stay cached */+ DOC_MEMO_WIDE = 32, /* a level wider than this is cached while descending */+ DOC_BISECT = 2,+};++/* Comment marker families (the slash forms are spelled out in words so this+ * comment does not nest). */+typedef enum {+ DOC_CS_OTHER = 0, /* another family (--, ;, %) */+ DOC_CS_LINE, /* slash-slash, or four and more slashes */+ DOC_CS_LINE_DOC, /* slash-slash-slash */+ DOC_CS_LINE_BANG, /* slash-slash-bang */+ DOC_CS_BLOCK, /* plain block, empty block, star-banner block */+ DOC_CS_BLOCK_DOC, /* slash-star-star */+ DOC_CS_BLOCK_BANG, /* slash-star-bang */+ DOC_CS_HASH, /* hash */+} doc_style_t;++/* One trivia element as a byte span. erow is the EFFECTIVE end row: a node+ * that ends at column 0 (a Rust line_comment includes its newline) ends on the+ * row before. */+typedef struct {+ uint32_t sb;+ uint32_t eb;+ uint32_t srow;+ uint32_t erow;+ uint8_t kind;+ uint8_t style;+} doc_span_t;++typedef struct {+ doc_span_t *items; /* trivia directly before the anchor, in source order */+ int count;+ int cap;+ bool code_before; /* a non-trivia sibling precedes items[0] */+ uint32_t code_erow; /* ... its effective end row */+ uint32_t code_eb; /* ... its end byte (Kotlin gap scan) */+ uint32_t code_end_row; /* ... its raw end row (Kotlin gap scan) */+} doc_trivia_t;++static CBMArena *doc_scratch(CBMExtractCtx *ctx) {+ return ctx->scratch ? ctx->scratch : ctx->arena;+}++static uint32_t doc_erow(TSNode n) {+ TSPoint s = ts_node_start_point(n);+ TSPoint e = ts_node_end_point(n);+ return (e.column == 0 && e.row > s.row) ? e.row - SKIP_ONE : e.row;+}++static bool doc_has_prefix(const char *p, uint32_t n, const char *lit) {+ size_t len = strlen(lit);+ return n >= len && memcmp(p, lit, len) == 0;+}++static doc_style_t doc_style_of(const char *src, uint32_t sb, uint32_t eb) {+ const char *p = src + sb;+ uint32_t n = eb > sb ? eb - sb : 0;+ if (doc_has_prefix(p, n, "////")) {+ return DOC_CS_LINE;+ }+ if (doc_has_prefix(p, n, "///")) {+ return DOC_CS_LINE_DOC;+ }+ if (doc_has_prefix(p, n, "//!")) {+ return DOC_CS_LINE_BANG;+ }+ if (doc_has_prefix(p, n, "//")) {+ return DOC_CS_LINE;+ }+ if (doc_has_prefix(p, n, "/**/") || doc_has_prefix(p, n, "/***")) {+ return DOC_CS_BLOCK;+ }+ if (doc_has_prefix(p, n, "/**")) {+ return DOC_CS_BLOCK_DOC;+ }+ if (doc_has_prefix(p, n, "/*!")) {+ return DOC_CS_BLOCK_BANG;+ }+ if (doc_has_prefix(p, n, "/*")) {+ return DOC_CS_BLOCK;+ }+ return doc_has_prefix(p, n, "#") ? DOC_CS_HASH : DOC_CS_OTHER;+}++/* Languages whose toolchain has a doc-comment syntax distinct from a plain+ * comment (javadoc / KDoc / JSDoc / PHPDoc, rustdoc, C# XML docs, Doxygen). */+static bool doc_lang_has_doc_syntax(CBMLanguage lang) {+ switch (lang) {+ case CBM_LANG_RUST:+ case CBM_LANG_C:+ case CBM_LANG_CPP:+ case CBM_LANG_CUDA:+ case CBM_LANG_OBJC:+ case CBM_LANG_CSHARP:+ case CBM_LANG_SWIFT:+ case CBM_LANG_DART:+ case CBM_LANG_JAVA:+ case CBM_LANG_KOTLIN:+ case CBM_LANG_SCALA:+ case CBM_LANG_GROOVY:+ case CBM_LANG_JAVASCRIPT:+ case CBM_LANG_TYPESCRIPT:+ case CBM_LANG_TSX:+ case CBM_LANG_ARKTS:+ case CBM_LANG_PHP:+ return true;+ default:+ return false;+ }+}++static bool doc_style_is_doc(CBMLanguage lang, int style) {+ switch (lang) {+ case CBM_LANG_C:+ case CBM_LANG_CPP:+ case CBM_LANG_CUDA:+ case CBM_LANG_OBJC:+ return style == DOC_CS_LINE_DOC || style == DOC_CS_LINE_BANG || style == DOC_CS_BLOCK_DOC ||+ style == DOC_CS_BLOCK_BANG;+ case CBM_LANG_RUST:+ case CBM_LANG_CSHARP:+ case CBM_LANG_SWIFT:+ case CBM_LANG_DART:+ return style == DOC_CS_LINE_DOC || style == DOC_CS_BLOCK_DOC;+ default:+ return doc_lang_has_doc_syntax(lang) && style == DOC_CS_BLOCK_DOC;+ }+}++/* Field/Variable docs are for code languages: config languages (YAML, TOML,+ * ...) mint a Variable per key, and a comment there is no API doc. */+static bool doc_lang_member_docs(CBMLanguage lang) {+ return doc_lang_has_doc_syntax(lang) || lang == CBM_LANG_GO || lang == CBM_LANG_PERL;+}++static uint8_t doc_trivia_kind(CBMLanguage lang, const char *kind) {+ if (is_comment_node(kind)) {+ return DOC_SPAN_COMMENT;+ }+ /* A Rust #[attr] is a sibling between the doc and the item. */+ if (lang == CBM_LANG_RUST && strcmp(kind, "attribute_item") == 0) {+ return DOC_SPAN_ATTR;+ }+ return 0;+}++static doc_span_t doc_span_of(TSNode n, const char *src, uint8_t kind) {+ doc_span_t sp;+ sp.sb = ts_node_start_byte(n);+ sp.eb = ts_node_end_byte(n);+ sp.srow = ts_node_start_point(n).row;+ sp.erow = doc_erow(n);+ sp.kind = kind;+ sp.style = (uint8_t)(kind == DOC_SPAN_COMMENT ? doc_style_of(src, sp.sb, sp.eb) : DOC_CS_OTHER);+ return sp;+}++static void doc_push_span(CBMArena *a, doc_trivia_t *t, const doc_span_t *sp) {+ if (t->count == t->cap) {+ int ncap = t->cap ? t->cap * DOC_SPAN_GROW : DOC_SPAN_INIT_CAP;+ doc_span_t *grown = (doc_span_t *)cbm_arena_alloc(a, (size_t)ncap * sizeof(doc_span_t));+ if (!grown) {+ return;+ }+ if (t->count > 0) {+ memcpy(grown, t->items, (size_t)t->count * sizeof(doc_span_t));}+ t->items = grown;+ t->cap = ncap;}+ t->items[t->count++] = *sp;+}- TSNode prev = ts_node_prev_sibling(node);- if (!ts_node_is_null(prev) && is_comment_node(ts_node_type(prev))) {- return extract_comment_text(a, prev, source);+/* A non-trivia sibling before the anchor: trivia seen so far are not its. */+static void doc_note_code(doc_trivia_t *t, TSNode code) {+ t->count = 0;+ t->code_before = true;+ t->code_erow = doc_erow(code);+ t->code_eb = ts_node_end_byte(code);+ t->code_end_row = ts_node_end_point(code).row;+}++/* End of the Kotlin block comment opening at src[i] (block comments nest). */+static uint32_t doc_kotlin_block_end(const char *src, uint32_t i, uint32_t to, uint32_t *row) {+ int depth = 0;+ while (i < to) {+ bool pair = i + SKIP_ONE < to;+ if (pair && src[i] == '/' && src[i + SKIP_ONE] == '*') {+ depth++;+ i += PAIR_LEN;+ continue;+ }+ if (pair && src[i] == '*' && src[i + SKIP_ONE] == '/') {+ depth--;+ i += PAIR_LEN;+ if (depth == 0) {+ return i;+ }+ continue;+ }+ if (src[i] == '\n') {+ (*row)++;+ }+ i++;}+ return i;+}- if (lang == CBM_LANG_PYTHON) {- return extract_python_docstring(a, node, source);+/* tree-sitter-kotlin's automatic-semicolon scanner can swallow a comment that+ * sits between two declarations (seen before `enum class` and `abstract+ * class`): it has no node and survives only as bytes in the gap between the+ * siblings. Recover the comments from that gap. Any other byte is a token the+ * tree does not show, and the run restarts after it. */+static void doc_scan_kotlin_gap(CBMArena *a, const char *src, uint32_t from, uint32_t to,+ uint32_t row, doc_trivia_t *t) {+ uint32_t i = from;+ while (i < to) {+ char c = src[i];+ if (c == '\n') {+ row++;+ i++;+ continue;+ }+ if (c == ' ' || c == '\t' || c == '\r' || c == '\f') {+ i++;+ continue;+ }+ if (c == '/' && i + SKIP_ONE < to &&+ (src[i + SKIP_ONE] == '/' || src[i + SKIP_ONE] == '*')) {+ doc_span_t sp = {i, i, row, row, DOC_SPAN_COMMENT, DOC_CS_OTHER};+ if (src[i + SKIP_ONE] == '/') {+ while (i < to && src[i] != '\n') {+ i++;+ }+ } else {+ i = doc_kotlin_block_end(src, i, to, &row);+ }+ sp.eb = i;+ sp.erow = row;+ sp.style = (uint8_t)doc_style_of(src, sp.sb, sp.eb);+ doc_push_span(a, t, &sp);+ continue;+ }+ t->count = 0;+ t->code_before = true;+ t->code_erow = row;+ i++;}- return NULL;+}++static bool doc_space_byte(char c) {+ return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f';+}++/* Start of the (nesting) block comment whose close ends at src[p), or+ * UINT32_MAX; *row moves to its start row. */+static uint32_t doc_back_block_start(const char *src, uint32_t p, uint32_t *row) {+ uint32_t q = p;+ int depth = 0;+ while (q >= PAIR_LEN) {+ if (src[q - PAIR_LEN] == '*' && src[q - SKIP_ONE] == '/') {+ depth++;+ q -= PAIR_LEN;+ continue;+ }+ if (src[q - PAIR_LEN] == '/' && src[q - SKIP_ONE] == '*') {+ depth--;+ q -= PAIR_LEN;+ if (depth == 0) {+ return q;+ }+ continue;+ }+ if (src[q - SKIP_ONE] == '\n' && *row > 0) {+ (*row)--;+ }+ q--;+ }+ return UINT32_MAX;+}++/* Start of a line comment that fills its line up to src[p), or UINT32_MAX. */+static uint32_t doc_back_line_start(const char *src, uint32_t p) {+ uint32_t ls = p;+ while (ls > 0 && src[ls - SKIP_ONE] != '\n') {+ ls--;+ }+ uint32_t k = ls;+ while (k < p && (src[k] == ' ' || src[k] == '\t')) {+ k++;+ }+ return (k + SKIP_ONE < p && src[k] == '/' && src[k + SKIP_ONE] == '/') ? k : UINT32_MAX;+}++/* tree-sitter-kotlin can also fold such a comment into the PREVIOUS node's+ * byte range (package_header ends after the comment that follows it), so+ * neither a sibling nor the gap holds it. Scan backwards from the anchor over+ * whitespace, block comments and whole-line line comments; stop at anything+ * else. The spans come out nearest-first and are appended in source order. */+static void doc_scan_kotlin_back(CBMArena *a, const char *src, uint32_t anchor_sb,+ uint32_t anchor_row, doc_trivia_t *t) {+ doc_trivia_t rev;+ memset(&rev, 0, sizeof(rev));+ uint32_t p = anchor_sb;+ uint32_t row = anchor_row;+ for (;;) {+ while (p > 0 && doc_space_byte(src[p - SKIP_ONE])) {+ if (src[p - SKIP_ONE] == '\n' && row > 0) {+ row--;+ }+ p--;+ }+ uint32_t srow = row;+ uint32_t start = (p >= PAIR_LEN && src[p - PAIR_LEN] == '*' && src[p - SKIP_ONE] == '/')+ ? doc_back_block_start(src, p, &srow)+ : doc_back_line_start(src, p);+ if (start == UINT32_MAX) {+ break;+ }+ doc_span_t sp = {+ start, p, srow, row, DOC_SPAN_COMMENT, (uint8_t)doc_style_of(src, start, p)};+ doc_push_span(a, &rev, &sp);+ p = start;+ row = srow;+ }+ for (int i = rev.count - SKIP_ONE; i >= 0; i--) {+ doc_push_span(a, t, &rev.items[i]);+ }+ t->code_before = p > 0;+ t->code_erow = row;+}++/* The memo: the child arrays of the last few parents looked up. A node's index+ * is a binary search on start bytes, and its parent is found by descending+ * from the deepest cached ancestor (the root is always reachable): the same+ * answers as the tree-sitter calls, O(log N) per lookup on wide levels. */+typedef struct {+ TSNode node;+ TSNode *kids;+ uint32_t *kid_sb;+ uint32_t n;+ uint32_t cap;+ uint32_t stamp;+ bool used;+} doc_memo_ent_t;++typedef struct {+ doc_memo_ent_t ent[DOC_MEMO_SLOTS];+ uint32_t clock;+} doc_memo_t;++/* The cached entry of `parent`, or the slot to reuse for it (unused first,+ * else least recently used). */+static doc_memo_ent_t *doc_memo_slot(doc_memo_t *m, TSNode parent, bool *hit) {+ doc_memo_ent_t *victim = &m->ent[0];+ for (int i = 0; i < DOC_MEMO_SLOTS; i++) {+ doc_memo_ent_t *e = &m->ent[i];+ if (e->used && ts_node_eq(e->node, parent)) {+ *hit = true;+ return e;+ }+ if (victim->used && (!e->used || e->stamp < victim->stamp)) {+ victim = e;+ }+ }+ *hit = false;+ return victim;+}++/* Fill `e` with the children of `parent`; false when the arrays cannot grow. */+static bool doc_memo_fill(CBMArena *a, doc_memo_ent_t *e, TSNode parent) {+ uint32_t n = ts_node_child_count(parent);+ if (n > e->cap) {+ TSNode *kids = (TSNode *)cbm_arena_alloc(a, (size_t)n * sizeof(TSNode));+ uint32_t *sb = (uint32_t *)cbm_arena_alloc(a, (size_t)n * sizeof(uint32_t));+ if (!kids || !sb) {+ e->used = false;+ return false;+ }+ e->kids = kids;+ e->kid_sb = sb;+ e->cap = n;+ }+ uint32_t k = 0;+ TSTreeCursor cur = ts_tree_cursor_new(parent);+ if (ts_tree_cursor_goto_first_child(&cur)) {+ do {+ e->kids[k] = ts_tree_cursor_current_node(&cur);+ e->kid_sb[k] = ts_node_start_byte(e->kids[k]);+ k++;+ } while (k < n && ts_tree_cursor_goto_next_sibling(&cur));+ }+ ts_tree_cursor_delete(&cur);+ e->node = parent;+ e->n = k;+ e->used = true;+ return true;+}++static const doc_memo_ent_t *doc_memo_get(CBMExtractCtx *ctx, TSNode parent) {+ CBMArena *a = doc_scratch(ctx);+ if (!ctx->doc_memo) {+ doc_memo_t *fresh = (doc_memo_t *)cbm_arena_alloc(a, sizeof(doc_memo_t));+ if (!fresh) {+ return NULL;+ }+ memset(fresh, 0, sizeof(*fresh));+ ctx->doc_memo = fresh;+ }+ doc_memo_t *m = (doc_memo_t *)ctx->doc_memo;+ m->clock++;+ bool hit = false;+ doc_memo_ent_t *e = doc_memo_slot(m, parent, &hit);+ if (!hit && !doc_memo_fill(a, e, parent)) {+ return NULL;+ }+ e->stamp = m->clock;+ return e;+}++/* Index of `node` among the cached children, or -1. */+static int doc_memo_index(const doc_memo_ent_t *e, TSNode node) {+ uint32_t s = ts_node_start_byte(node);+ uint32_t lo = 0;+ uint32_t hi = e->n;+ while (lo < hi) {+ uint32_t mid = lo + ((hi - lo) / DOC_BISECT);+ if (e->kid_sb[mid] < s) {+ lo = mid + SKIP_ONE;+ } else {+ hi = mid;+ }+ }+ for (uint32_t i = lo; i < e->n && e->kid_sb[i] == s; i++) {+ if (ts_node_eq(e->kids[i], node)) {+ return (int)i;+ }+ }+ return CBM_NOT_FOUND;+}++/* Index of the cached child whose byte range contains `node`, or -1. */+static int doc_memo_containing(const doc_memo_ent_t *e, TSNode node) {+ uint32_t s = ts_node_start_byte(node);+ uint32_t en = ts_node_end_byte(node);+ uint32_t lo = 0;+ uint32_t hi = e->n;+ while (lo < hi) {+ uint32_t mid = lo + ((hi - lo) / DOC_BISECT);+ if (e->kid_sb[mid] <= s) {+ lo = mid + SKIP_ONE;+ } else {+ hi = mid;+ }+ }+ for (int i = (int)lo - SKIP_ONE; i >= 0; i--) {+ TSNode kid = e->kids[i];+ uint32_t ke = ts_node_end_byte(kid);+ if (ke >= en && (ke > e->kid_sb[i] || ts_node_eq(kid, node))) {+ return i;+ }+ if (ke <= s && e->kid_sb[i] < s) {+ break;+ }+ }+ return CBM_NOT_FOUND;+}++/* The deepest cached node that contains `node` (the node itself excluded). */+static const doc_memo_ent_t *doc_memo_ancestor(const doc_memo_t *m, TSNode node) {+ const doc_memo_ent_t *best = NULL;+ uint32_t ns = ts_node_start_byte(node);+ uint32_t ne = ts_node_end_byte(node);+ for (int i = 0; m && i < DOC_MEMO_SLOTS; i++) {+ const doc_memo_ent_t *e = &m->ent[i];+ if (!e->used || ts_node_eq(e->node, node)) {+ continue;+ }+ uint32_t es = ts_node_start_byte(e->node);+ uint32_t ee = ts_node_end_byte(e->node);+ if (es <= ns && ne <= ee &&+ (!best || ee - es < ts_node_end_byte(best->node) - ts_node_start_byte(best->node))) {+ best = e;+ }+ }+ return best;+}++/* ts_node_parent() through the memo: descend from the deepest cached ancestor,+ * binary search on wide levels (cached on the way), linear on narrow ones. */+static TSNode doc_parent(CBMExtractCtx *ctx, TSNode node) {+ if (ts_node_eq(node, ctx->root)) {+ TSNode none = {0};+ return none;+ }+ const doc_memo_ent_t *e = doc_memo_ancestor((const doc_memo_t *)ctx->doc_memo, node);+ if (!e) {+ e = doc_memo_get(ctx, ctx->root);+ }+ while (e) {+ int k = doc_memo_containing(e, node);+ if (k < 0) {+ break;+ }+ TSNode cur = e->kids[k];+ if (ts_node_eq(cur, node)) {+ return e->node;+ }+ e = NULL;+ while (!ts_node_is_null(cur)) {+ if (ts_node_child_count(cur) > DOC_MEMO_WIDE) {+ e = doc_memo_get(ctx, cur);+ break;+ }+ TSNode next = ts_node_child_with_descendant(cur, node);+ if (ts_node_is_null(next)) {+ break;+ }+ if (ts_node_eq(next, node)) {+ return cur;+ }+ cur = next;+ }+ }+ return ts_node_parent(node); /* not reachable through the memo */+}++/* Trivia before the child at index mk of a cached child array. */+static void doc_collect_cached(CBMExtractCtx *ctx, const doc_memo_ent_t *me, int mk,+ doc_trivia_t *t) {+ int j = mk - SKIP_ONE;+ while (j >= 0 && doc_trivia_kind(ctx->language, ts_node_type(me->kids[j]))) {+ j--;+ }+ if (j >= 0) {+ doc_note_code(t, me->kids[j]);+ }+ for (int i = j + SKIP_ONE; i < mk; i++) {+ uint8_t kind = doc_trivia_kind(ctx->language, ts_node_type(me->kids[i]));+ doc_span_t sp = doc_span_of(me->kids[i], ctx->source, kind);+ doc_push_span(doc_scratch(ctx), t, &sp);+ }+}++/* The same without the memo (its arrays could not be allocated): one cursor+ * pass over the parent. False when `anchor` is not among its children. */+static bool doc_collect_cursor(CBMExtractCtx *ctx, TSNode parent, TSNode anchor, doc_trivia_t *t) {+ bool found = false;+ TSTreeCursor cur = ts_tree_cursor_new(parent);+ if (ts_tree_cursor_goto_first_child(&cur)) {+ do {+ TSNode ch = ts_tree_cursor_current_node(&cur);+ if (ts_node_eq(ch, anchor)) {+ found = true;+ break;+ }+ uint8_t kind = doc_trivia_kind(ctx->language, ts_node_type(ch));+ if (kind) {+ doc_span_t sp = doc_span_of(ch, ctx->source, kind);+ doc_push_span(doc_scratch(ctx), t, &sp);+ } else {+ doc_note_code(t, ch);+ }+ } while (ts_tree_cursor_goto_next_sibling(&cur));+ }+ ts_tree_cursor_delete(&cur);+ return found;+}++/* Kotlin comments the grammar hid: in the gap after the previous sibling, else+ * folded into the previous node (see the two scanners above). */+static void doc_collect_kotlin(CBMExtractCtx *ctx, TSNode parent, TSNode anchor, doc_trivia_t *t) {+ CBMArena *a = doc_scratch(ctx);+ uint32_t from = t->code_before ? t->code_eb : ts_node_start_byte(parent);+ uint32_t row = t->code_before ? t->code_end_row : ts_node_start_point(parent).row;+ uint32_t to = ts_node_start_byte(anchor);+ if (from < to) {+ doc_scan_kotlin_gap(a, ctx->source, from, to, row, t);+ }+ if (t->count == 0) {+ doc_scan_kotlin_back(a, ctx->source, to, ts_node_start_point(anchor).row, t);+ }+}++/* Leading trivia of `anchor`, in source order. */+static void doc_collect_trivia(CBMExtractCtx *ctx, TSNode anchor, doc_trivia_t *t) {+ memset(t, 0, sizeof(*t));+ TSNode parent = doc_parent(ctx, anchor);+ if (ts_node_is_null(parent)) {+ return;+ }+ const doc_memo_ent_t *me = doc_memo_get(ctx, parent);+ int mk = me ? doc_memo_index(me, anchor) : CBM_NOT_FOUND;+ bool found = mk >= 0;+ if (found) {+ doc_collect_cached(ctx, me, mk, t);+ } else {+ found = doc_collect_cursor(ctx, parent, anchor, t);+ }+ if (!found) {+ memset(t, 0, sizeof(*t));+ return;+ }+ if (t->count == 0 && ctx->language == CBM_LANG_KOTLIN) {+ doc_collect_kotlin(ctx, parent, anchor, t);+ }+}++/* go/ast CommentGroup.Text: a line comment with no space after the slashes is+ * a directive, not documentation: //line, //extern, //export and the+ * //[a-z0-9]+:[a-z0-9] form (//go:generate, //nolint:errcheck, ...). */+static bool doc_go_is_directive(const char *src, const doc_span_t *sp) {+ const char *p = src + sp->sb;+ uint32_t n = sp->eb > sp->sb ? sp->eb - sp->sb : 0;+ if (n <= PAIR_LEN || p[0] != '/' || p[SKIP_ONE] != '/' || p[PAIR_LEN] == ' ') {+ return false;+ }+ p += PAIR_LEN;+ n -= PAIR_LEN;+ if (doc_has_prefix(p, n, "line ") || doc_has_prefix(p, n, "extern ") ||+ doc_has_prefix(p, n, "export ")) {+ return true;+ }+ uint32_t colon = 0;+ while (colon < n && p[colon] != ':' && p[colon] != '\n') {+ colon++;+ }+ if (colon == 0 || colon >= n || p[colon] != ':' || colon + SKIP_ONE >= n) {+ return false;+ }+ for (uint32_t i = 0; i <= colon + SKIP_ONE; i++) {+ char b = p[i];+ if (i != colon && !((b >= 'a' && b <= 'z') || (b >= '0' && b <= '9'))) {+ return false;+ }+ }+ return true;+}++/* UTF-8 bytes of the banner characters (see doc_has_words). */+enum {+ DOC_U8_ASCII_END = 0x80,+ DOC_U8_CONT_MASK = 0xC0, /* 10xxxxxx: continuation byte */+ DOC_U8_CONT = 0x80,+ DOC_U8_LATIN1 = 0xC2, /* lead byte of U+0080..U+00BF */+ DOC_U8_MIDDLE_DOT = 0xB7, /* U+00B7 */+ DOC_U8_U2000 = 0xE2, /* lead byte of U+2000..U+2FFF */+ DOC_U8_PUNCT = 0x80, /* U+2000..U+203F */+ DOC_U8_DASH_FIRST = 0x90, /* U+2010 hyphen */+ DOC_U8_BULLET_LAST = 0xA7, /* U+2027 hyphenation point */+ DOC_U8_BOX_FIRST = 0x94, /* U+2500 box drawing */+ DOC_U8_SHAPES_LAST = 0x97, /* U+25FF, end of the geometric shapes */+ DOC_U8_LATIN1_LEN = 2,+ DOC_U8_U2000_LEN = 3,+};++static bool doc_banner_char(const unsigned char *p, size_t left) {+ if (left >= DOC_U8_LATIN1_LEN && p[0] == DOC_U8_LATIN1 && p[SKIP_ONE] == DOC_U8_MIDDLE_DOT) {+ return true;+ }+ if (left < DOC_U8_U2000_LEN || p[0] != DOC_U8_U2000) {+ return false;+ }+ if (p[SKIP_ONE] >= DOC_U8_BOX_FIRST && p[SKIP_ONE] <= DOC_U8_SHAPES_LAST) {+ return true;+ }+ return p[SKIP_ONE] == DOC_U8_PUNCT && p[PAIR_LEN] >= DOC_U8_DASH_FIRST &&+ p[PAIR_LEN] <= DOC_U8_BULLET_LAST;+}++/* A comment with no letter or digit is a divider or banner, not documentation.+ * Any non-ASCII character counts as a letter except the ones banners are drawn+ * with: U+00B7, the dashes and bullets U+2010..U+2027, and the box-drawing,+ * block and geometric-shape characters U+2500..U+25FF. */+static bool doc_has_words(const char *s, size_t n) {+ const unsigned char *u = (const unsigned char *)s;+ for (size_t i = 0; i < n; i++) {+ if (u[i] < DOC_U8_ASCII_END) {+ if (isalnum(u[i])) {+ return true;+ }+ } else if ((u[i] & DOC_U8_CONT_MASK) != DOC_U8_CONT && !doc_banner_char(u + i, n - i)) {+ return true;+ }+ }+ return false;+}++static bool doc_span_kept(const char *src, const doc_span_t *sp, bool go_directives) {+ return sp->kind == DOC_SPAN_COMMENT && !(go_directives && doc_go_is_directive(src, sp));+}++/* Text of items[first..last]: one comment keeps its exact bytes; several are+ * joined by newlines, each without its own trailing newline. NULL when nothing+ * is left (Go directives only) or when no comment has a letter or digit. */+static const char *doc_run_text(CBMExtractCtx *ctx, const doc_trivia_t *t, int first, int last,+ bool go_directives) {+ const char *src = ctx->source;+ int kept = 0;+ bool words = false;+ size_t total = 0;+ for (int k = first; k <= last; k++) {+ const doc_span_t *sp = &t->items[k];+ if (!doc_span_kept(src, sp, go_directives)) {+ continue;+ }+ total += (size_t)(sp->eb - sp->sb) + SKIP_ONE;+ words = words || doc_has_words(src + sp->sb, sp->eb - sp->sb);+ kept++;+ }+ if (kept == 0 || !words) {+ return NULL;+ }+ char *buf = (char *)cbm_arena_alloc(ctx->arena, total + SKIP_ONE);+ if (!buf) {+ return NULL;+ }+ size_t w = 0;+ for (int k = first; k <= last; k++) {+ const doc_span_t *sp = &t->items[k];+ if (!doc_span_kept(src, sp, go_directives)) {+ continue;+ }+ uint32_t eb = sp->eb;+ while (kept > SKIP_ONE && eb > sp->sb &&+ (src[eb - SKIP_ONE] == '\n' || src[eb - SKIP_ONE] == '\r')) {+ eb--;+ }+ if (w > 0) {+ buf[w++] = '\n';+ }+ memcpy(buf + w, src + sp->sb, eb - sp->sb);+ w += eb - sp->sb;+ }+ buf[w] = '\0';+ return buf;+}++/* Index of the comment nearest the item, above its Rust attributes, or -1. */+static int doc_nearest_comment(const doc_trivia_t *t) {+ int k = t->count - SKIP_ONE;+ while (k >= 0 && t->items[k].kind == DOC_SPAN_ATTR) {+ k--;+ }+ return (k >= 0 && t->items[k].kind == DOC_SPAN_COMMENT) ? k : CBM_NOT_FOUND;+}++/* A nearer plain comment does not hide a doc-style comment directly above it+ * (a javadoc, then `// NOSONAR`): the index of that doc comment, else `near`. */+static int doc_prefer_doc_style(CBMLanguage lang, const doc_trivia_t *t, int near) {+ if (!doc_lang_has_doc_syntax(lang) || doc_style_is_doc(lang, t->items[near].style)) {+ return near;+ }+ for (int j = near - SKIP_ONE; j >= 0; j--) {+ const doc_span_t *sp = &t->items[j];+ if (t->items[j + SKIP_ONE].srow > sp->erow + SKIP_ONE) {+ break; /* a blank line */+ }+ if (sp->kind == DOC_SPAN_ATTR) {+ continue;+ }+ if (sp->kind != DOC_SPAN_COMMENT) {+ break;+ }+ if (doc_style_is_doc(lang, sp->style)) {+ return j;+ }+ }+ return near;+}++/* First index of the run that ends at `near`: contiguous, one style (a Go+ * comment group mixes styles), not the previous code's trailing comment. */+static int doc_run_first(CBMLanguage lang, const doc_trivia_t *t, int near) {+ int style = t->items[near].style;+ int first = near;+ for (int j = near - SKIP_ONE; j >= 0; j--) {+ const doc_span_t *sp = &t->items[j];+ if (sp->kind != DOC_SPAN_COMMENT || t->items[j + SKIP_ONE].srow > sp->erow + SKIP_ONE) {+ break; /* an attribute or a blank line ends the run */+ }+ if (lang != CBM_LANG_GO && sp->style != style) {+ break;+ }+ if (j == 0 && t->code_before && sp->srow == t->code_erow) {+ break;+ }+ first = j;+ }+ return first;+}++/* The doc of an anchor from its leading trivia. */+static const char *doc_from_trivia(CBMExtractCtx *ctx, const doc_trivia_t *t,+ uint32_t anchor_srow) {+ CBMLanguage lang = ctx->language;+ int near = t->items ? doc_nearest_comment(t) : CBM_NOT_FOUND;+ if (near < 0) {+ return NULL;+ }+ near = doc_prefer_doc_style(lang, t, near);+ int style = t->items[near].style;+ if (lang == CBM_LANG_RUST && (style == DOC_CS_LINE_BANG || style == DOC_CS_BLOCK_BANG)) {+ return NULL; /* an inner doc: it documents the enclosing module */+ }+ if (near == 0 && t->code_before && t->items[0].srow == t->code_erow) {+ return NULL; /* the previous code's trailing comment */+ }+ uint32_t next_row = near + SKIP_ONE < t->count ? t->items[near + SKIP_ONE].srow : anchor_srow;+ bool detached = next_row > t->items[near].erow + SKIP_ONE;+ if (detached && (lang == CBM_LANG_GO || !doc_style_is_doc(lang, style))) {+ return NULL;+ }+ return doc_run_text(ctx, t, doc_run_first(lang, t, near), near, lang == CBM_LANG_GO);+}++static const char *doc_for_anchor(CBMExtractCtx *ctx, TSNode anchor) {+ doc_trivia_t t;+ doc_collect_trivia(ctx, anchor, &t);+ return doc_from_trivia(ctx, &t, ts_node_start_point(anchor).row);+}++static bool doc_kind_is(TSNode n, const char *kind) {+ return !ts_node_is_null(n) && strcmp(ts_node_type(n), kind) == 0;+}++/* JS/TS: a function or class expression is documented above its declarator,+ * field, pair or assignment statement, a declarator above its declaration,+ * and any of them above `declare` and `export`. */+static TSNode doc_anchor_js(CBMExtractCtx *ctx, TSNode cur) {+ const char *k = ts_node_type(cur);+ if (strcmp(k, "arrow_function") == 0 || strcmp(k, "function_expression") == 0 ||+ strcmp(k, "generator_function") == 0 || strcmp(k, "class") == 0) {+ TSNode p = doc_parent(ctx, cur);+ if (doc_kind_is(p, "variable_declarator") || doc_kind_is(p, "public_field_definition") ||+ doc_kind_is(p, "field_definition") || doc_kind_is(p, "pair")) {+ cur = p;+ } else if (doc_kind_is(p, "assignment_expression")) {+ TSNode pp = doc_parent(ctx, p);+ if (doc_kind_is(pp, "expression_statement")) {+ cur = pp;+ }+ }+ }+ if (doc_kind_is(cur, "variable_declarator")) {+ TSNode p = doc_parent(ctx, cur);+ if (doc_kind_is(p, "lexical_declaration") || doc_kind_is(p, "variable_declaration")) {+ cur = p;+ }+ }+ TSNode p = doc_parent(ctx, cur);+ if (doc_kind_is(p, "ambient_declaration")) {+ cur = p;+ p = doc_parent(ctx, cur);+ }+ return doc_kind_is(p, "export_statement") ? p : cur;+}++/* C#: a field is documented above its field declaration, a top-level+ * function above its global statement. */+static TSNode doc_anchor_csharp(CBMExtractCtx *ctx, TSNode cur) {+ if (doc_kind_is(cur, "variable_declarator")) {+ TSNode p = doc_parent(ctx, cur);+ if (doc_kind_is(p, "variable_declaration")) {+ cur = p;+ }+ }+ if (doc_kind_is(cur, "variable_declaration")) {+ TSNode p = doc_parent(ctx, cur);+ if (doc_kind_is(p, "field_declaration") || doc_kind_is(p, "event_field_declaration")) {+ cur = p;+ }+ }+ TSNode p = doc_parent(ctx, cur);+ return doc_kind_is(p, "global_statement") ? p : cur;+}++/* C family: `typedef struct X {..} Y;` and `struct X {..} v;` are documented+ * above the declaration. Only a specifier WITH a body is a definition; a+ * bodiless one is a usage. */+static TSNode doc_anchor_c(CBMExtractCtx *ctx, TSNode cur) {+ if ((doc_kind_is(cur, "struct_specifier") || doc_kind_is(cur, "enum_specifier") ||+ doc_kind_is(cur, "union_specifier") || doc_kind_is(cur, "class_specifier")) &&+ !ts_node_is_null(ts_node_child_by_field_name(cur, TS_FIELD("body")))) {+ TSNode p = doc_parent(ctx, cur);+ if (doc_kind_is(p, "type_definition") || doc_kind_is(p, "declaration")) {+ return p;+ }+ }+ return cur;+}++/* Go: `var x = 1` (no parentheses) is documented above the declaration. */+static TSNode doc_anchor_go(CBMExtractCtx *ctx, TSNode cur) {+ if (doc_kind_is(cur, "var_spec") || doc_kind_is(cur, "const_spec")) {+ TSNode p = doc_parent(ctx, cur);+ if ((doc_kind_is(p, "var_declaration") || doc_kind_is(p, "const_declaration")) &&+ ts_node_eq(ts_node_child(p, SECOND_CHILD_IDX), cur)) {+ return p;+ }+ }+ return cur;+}++/* The node whose leading comments document `node`. */+static TSNode doc_anchor(CBMExtractCtx *ctx, TSNode node) {+ switch (ctx->language) {+ case CBM_LANG_JAVASCRIPT:+ case CBM_LANG_TYPESCRIPT:+ case CBM_LANG_TSX:+ case CBM_LANG_ARKTS:+ return doc_anchor_js(ctx, node);+ case CBM_LANG_CSHARP:+ return doc_anchor_csharp(ctx, node);+ case CBM_LANG_C:+ case CBM_LANG_CPP:+ case CBM_LANG_CUDA:+ case CBM_LANG_OBJC:+ return doc_anchor_c(ctx, node);+ case CBM_LANG_GO:+ return doc_anchor_go(ctx, node);+ default:+ return node;+ }+}++/* ── Perl POD: the =head / =item section that names the sub ── */++enum { POD_HEAD = 1, POD_ITEM, POD_OVER, POD_BACK, POD_CUT, POD_OTHER };++typedef struct {+ uint32_t sb; /* command paragraph */+ uint32_t eb;+ uint32_t wb; /* end of the command word: its argument starts here */+ uint8_t type;+ uint8_t level; /* =headN: N; =item and =back: list depth */+} pod_cmd_t;++typedef struct {+ const char *name;+ size_t name_len;+ uint32_t sb;+ uint32_t eb;+} pod_sec_t;++typedef struct {+ pod_sec_t *items;+ int count;+ int cap;+} pod_index_t;++static bool pod_line_is_blank(const char *src, uint32_t b, uint32_t e) {+ for (uint32_t k = b; k < e; k++) {+ if (src[k] != ' ' && src[k] != '\t' && src[k] != '\r') {+ return false;+ }+ }+ return true;+}++/* Next POD paragraph (a maximal run of non-blank lines) in src[*pos, pe). */+static bool pod_next_para(const char *src, uint32_t *pos, uint32_t pe, uint32_t *psb,+ uint32_t *peb) {+ uint32_t i = *pos;+ bool in = false;+ uint32_t s = 0;+ while (i < pe) {+ uint32_t e = i;+ while (e < pe && src[e] != '\n') {+ e++;+ }+ bool blank = pod_line_is_blank(src, i, e);+ if (!blank && !in) {+ in = true;+ s = i;+ }+ if (blank && in) {+ *psb = s;+ *peb = i;+ *pos = i;+ return true;+ }+ i = e < pe ? e + SKIP_ONE : pe;+ }+ *pos = pe;+ if (in) {+ *psb = s;+ *peb = pe;+ }+ return in;+}++static bool pod_word_is(const char *w, size_t wl, const char *lit) {+ return wl == strlen(lit) && strncmp(w, lit, wl) == 0;+}++/* Classify one command paragraph; `depth` is the =over nesting. */+static pod_cmd_t pod_classify(const char *src, uint32_t sb, uint32_t eb, int *depth) {+ uint32_t wb = sb + SKIP_ONE;+ while (wb < eb && (isalnum((unsigned char)src[wb]) || src[wb] == '_')) {+ wb++;+ }+ const char *w = src + sb + SKIP_ONE;+ size_t wl = wb - sb - SKIP_ONE;+ pod_cmd_t c = {sb, eb, wb, POD_OTHER, 0};+ size_t head_len = strlen("head");+ if (wl == head_len + SKIP_ONE && strncmp(w, "head", head_len) == 0 &&+ isdigit((unsigned char)w[head_len])) {+ c.type = POD_HEAD;+ c.level = (uint8_t)(w[head_len] - '0');+ } else if (pod_word_is(w, wl, "item")) {+ c.type = POD_ITEM;+ c.level = (uint8_t)*depth;+ } else if (pod_word_is(w, wl, "over")) {+ c.type = POD_OVER;+ (*depth)++;+ } else if (pod_word_is(w, wl, "back")) {+ c.type = POD_BACK;+ c.level = (uint8_t)*depth;+ if (*depth > 0) {+ (*depth)--;+ }+ } else if (pod_word_is(w, wl, "cut")) {+ c.type = POD_CUT;+ }+ return c;+}++/* Command paragraphs of one POD block, in order. */+static int pod_commands(CBMArena *a, const char *src, uint32_t pb, uint32_t pe, pod_cmd_t **out) {+ int cap = DOC_SPAN_INIT_CAP;+ int n = 0;+ pod_cmd_t *cmds = (pod_cmd_t *)cbm_arena_alloc(a, (size_t)cap * sizeof(pod_cmd_t));+ *out = cmds;+ if (!cmds) {+ return 0;+ }+ int depth = 0;+ uint32_t pos = pb;+ uint32_t sb = 0;+ uint32_t eb = 0;+ while (pod_next_para(src, &pos, pe, &sb, &eb)) {+ if (src[sb] != '=' || sb + SKIP_ONE >= eb || !isalpha((unsigned char)src[sb + SKIP_ONE])) {+ continue;+ }+ if (n == cap) {+ int ncap = cap * DOC_SPAN_GROW;+ pod_cmd_t *grown = (pod_cmd_t *)cbm_arena_alloc(a, (size_t)ncap * sizeof(pod_cmd_t));+ if (!grown) {+ break;+ }+ memcpy(grown, cmds, (size_t)n * sizeof(pod_cmd_t));+ cmds = grown;+ cap = ncap;+ *out = cmds;+ }+ cmds[n++] = pod_classify(src, sb, eb, &depth);+ }+ return n;+}++/* End byte of the section that cmds[ci] (a =head or =item) opens. */+static uint32_t pod_section_end(const pod_cmd_t *cmds, int n, int ci, uint32_t pe) {+ const pod_cmd_t *c = &cmds[ci];+ for (int j = ci + SKIP_ONE; j < n; j++) {+ const pod_cmd_t *d = &cmds[j];+ if (d->type == POD_CUT) {+ return d->sb;+ }+ if (d->type == POD_HEAD && (c->type == POD_ITEM || d->level <= c->level)) {+ return d->sb;+ }+ if (c->type == POD_ITEM && (d->type == POD_ITEM || d->type == POD_BACK) &&+ d->level == c->level) {+ return d->sb;+ }+ }+ return pe;+}++/* A heading's text with POD formatting codes (X<..>, X<< .. >>) removed and+ * newlines and tabs as spaces. */+static char *pod_plain_heading(CBMArena *a, const char *src, uint32_t b, uint32_t e) {+ char *buf = (char *)cbm_arena_alloc(a, (size_t)(e - b) + SKIP_ONE);+ if (!buf) {+ return NULL;+ }+ size_t w = 0;+ for (uint32_t i = b; i < e; i++) {+ char c = src[i];+ if (c >= 'A' && c <= 'Z' && i + SKIP_ONE < e && src[i + SKIP_ONE] == '<') {+ uint32_t j = i + SKIP_ONE;+ while (j < e && src[j] == '<') {+ j++;+ }+ if (j - i > PAIR_LEN && j < e && (src[j] == ' ' || src[j] == '\t')) {+ j++; /* X<< text >> */+ }+ i = j - SKIP_ONE;+ continue;+ }+ if (c == '>' && !(w > 0 && buf[w - SKIP_ONE] == '-')) {+ continue;+ }+ buf[w++] = (c == '\n' || c == '\r' || c == '\t') ? ' ' : c;+ }+ buf[w] = '\0';+ return buf;+}++static const char *pod_skip_spaces(const char *p) {+ while (*p == ' ') {+ p++;+ }+ return p;+}++/* The sub a heading names: after a list bullet ("*", "1."), a `$obj->`+ * receiver and a sigil, the identifier that follows. Returns its length+ * (0: none). */+static size_t pod_heading_name(CBMArena *a, const char *src, uint32_t b, uint32_t e,+ const char **name) {+ const char *buf = pod_plain_heading(a, src, b, e);+ if (!buf) {+ return 0;+ }+ const char *p = pod_skip_spaces(buf);+ if (*p == '*') {+ p++;+ } else if (isdigit((unsigned char)*p)) {+ while (isdigit((unsigned char)*p)) {+ p++;+ }+ if (*p == '.') {+ p++;+ }+ }+ p = pod_skip_spaces(p);+ if (*p == '$') {+ const char *q = p + SKIP_ONE;+ while (isalnum((unsigned char)*q) || *q == '_') {+ q++;+ }+ q = pod_skip_spaces(q);+ if (q > p + SKIP_ONE && q[0] == '-' && q[SKIP_ONE] == '>') {+ p = pod_skip_spaces(q + PAIR_LEN);+ }+ }+ if (*p == '$' || *p == '@' || *p == '%' || *p == '&') {+ p++;+ }+ if (!(isalpha((unsigned char)*p) || *p == '_')) {+ return 0;+ }+ const char *s = p;+ while (isalnum((unsigned char)*p) || *p == '_') {+ p++;+ }+ *name = s;+ return (size_t)(p - s);+}++static uint32_t pod_trim_end(const char *src, uint32_t sb, uint32_t eb) {+ while (eb > sb && (src[eb - SKIP_ONE] == '\n' || src[eb - SKIP_ONE] == '\r' ||+ src[eb - SKIP_ONE] == ' ' || src[eb - SKIP_ONE] == '\t')) {+ eb--;+ }+ return eb;+}++static void pod_index_push(CBMArena *a, pod_index_t *ix, const pod_sec_t *sec) {+ if (ix->count == ix->cap) {+ int ncap = ix->cap ? ix->cap * DOC_SPAN_GROW : DOC_SPAN_INIT_CAP;+ pod_sec_t *grown = (pod_sec_t *)cbm_arena_alloc(a, (size_t)ncap * sizeof(pod_sec_t));+ if (!grown) {+ return;+ }+ if (ix->count > 0) {+ memcpy(grown, ix->items, (size_t)ix->count * sizeof(pod_sec_t));+ }+ ix->items = grown;+ ix->cap = ncap;+ }+ ix->items[ix->count++] = *sec;+}++/* Index the named =head / =item sections of one POD block. */+static void pod_index_block(CBMExtractCtx *ctx, pod_index_t *ix, TSNode pod) {+ CBMArena *a = doc_scratch(ctx);+ uint32_t pb = ts_node_start_byte(pod);+ uint32_t pe = ts_node_end_byte(pod);+ pod_cmd_t *cmds = NULL;+ int n = pod_commands(a, ctx->source, pb, pe, &cmds);+ for (int ci = 0; ci < n; ci++) {+ if (cmds[ci].type != POD_HEAD && cmds[ci].type != POD_ITEM) {+ continue;+ }+ const char *nm = NULL;+ size_t nl = pod_heading_name(a, ctx->source, cmds[ci].wb, cmds[ci].eb, &nm);+ if (nl == 0) {+ continue;+ }+ uint32_t se = pod_section_end(cmds, n, ci, pe);+ pod_sec_t sec = {nm, nl, cmds[ci].sb, pod_trim_end(ctx->source, cmds[ci].sb, se)};+ pod_index_push(a, ix, &sec);+ }+}++/* The file's POD sections by name, built on the first Perl sub. */+static const pod_index_t *pod_index(CBMExtractCtx *ctx) {+ if (ctx->doc_pod_index) {+ return (const pod_index_t *)ctx->doc_pod_index;+ }+ pod_index_t *ix = (pod_index_t *)cbm_arena_alloc(doc_scratch(ctx), sizeof(pod_index_t));+ if (!ix) {+ return NULL;+ }+ memset(ix, 0, sizeof(*ix));+ TSTreeCursor cur = ts_tree_cursor_new(ctx->root);+ if (ts_tree_cursor_goto_first_child(&cur)) {+ do {+ TSNode ch = ts_tree_cursor_current_node(&cur);+ if (strcmp(ts_node_type(ch), "pod") == 0) {+ pod_index_block(ctx, ix, ch);+ }+ } while (ts_tree_cursor_goto_next_sibling(&cur));+ }+ ts_tree_cursor_delete(&cur);+ ctx->doc_pod_index = ix;+ return ix;+}++static bool pod_sec_names(const pod_sec_t *s, const char *name, size_t nl) {+ return s->name_len == nl && memcmp(s->name, name, nl) == 0;+}++/* Every POD section whose =head / =item names the sub exactly, joined in+ * document order; NULL when none does. */+static const char *pod_doc_by_name(CBMExtractCtx *ctx, const char *name) {+ const pod_index_t *ix = pod_index(ctx);+ if (!ix || !name || !name[0]) {+ return NULL;+ }+ size_t nl = strlen(name);+ size_t total = 0;+ for (int i = 0; i < ix->count; i++) {+ if (pod_sec_names(&ix->items[i], name, nl)) {+ total += (size_t)(ix->items[i].eb - ix->items[i].sb) + PAIR_LEN;+ }+ }+ if (total == 0) {+ return NULL;+ }+ char *buf = (char *)cbm_arena_alloc(ctx->arena, total + SKIP_ONE);+ if (!buf) {+ return NULL;+ }+ size_t w = 0;+ for (int i = 0; i < ix->count; i++) {+ const pod_sec_t *s = &ix->items[i];+ if (!pod_sec_names(s, name, nl)) {+ continue;+ }+ if (w > 0) {+ buf[w++] = '\n';+ buf[w++] = '\n';+ }+ memcpy(buf + w, ctx->source + s->sb, s->eb - s->sb);+ w += s->eb - s->sb;+ }+ buf[w] = '\0';+ return buf;+}++/* ── Doc entry points ── */++/* go/doc: a type spec's own doc, else its declaration's (the doc of a+ * single-spec `type X ...` sits above the `type` keyword). */+static const char *doc_go_type(CBMExtractCtx *ctx, TSNode node) {+ const char *own = doc_for_anchor(ctx, node);+ TSNode parent = doc_parent(ctx, node);+ if (own || !doc_kind_is(parent, "type_declaration")) {+ return own;+ }+ return doc_for_anchor(ctx, parent);+}++/* Doc of a Function/Method/Class-like definition. */+static const char *extract_docstring(CBMExtractCtx *ctx, TSNode node, const char *name) {+ CBMLanguage lang = ctx->language;+ if (lang == CBM_LANG_PERL) {+ const char *pod = pod_doc_by_name(ctx, name);+ if (pod) {+ return pod;+ }+ }+ if (lang == CBM_LANG_GO &&+ (doc_kind_is(node, "type_spec") || doc_kind_is(node, "type_alias"))) {+ return doc_go_type(ctx, node);+ }+ const char *doc = doc_for_anchor(ctx, doc_anchor(ctx, node));+ if (!doc && lang == CBM_LANG_PYTHON) {+ return extract_python_docstring(ctx->arena, node, ctx->source);+ }+ return doc;+}++/* Doc of a Field, Variable, enum member or Macro (code languages only). */+static const char *extract_member_docstring(CBMExtractCtx *ctx, TSNode node) {+ if (!doc_lang_member_docs(ctx->language)) {+ return NULL;+ }+ return doc_for_anchor(ctx, doc_anchor(ctx, node));+}++/* Go package comment: the comment group touching `package`, directives+ * dropped (go/doc). */+static const char *doc_go_package(CBMExtractCtx *ctx) {+ doc_trivia_t t;+ memset(&t, 0, sizeof(t));+ TSNode pkg = {0};+ bool found = false;+ TSTreeCursor cur = ts_tree_cursor_new(ctx->root);+ if (ts_tree_cursor_goto_first_child(&cur)) {+ do {+ TSNode ch = ts_tree_cursor_current_node(&cur);+ const char *k = ts_node_type(ch);+ if (strcmp(k, "package_clause") == 0) {+ pkg = ch;+ found = true;+ break;+ }+ if (is_comment_node(k)) {+ doc_span_t sp = doc_span_of(ch, ctx->source, DOC_SPAN_COMMENT);+ doc_push_span(doc_scratch(ctx), &t, &sp);+ } else {+ t.count = 0;+ }+ } while (ts_tree_cursor_goto_next_sibling(&cur));+ }+ ts_tree_cursor_delete(&cur);+ int last = t.count - SKIP_ONE;+ if (!found || last < 0 || ts_node_start_point(pkg).row > t.items[last].erow + SKIP_ONE) {+ return NULL;+ }+ int first = last;+ while (first > 0 && t.items[first].srow <= t.items[first - SKIP_ONE].erow + SKIP_ONE) {+ first--;+ }+ return doc_run_text(ctx, &t, first, last, true);+}++/* Rust inner docs (//! and the bang block) before the first item. */+static const char *doc_rust_inner(CBMExtractCtx *ctx) {+ doc_trivia_t t;+ memset(&t, 0, sizeof(t));+ TSTreeCursor cur = ts_tree_cursor_new(ctx->root);+ if (ts_tree_cursor_goto_first_child(&cur)) {+ do {+ TSNode ch = ts_tree_cursor_current_node(&cur);+ const char *k = ts_node_type(ch);+ if (is_comment_node(k)) {+ doc_span_t sp = doc_span_of(ch, ctx->source, DOC_SPAN_COMMENT);+ if (sp.style == DOC_CS_LINE_BANG || sp.style == DOC_CS_BLOCK_BANG) {+ doc_push_span(doc_scratch(ctx), &t, &sp);+ }+ continue;+ }+ if (strcmp(k, "inner_attribute_item") != 0) {+ break; /* the first item: the inner docs are over */+ }+ } while (ts_tree_cursor_goto_next_sibling(&cur));+ }+ ts_tree_cursor_delete(&cur);+ return t.count > 0 ? doc_run_text(ctx, &t, 0, t.count - SKIP_ONE, false) : NULL;+}++/* A file's own doc, for its File node: the Go package comment or the Rust+ * inner docs. */+static const char *extract_module_doc(CBMExtractCtx *ctx) {+ if (ctx->language == CBM_LANG_GO) {+ return doc_go_package(ctx);+ }+ return ctx->language == CBM_LANG_RUST ? doc_rust_inner(ctx) : NULL;}static int find_jvm_modifiers(TSNode node, CBMLanguage lang, TSNode *out, int max);@@ -4124,7 +5487,7 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpecdef.is_test = def.is_test || ctx->result->is_test_file;// Docstring- def.docstring = extract_docstring(a, node, ctx->source, ctx->language);+ def.docstring = extract_docstring(ctx, node, name);// Complexityif (spec->branching_node_types && spec->branching_node_types[0]) {@@ -4248,8 +5611,8 @@ static void utf8_trim_partial_tail(char *text) {}/* Collapse `len` bytes of raw prose into a single-spaced value capped at- * MAX_COMMENT_LEN (the same 500-byte ceiling docstrings already use, which- * leaves room inside the 2 KB properties buffer that carries it).+ * MAX_COMMENT_LEN (500 bytes, which leaves room inside the 2 KB properties+ * buffer that carries it).** The output buffer is FIXED at that cap, so a section body of any size costs a* bounded copy rather than a copy of the whole section. Returns NULL when@@ -4953,7 +6316,7 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpecdef.is_exported = cbm_is_exported(name, ctx->language);def.base_classes = extract_base_classes(a, node, ctx->source, ctx->language);def.decorators = extract_decorators(a, node, ctx->source, ctx->language, spec);- def.docstring = extract_docstring(a, node, ctx->source, ctx->language);+ def.docstring = extract_docstring(ctx, node, name);cbm_defs_push(&ctx->result->defs, a, def);@@ -5324,7 +6687,7 @@ static void push_method_def(CBMExtractCtx *ctx, TSNode child, TSNode class_node,def.route_path = prefix ? join_route_paths(a, prefix, def.route_path) : NULL;def.route_method = prefix ? def.route_method : NULL;}- def.docstring = extract_docstring(a, child, ctx->source, ctx->language);+ def.docstring = extract_docstring(ctx, child, name);if (spec->branching_node_types && spec->branching_node_types[0]) {set_def_complexity(&def, child, spec);@@ -5565,6 +6928,7 @@ static void extract_rust_impl(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec// MinHash fingerprintcompute_fingerprint(ctx, &def, child);+ def.docstring = extract_docstring(ctx, child, name);cbm_defs_push(&ctx->result->defs, a, def);}@@ -5718,6 +7082,7 @@ static void push_var_def_qn(CBMExtractCtx *ctx, const char *name, const char *qndef.start_line = ts_node_start_point(node).row + TS_LINE_OFFSET;def.end_line = ts_node_end_point(node).row + TS_LINE_OFFSET;def.is_exported = cbm_is_exported(name, ctx->language);+ def.docstring = extract_member_docstring(ctx, node);cbm_defs_push(&ctx->result->defs, a, def);}@@ -5851,6 +7216,7 @@ static void extract_enum_members(CBMExtractCtx *ctx, TSNode node, const char *clmdef.file_path = ctx->rel_path;mdef.start_line = ts_node_start_point(member).row + TS_LINE_OFFSET;mdef.end_line = ts_node_end_point(member).row + TS_LINE_OFFSET;+ mdef.docstring = extract_member_docstring(ctx, member);cbm_defs_push(&ctx->result->defs, a, mdef);}}@@ -7447,6 +8813,7 @@ static void extract_class_fields(CBMExtractCtx *ctx, TSNode class_node, const ch* TS @Input()-style fields, JVM field annotations via the modifiers* child) — same extraction the class and method paths already run. */def.decorators = extract_decorators(a, child, ctx->source, ctx->language, spec);+ def.docstring = extract_member_docstring(ctx, child);cbm_defs_push(&ctx->result->defs, a, def);}@@ -7998,6 +9365,7 @@ static void extract_c_macro_def(CBMExtractCtx *ctx, TSNode node) {if (!ts_node_is_null(params)) {def.signature = cbm_node_text(a, params, ctx->source);}+ def.docstring = extract_member_docstring(ctx, node);cbm_defs_push(&ctx->result->defs, a, def);}@@ -8551,6 +9919,7 @@ void cbm_extract_definitions(CBMExtractCtx *ctx) {}int mod_idx = ctx->result->defs.count;cbm_defs_push(&ctx->result->defs, a, mod);+ ctx->result->module_doc = extract_module_doc(ctx);cbm_extract_definitions_without_module(ctx);diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.cindex 90c8c6d82..bd8b9b8a9 100644--- a/internal/cbm/result_compact.c+++ b/internal/cbm/result_compact.c@@ -385,6 +385,7 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) {cr_list(c, &r->macros);cr_str(c, &r->error_msg);cr_str(c, &r->error_ranges);+ cr_str(c, &r->module_doc);cr_blob(c, (const void **)&r->source, r->source ? (size_t)r->source_len + SKIP_ONE : 0);}diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.cindex 804ffba65..f3a2cd948 100644--- a/src/pipeline/pass_definitions.c+++ b/src/pipeline/pass_definitions.c@@ -22,6 +22,7 @@ enum { PD_JSON_FIELD_OVERHEAD = 6 };#include "pipeline/pipeline_internal.h"#include "graph_buffer/graph_buffer.h"#include "foundation/log.h"+#include "foundation/mem_core.h" /* cbm_alloc/cbm_free: the grown properties buffer */#include "foundation/compat.h"#include "foundation/compat_fs.h"#include "foundation/limits.h"@@ -321,16 +322,74 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)}}-/* Process one definition: create node, register, DEFINES + DEFINES_METHOD edges. */-static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *rel) {+/* A def's properties buffer: CBM_SZ_2K for every other field plus the whole+ * serialized docstring field, which has no length cap (a field that does not+ * fit is dropped whole). Returns `stack` for a def without a docstring, or+ * when the larger buffer cannot be allocated. Twin of pass_parallel.c -- keep+ * both in sync. */+static char *pd_props_buf(const CBMDefinition *def, char *stack, size_t *size) {+ if (!def->docstring || !def->docstring[0]) {+ return stack;+ }+ size_t need =+ *size + strlen("docstring") + def_json_escaped_len(def->docstring) + PD_JSON_FIELD_OVERHEAD;+ char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need);+ if (!buf) {+ return stack;+ }+ *size = need;+ return buf;+}++/* Add a file's own doc (Go package comment, Rust inner docs) to its File+ * node as "docstring". Twin of pass_parallel.c -- keep both in sync. */+static void pd_add_file_doc(const cbm_gbuf_node_t *file_node, const char *doc) {+ if (!file_node || !doc || !doc[0]) {+ return;+ }+ const char *old = file_node->properties_json ? file_node->properties_json : "{}";+ size_t olen = strlen(old);+ if (olen < PAIR_LEN || old[olen - SKIP_ONE] != '}') {+ return; /* not a JSON object -- leave it untouched */+ }+ size_t cap = olen + strlen("docstring") + def_json_escaped_len(doc) + PD_JSON_FIELD_OVERHEAD ++ PD_ESC_SPACE + SKIP_ONE;+ char *neu = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, cap);+ if (!neu) {+ return;+ }+ size_t pos = olen - SKIP_ONE; /* without the closing brace */+ memcpy(neu, old, pos);+ neu[pos] = '\0';+ append_json_string(neu, cap, &pos, "docstring", doc);+ if (olen == PAIR_LEN && pos > PAIR_LEN) { /* "{}": drop the leading comma */+ memmove(neu + SKIP_ONE, neu + PAIR_LEN, pos - SKIP_ONE);+ pos--;+ }+ neu[pos++] = '}';+ neu[pos] = '\0';+ (void)cbm_gbuf_node_set_properties_json((cbm_gbuf_node_t *)file_node, neu);+ cbm_free(CBM_MEM_CLASS_GBUF_STRING, neu);+}++/* Process one definition: create node, register, DEFINES + DEFINES_METHOD edges.+ * `file_doc`, the file's own doc (passed with its first def only), goes on the+ * File node. */+static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *rel,+ const char *file_doc) {if (!def->qualified_name || !def->name) {return;}- char props[CBM_SZ_2K];- build_def_props(props, sizeof(props), def);+ char stack[CBM_SZ_2K];+ size_t props_size = sizeof(stack);+ char *props = pd_props_buf(def, stack, &props_size);+ build_def_props(props, props_size, def);int64_t node_id = cbm_gbuf_upsert_node(ctx->gbuf, def->label ? def->label : "Function", def->name, def->qualified_name,def->file_path ? def->file_path : rel, (int)def->start_line, (int)def->end_line, props);+ if (props != stack) {+ cbm_free(CBM_MEM_CLASS_GBUF_STRING, props);+ }/* Registry membership is defined ONCE by cbm_label_is_registry_symbol* (helpers.c): callables + type-like containers (INHERITS/IMPLEMENTS/method/* field resolution), Variable/Field (READS/WRITES resolution), and Table/View@@ -341,6 +400,7 @@ static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const}char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__");const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn);+ pd_add_file_doc(file_node, file_doc);if (file_node && node_id > 0) {cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, node_id, "DEFINES", "{}");}@@ -860,8 +920,10 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t}/* Create nodes for each definition */+ const char *file_doc = result->module_doc; /* goes with the first def */for (int d = 0; d < result->defs.count; d++) {- process_def(ctx, &result->defs.items[d], rel);+ process_def(ctx, &result->defs.items[d], rel, file_doc);+ file_doc = NULL;total_defs++;}diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.cindex 9df46c1af..0c12e6c4a 100644--- a/src/pipeline/pass_parallel.c+++ b/src/pipeline/pass_parallel.c@@ -697,15 +697,39 @@ typedef struct {* to the response/logfile — this only throttles the stderr noise). */enum { PP_OVERSIZED_WARN_MAX = 32 };+/* A def's properties buffer: CBM_SZ_2K for every other field plus the whole+ * serialized docstring field, which has no length cap (a field that does not+ * fit is dropped whole). Returns `stack` for a def without a docstring, or+ * when the larger buffer cannot be allocated. Twin of pass_definitions.c --+ * keep both in sync. */+static char *pp_props_buf(const CBMDefinition *def, char *stack, size_t *size) {+ if (!def->docstring || !def->docstring[0]) {+ return stack;+ }+ size_t need =+ *size + strlen("docstring") + pp_json_escaped_len(def->docstring) + PP_JSON_FIELD_OVERHEAD;+ char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need);+ if (!buf) {+ return stack;+ }+ *size = need;+ return buf;+}+/* Insert one definition node (and its route if present) into the local gbuf. */static void insert_def_into_gbuf(extract_worker_state_t *ws, const cbm_file_info_t *fi,CBMDefinition *def) {- char props[CBM_SZ_2K];- build_def_props(props, sizeof(props), def);+ char stack[CBM_SZ_2K];+ size_t props_size = sizeof(stack);+ char *props = pp_props_buf(def, stack, &props_size);+ build_def_props(props, props_size, def);int64_t func_id =cbm_gbuf_upsert_node(ws->local_gbuf, def->label ? def->label : "Function", def->name,def->qualified_name, def->file_path ? def->file_path : fi->rel_path,(int)def->start_line, (int)def->end_line, props);+ if (props != stack) {+ cbm_free(CBM_MEM_CLASS_GBUF_STRING, props);+ }ws->nodes_created++;if (def->route_path && def->route_path[0] != '\0') {const char *rm = def->route_method ? def->route_method : "ANY";@@ -1692,6 +1716,37 @@ static int register_and_link_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *dreturn edges;}+/* Add a file's own doc (Go package comment, Rust inner docs) to its File+ * node as "docstring". Twin of pass_definitions.c -- keep both in sync. */+static void pp_add_file_doc(const cbm_gbuf_node_t *file_node, const char *doc) {+ if (!file_node || !doc || !doc[0]) {+ return;+ }+ const char *old = file_node->properties_json ? file_node->properties_json : "{}";+ size_t olen = strlen(old);+ if (olen < PAIR_LEN || old[olen - SKIP_ONE] != '}') {+ return; /* not a JSON object -- leave it untouched */+ }+ size_t cap = olen + strlen("docstring") + pp_json_escaped_len(doc) + PP_JSON_FIELD_OVERHEAD ++ PP_ESC_SPACE + SKIP_ONE;+ char *neu = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, cap);+ if (!neu) {+ return;+ }+ size_t pos = olen - SKIP_ONE; /* without the closing brace */+ memcpy(neu, old, pos);+ neu[pos] = '\0';+ append_json_string(neu, cap, &pos, "docstring", doc);+ if (olen == PAIR_LEN && pos > PAIR_LEN) { /* "{}": drop the leading comma */+ memmove(neu + SKIP_ONE, neu + PAIR_LEN, pos - SKIP_ONE);+ pos--;+ }+ neu[pos++] = '}';+ neu[pos] = '\0';+ (void)cbm_gbuf_node_set_properties_json((cbm_gbuf_node_t *)file_node, neu);+ cbm_free(CBM_MEM_CLASS_GBUF_STRING, neu);+}+/* Create IMPORTS edges for one file's imports (parallel path). */static int create_imports_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result,const char *rel, CBMHashTable *namespace_map) {@@ -1826,6 +1881,7 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_tconst cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn);int64_t file_node_id = file_node ? file_node->id : 0;free(file_qn);+ pp_add_file_doc(file_node, result->module_doc);for (int d = 0; d < result->defs.count; d++) {defines_edges +=register_and_link_def(ctx, &result->defs.items[d], file_node_id, ®_entries);diff --git a/tests/test_extraction.c b/tests/test_extraction.cindex 0b6800cd1..fe46d7246 100644--- a/tests/test_extraction.c+++ b/tests/test_extraction.c@@ -7064,12 +7064,12 @@ TEST(extract_python_method_test_dir_marks_is_test_issue1294) {PASS();}-/* #1017: docstring truncation at MAX_COMMENT_LEN (500 bytes) can split a+/* #1017: docstring truncation at MAX_COMMENT_LEN (500 bytes) could split a* multi-byte UTF-8 character, leaving an incomplete byte sequence.* Craft a Go comment whose 498th-500th bytes are a 3-byte CJK character- * (U+6210 = 成 = e6 88 90). The raw byte truncation at offset 500 lands- * one byte past the character start, splitting it. After the fix the- * truncated string must end on a complete codepoint boundary. */+ * (U+6210 = 成 = e6 88 90), where a raw byte cut at offset 500 lands one+ * byte past the character start. Docstrings are no longer cut at all, so+ * the whole comment is kept and still ends on a complete codepoint. */TEST(docstring_utf8_truncation_boundary_issue1017) {/* Build a comment: "// " (3 bytes) + 495 ASCII 'A' + "成成成" (9 bytes)* Total comment text = 3 + 495 + 9 = 507 bytes.@@ -7110,10 +7110,10 @@ TEST(docstring_utf8_truncation_boundary_issue1017) {}ASSERT_NOT_NULL(doc);- /* Verify every byte in the truncated docstring is valid UTF-8:- * no trailing incomplete multi-byte sequence. */+ /* The whole comment, and every byte of it valid UTF-8: no trailing+ * incomplete multi-byte sequence. */+ ASSERT_STR_EQ(doc, comment);size_t len = strlen(doc);- ASSERT_TRUE(len <= 500);const unsigned char *u = (const unsigned char *)doc;size_t i = 0;while (i < len) {@@ -7139,6 +7139,298 @@ TEST(docstring_utf8_truncation_boundary_issue1017) {PASS();}+/* ═══════════════════════════════════════════════════════════════════+ * Doc comments: the whole comment run above a definition, above its+ * attributes / export / declaration wrapper, never a detached or+ * banner-only run, and never cut at 500 bytes.+ * ═══════════════════════════════════════════════════════════════════ */++/* The def with this label and name, or NULL. */+static const CBMDefinition *doc_def(CBMFileResult *r, const char *label, const char *name) {+ for (int i = 0; i < r->defs.count; i++) {+ const CBMDefinition *d = &r->defs.items[i];+ if (d->label && d->name && strcmp(d->label, label) == 0 && strcmp(d->name, name) == 0) {+ return d;+ }+ }+ return NULL;+}++TEST(doc_go_comment_group_without_directives) {+ CBMFileResult *r = extract("package knob\n\n"+ "// Turn rotates the knob by one step.\n"+ "// It is safe for concurrent use.\n"+ "//go:noinline\n"+ "func Turn() {}\n",+ CBM_LANG_GO, "t", "knob.go");+ ASSERT_NOT_NULL(r);+ const CBMDefinition *d = doc_def(r, "Function", "Turn");+ ASSERT_NOT_NULL(d);+ /* go/ast CommentGroup.Text: the whole group, the //go: directive dropped. */+ ASSERT_STR_EQ(d->docstring, "// Turn rotates the knob by one step.\n"+ "// It is safe for concurrent use.");+ cbm_free_result(r);+ PASS();+}++TEST(doc_go_struct_field) {+ CBMFileResult *r = extract("package knob\n\n"+ "// Knob is a rotary control.\n"+ "type Knob struct {\n"+ "\t// Steps counts the turns so far.\n"+ "\tSteps int\n"+ "}\n",+ CBM_LANG_GO, "t", "knob.go");+ ASSERT_NOT_NULL(r);+ const CBMDefinition *knob = doc_def(r, "Struct", "Knob");+ ASSERT_NOT_NULL(knob);+ ASSERT_STR_EQ(knob->docstring, "// Knob is a rotary control.");+ const CBMDefinition *steps = doc_def(r, "Field", "Steps");+ ASSERT_NOT_NULL(steps);+ ASSERT_STR_EQ(steps->docstring, "// Steps counts the turns so far.");+ cbm_free_result(r);+ PASS();+}++TEST(doc_go_detached_comment_is_not_doc) {+ /* go/doc: a comment group separated by a blank line documents nothing. */+ CBMFileResult *r = extract("package knob\n\n"+ "// Section: helpers.\n"+ "\n"+ "func Reset() {}\n",+ CBM_LANG_GO, "t", "knob.go");+ ASSERT_NOT_NULL(r);+ const CBMDefinition *d = doc_def(r, "Function", "Reset");+ ASSERT_NOT_NULL(d);+ ASSERT_NULL(d->docstring);+ cbm_free_result(r);+ PASS();+}++TEST(doc_python_comment_run_and_detached) {+ CBMFileResult *r = extract("# Turns the knob.\n"+ "# One step per call.\n"+ "def turn():\n"+ " pass\n"+ "\n"+ "# Section: helpers.\n"+ "\n"+ "def reset():\n"+ " pass\n",+ CBM_LANG_PYTHON, "t", "knob.py");+ ASSERT_NOT_NULL(r);+ const CBMDefinition *turn = doc_def(r, "Function", "turn");+ ASSERT_NOT_NULL(turn);+ ASSERT_STR_EQ(turn->docstring, "# Turns the knob.\n# One step per call.");+ const CBMDefinition *reset = doc_def(r, "Function", "reset");+ ASSERT_NOT_NULL(reset);+ ASSERT_NULL(reset->docstring);+ cbm_free_result(r);+ PASS();+}++TEST(doc_rust_attribute_impl_method_and_inner_doc) {+ CBMFileResult *r = extract("//! The knob crate.\n"+ "\n"+ "/// Adds two steps.\n"+ "/// Wraps on overflow.\n"+ "#[inline]\n"+ "pub fn add(a: u8, b: u8) -> u8 { a.wrapping_add(b) }\n"+ "\n"+ "pub struct Knob;\n"+ "\n"+ "impl Knob {\n"+ " /// Turns the knob once.\n"+ " pub fn turn(&self) {}\n"+ "}\n",+ CBM_LANG_RUST, "t", "lib.rs");+ ASSERT_NOT_NULL(r);+ /* The doc sits above the attribute; a run is joined line by line. */+ const CBMDefinition *add = doc_def(r, "Function", "add");+ ASSERT_NOT_NULL(add);+ ASSERT_STR_EQ(add->docstring, "/// Adds two steps.\n/// Wraps on overflow.");+ /* impl methods get their doc too; a single comment keeps its bytes. */+ const CBMDefinition *turn = doc_def(r, "Method", "turn");+ ASSERT_NOT_NULL(turn);+ ASSERT_STR_EQ(turn->docstring, "/// Turns the knob once.\n");+ /* The //! inner doc documents the crate (the File node), not the struct. */+ const CBMDefinition *knob = doc_def(r, "Struct", "Knob");+ ASSERT_NOT_NULL(knob);+ ASSERT_NULL(knob->docstring);+ cbm_free_result(r);+ PASS();+}++TEST(doc_ts_export_and_arrow_wrapper) {+ CBMFileResult *r = extract("/** Exported helper. */\n"+ "export function helper(): number {\n"+ " return 1;\n"+ "}\n"+ "\n"+ "/** Arrow helper. */\n"+ "export const arrow = (): number => 2;\n",+ CBM_LANG_TYPESCRIPT, "t", "knob.ts");+ ASSERT_NOT_NULL(r);+ const CBMDefinition *helper = doc_def(r, "Function", "helper");+ ASSERT_NOT_NULL(helper);+ ASSERT_STR_EQ(helper->docstring, "/** Exported helper. */");+ const CBMDefinition *arrow = doc_def(r, "Function", "arrow");+ ASSERT_NOT_NULL(arrow);+ ASSERT_STR_EQ(arrow->docstring, "/** Arrow helper. */");+ cbm_free_result(r);+ PASS();+}++TEST(doc_csharp_xml_doc_run_and_top_level_function) {+ CBMFileResult *r = extract("/// Adds one.\n"+ "static int AddOne(int x) => x + 1;\n"+ "\n"+ "namespace Knobs {\n"+ " public class Knob {\n"+ " /// <summary>\n"+ " /// Turns the knob.\n"+ " /// </summary>\n"+ " public void Turn() {}\n"+ " }\n"+ "}\n",+ CBM_LANG_CSHARP, "t", "Knob.cs");+ ASSERT_NOT_NULL(r);+ const CBMDefinition *turn = doc_def(r, "Method", "Turn");+ ASSERT_NOT_NULL(turn);+ ASSERT_STR_EQ(turn->docstring, "/// <summary>\n/// Turns the knob.\n/// </summary>");+ const CBMDefinition *add = doc_def(r, "Function", "AddOne");+ ASSERT_NOT_NULL(add);+ ASSERT_STR_EQ(add->docstring, "/// Adds one.");+ cbm_free_result(r);+ PASS();+}++TEST(doc_java_javadoc_over_line_comment_and_field) {+ CBMFileResult *r = extract("class Knob {\n"+ " /** Turns taken so far. */\n"+ " private int steps;\n"+ "\n"+ " /** Turns the knob. */\n"+ " // NOSONAR\n"+ " void turn() {}\n"+ "}\n",+ CBM_LANG_JAVA, "t", "Knob.java");+ ASSERT_NOT_NULL(r);+ /* A plain comment between the Javadoc and the method does not replace it. */+ const CBMDefinition *turn = doc_def(r, "Method", "turn");+ ASSERT_NOT_NULL(turn);+ ASSERT_STR_EQ(turn->docstring, "/** Turns the knob. */");+ const CBMDefinition *steps = doc_def(r, "Field", "steps");+ ASSERT_NOT_NULL(steps);+ ASSERT_STR_EQ(steps->docstring, "/** Turns taken so far. */");+ cbm_free_result(r);+ PASS();+}++TEST(doc_kotlin_kdoc_folded_after_imports) {+ /* tree-sitter-kotlin folds this KDoc into the node before the class, so it+ * is no sibling of it. */+ CBMFileResult *r = extract("package knobs\n"+ "\n"+ "import knobs.core.Step\n"+ "import knobs.core.Turn\n"+ "\n"+ "/** Base class of every knob. */\n"+ "abstract class BaseKnob(step: Step) : Turn(step)\n",+ CBM_LANG_KOTLIN, "t", "BaseKnob.kt");+ ASSERT_NOT_NULL(r);+ const CBMDefinition *d = doc_def(r, "Class", "BaseKnob");+ ASSERT_NOT_NULL(d);+ ASSERT_STR_EQ(d->docstring, "/** Base class of every knob. */");+ cbm_free_result(r);+ PASS();+}++TEST(doc_c_typedef_struct) {+ CBMFileResult *r = extract("/* A file event. */\n"+ "typedef struct aeFileEvent {\n"+ " int mask;\n"+ "} aeFileEvent;\n",+ CBM_LANG_C, "t", "ae.h");+ ASSERT_NOT_NULL(r);+ const CBMDefinition *d = doc_def(r, "Class", "aeFileEvent");+ ASSERT_NOT_NULL(d);+ ASSERT_STR_EQ(d->docstring, "/* A file event. */");+ cbm_free_result(r);+ PASS();+}++TEST(doc_perl_pod_section_by_sub_name) {+ CBMFileResult *r = extract("package Knob;\n"+ "\n"+ "sub turn { return 1; }\n"+ "\n"+ "1;\n"+ "__END__\n"+ "\n"+ "=head1 METHODS\n"+ "\n"+ "=head2 turn\n"+ "\n"+ "Turns the knob once.\n"+ "\n"+ "=cut\n",+ CBM_LANG_PERL, "t", "Knob.pm");+ ASSERT_NOT_NULL(r);+ const CBMDefinition *d = doc_def(r, "Function", "turn");+ ASSERT_NOT_NULL(d);+ ASSERT_STR_EQ(d->docstring, "=head2 turn\n\nTurns the knob once.");+ cbm_free_result(r);+ PASS();+}++TEST(doc_long_comment_kept_whole) {+ /* One 12-line block comment well past the old 500-byte cut, kept byte for+ * byte. */+ char src[4096];+ char want[2048];+ int w = snprintf(want, sizeof(want), "/*");+ for (int i = 0; i < 12; i++) {+ w += snprintf(want + w, sizeof(want) - (size_t)w,+ "\n * Line %02d of a long doc that keeps going past the cut.", i);+ }+ snprintf(want + w, sizeof(want) - (size_t)w, "\n */");+ snprintf(src, sizeof(src), "package knob\n\n%s\nfunc Long() {}\n", want);+ ASSERT_GT((int)strlen(want), 500);++ CBMFileResult *r = extract(src, CBM_LANG_GO, "t", "knob.go");+ ASSERT_NOT_NULL(r);+ const CBMDefinition *d = doc_def(r, "Function", "Long");+ ASSERT_NOT_NULL(d);+ ASSERT_STR_EQ(d->docstring, want);+ cbm_free_result(r);+ PASS();+}++TEST(doc_divider_only_run_is_not_doc) {+ CBMFileResult *r = extract("/*****************************/\n"+ "void knob(void) {}\n"+ "\n"+ "// \xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\n"+ "void dial(void) {}\n"+ "\n"+ "/* Resets the dial. */\n"+ "void reset(void) {}\n",+ CBM_LANG_C, "t", "knob.c");+ ASSERT_NOT_NULL(r);+ const CBMDefinition *knob = doc_def(r, "Function", "knob");+ ASSERT_NOT_NULL(knob);+ ASSERT_NULL(knob->docstring);+ const CBMDefinition *dial = doc_def(r, "Function", "dial"); /* a box-drawing banner */+ ASSERT_NOT_NULL(dial);+ ASSERT_NULL(dial->docstring);+ const CBMDefinition *reset = doc_def(r, "Function", "reset");+ ASSERT_NOT_NULL(reset);+ ASSERT_STR_EQ(reset->docstring, "/* Resets the dial. */");+ cbm_free_result(r);+ PASS();+}+/* Reproduce-first (ms-typescript reallyLargeFile.ts, 2026-07-07): a file* whose root node has hundreds of thousands of FLAT SIBLINGS (580k ////* comment lines in the 3.5 MB fourslash fixture) hung extraction for over@@ -9291,6 +9583,19 @@ SUITE(extraction) {RUN_TEST(extract_c_test_dir_marks_is_test_issue1294);RUN_TEST(extract_python_method_test_dir_marks_is_test_issue1294);RUN_TEST(docstring_utf8_truncation_boundary_issue1017);+ RUN_TEST(doc_go_comment_group_without_directives);+ RUN_TEST(doc_go_struct_field);+ RUN_TEST(doc_go_detached_comment_is_not_doc);+ RUN_TEST(doc_python_comment_run_and_detached);+ RUN_TEST(doc_rust_attribute_impl_method_and_inner_doc);+ RUN_TEST(doc_ts_export_and_arrow_wrapper);+ RUN_TEST(doc_csharp_xml_doc_run_and_top_level_function);+ RUN_TEST(doc_java_javadoc_over_line_comment_and_field);+ RUN_TEST(doc_kotlin_kdoc_folded_after_imports);+ RUN_TEST(doc_c_typedef_struct);+ RUN_TEST(doc_perl_pod_section_by_sub_name);+ RUN_TEST(doc_long_comment_kept_whole);+ RUN_TEST(doc_divider_only_run_is_not_doc);RUN_TEST(extract_ts_decorators_survive_interleaved_comment);/* #518/#519 — prose carried into docstring so nodes_fts can index it */diff --git a/tests/test_pipeline.c b/tests/test_pipeline.cindex 56be6b88e..9a4c461c6 100644--- a/tests/test_pipeline.c+++ b/tests/test_pipeline.c@@ -9578,6 +9578,83 @@ TEST(pipeline_docstring_go_class) {PASS();}+/* Index one file; true when the node with this label and name stores `want` in+ * its properties JSON. */+static bool doc_props_contain(const char *file, const char *content, const char *label,+ const char *name, const char *want) {+ const char *files[] = {file};+ const char *contents[] = {content};+ if (setup_lang_repo(files, contents, 1) != 0) {+ return false;+ }+ char db[512];+ snprintf(db, sizeof(db), "%s/test.db", g_lang_tmpdir);+ cbm_pipeline_t *p = cbm_pipeline_new(g_lang_tmpdir, db, CBM_MODE_FULL);+ bool found = false;+ if (p && cbm_pipeline_run(p) == 0) {+ cbm_store_t *s = cbm_store_open_path(db);+ cbm_node_t *nodes = NULL;+ int nc = 0;+ if (s && cbm_store_find_nodes_by_label(s, cbm_pipeline_project_name(p), label, &nodes,+ &nc) == 0) {+ for (int i = 0; i < nc; i++) {+ if (nodes[i].name && strcmp(nodes[i].name, name) == 0 && nodes[i].properties_json &&+ strstr(nodes[i].properties_json, want)) {+ found = true;+ }+ }+ cbm_store_free_nodes(nodes, nc);+ }+ if (s) {+ cbm_store_close(s);+ }+ }+ if (p) {+ cbm_pipeline_free(p);+ }+ teardown_lang_repo();+ return found;+}++TEST(pipeline_doc_go_package_comment_on_file_node) {+ /* go/doc: the comment group touching `package` documents the package; it is+ * stored on the File node. */+ ASSERT_TRUE(doc_props_contain("knob.go",+ "// Package knob turns knobs.\n"+ "package knob\n\n"+ "func Turn() {}\n",+ "File", "knob.go",+ "\"docstring\":\"// Package knob turns knobs.\""));+ PASS();+}++TEST(pipeline_doc_rust_inner_doc_on_file_node) {+ ASSERT_TRUE(doc_props_contain("lib.rs",+ "//! Knob crate.\n"+ "//! Turns knobs.\n"+ "\n"+ "pub fn turn() {}\n",+ "File", "lib.rs",+ "\"docstring\":\"//! Knob crate.\\n//! Turns knobs.\""));+ PASS();+}++TEST(pipeline_doc_long_docstring_stored_whole) {+ /* A 30-line doc (longer than the old fixed 2 KB properties buffer on its+ * own) is stored whole, and the fields after it are not dropped. */+ char src[4096];+ int s = snprintf(src, sizeof(src), "package knob\n\n");+ for (int i = 0; i < 30; i++) {+ s += snprintf(src + s, sizeof(src) - (size_t)s,+ "// Line %02d of a long doc comment that keeps on going.\n", i);+ }+ snprintf(src + s, sizeof(src) - (size_t)s, "func Long(steps int) int { return steps }\n");+ ASSERT_TRUE(doc_props_contain("knob.go", src, "Function", "Long",+ "that keeps on going.\\n// Line 29 of a long doc"));+ ASSERT_TRUE(doc_props_contain("knob.go", src, "Function", "Long", "\"signature\":"));+ PASS();+}+TEST(project_name_from_path) {/* Port of TestProjectNameFromPath — more cases than integ_pipeline_project_name */struct {@@ -16405,6 +16482,9 @@ SUITE(pipeline) {RUN_TEST(pipeline_docstring_java_method);RUN_TEST(pipeline_docstring_kotlin_function);RUN_TEST(pipeline_docstring_go_class);+ RUN_TEST(pipeline_doc_go_package_comment_on_file_node);+ RUN_TEST(pipeline_doc_rust_inner_doc_on_file_node);+ RUN_TEST(pipeline_doc_long_docstring_stored_whole);/* Project name */RUN_TEST(project_name_from_path);RUN_TEST(project_name_drive_letter_case_insensitive_issue394);