Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(extract): AST walker stacks grow instead of silently capping content
Martin Vogel7707c9d79182
diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.cindex 49543e59a..1e63d26fc 100644--- a/internal/cbm/extract_defs.c+++ b/internal/cbm/extract_defs.c@@ -87,10 +87,18 @@ static bool try_append_ident(const char *source, uint32_t s, int len, uint32_t *}/* Walk AST body, collect unique identifier text as space-separated string.- * Returns arena-allocated string or NULL. */+ * Contract: the first BT_MAX_IDENTS unique identifiers in pre-order (source+ * order), bounded by BT_BUF bytes. The ident/byte caps ARE the sampling+ * contract; the pending stack is not — BT_STACK is only the on-stack first+ * chunk and spills to the heap, because a full fixed stack dropped a wide+ * body's FIRST children and sampled a mid-body window instead.+ * Returns arena-allocated string, or NULL when there are no identifiers or the+ * stack could not be grown (logged; no truncated sample is ever returned). */static char *extract_body_ident_tokens(CBMExtractCtx *ctx, TSNode body) {enum { BT_STACK = 512, BT_BUF = 2048, BT_MAX_IDENTS = 128, BT_SEEN = 256, BT_SEEN_MASK = 255 };- TSNode bt_stack[BT_STACK];+ TSNode bt_inline[BT_STACK];+ TSNode *bt_stack = bt_inline;+ int bt_cap = BT_STACK;int bt_top = 0;bt_stack[bt_top++] = body;char bt_buf[BT_BUF];@@ -120,11 +128,17 @@ static char *extract_body_ident_tokens(CBMExtractCtx *ctx, TSNode body) {}}} else {- for (int i = (int)nc - SKIP_ONE; i >= 0 && bt_top < BT_STACK; i--) {+ if (!cbm_walk_stack_reserve((void **)&bt_stack, &bt_cap, bt_top + (int)nc,+ sizeof(TSNode), bt_inline, "body_ident_tokens")) {+ bt_pos = 0; /* sentinel: no sample rather than a truncated one */+ break;+ }+ for (int i = (int)nc - SKIP_ONE; i >= 0; i--) {bt_stack[bt_top++] = ts_node_child(nd, (uint32_t)i);}}}+ cbm_walk_stack_release(bt_stack, bt_inline);if (bt_pos > 0) {bt_buf[bt_pos] = '\0';return cbm_arena_strdup(ctx->arena, bt_buf);diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.cindex 50c75bdf2..4c4d340f0 100644--- a/internal/cbm/helpers.c+++ b/internal/cbm/helpers.c@@ -4,8 +4,10 @@#include "lang_specs.h"#include "tree_sitter/api.h" // TSNode, ts_node_*#include "foundation/constants.h"-#include "foundation/compat.h" // CBM_TLS-#include <stdlib.h> // calloc/free for the symbol-set cache+#include "foundation/compat.h" // CBM_TLS+#include "foundation/log.h" // cbm_log_error -- walker stack allocation failure+#include "foundation/mem_core.h" // cbm_alloc/cbm_realloc/cbm_free -- walker stacks+#include <stdlib.h> // calloc/free for the symbol-set cacheenum {MIN_ROUTE_LEN = 3,@@ -25,7 +27,8 @@ enum {/* Prefix length helper for strncmp with string literals. */#define SLEN(s) (sizeof(s) - SKIP_ONE)-#include <stdint.h> // uint32_t+#include <stdint.h> // uint32_t, SIZE_MAX+#include <limits.h> // INT_MAX#include <string.h>#include <ctype.h>#include <stdio.h>@@ -630,10 +633,52 @@ bool cbm_has_ancestor_kind(TSNode node, const char *kind, int max_depth) {return false;}-// Recursive branching count-#define BRANCHING_STACK_CAP 4096+bool cbm_walk_stack_reserve(void **items, int *cap, int need, size_t elem_size,+ const void *inline_buf, const char *walker) {+ if (need <= *cap) {+ return true;+ }+ size_t new_cap = (size_t)*cap;+ while (new_cap < (size_t)need) {+ new_cap *= 2;+ }+ void *grown = NULL;+ if (new_cap <= (size_t)INT_MAX && new_cap <= SIZE_MAX / elem_size) {+ if (*items == inline_buf) {+ grown = cbm_alloc(CBM_MEM_CLASS_EXTRACT, new_cap * elem_size);+ if (grown) {+ memcpy(grown, inline_buf, (size_t)*cap * elem_size);+ }+ } else {+ grown = cbm_realloc(CBM_MEM_CLASS_EXTRACT, *items, new_cap * elem_size);+ }+ }+ if (!grown) {+ char pending[24];+ snprintf(pending, sizeof(pending), "%d", need);+ cbm_log_error("extract.walk_stack_alloc_failed", "walker", walker, "pending", pending);+ return false;+ }+ *items = grown;+ *cap = (int)new_cap;+ return true;+}++void cbm_walk_stack_release(void *items, const void *inline_buf) {+ if (items != inline_buf) {+ cbm_free(CBM_MEM_CLASS_EXTRACT, items);+ }+}++// On-stack first chunk of the walker stacks. Normal functions never leave it;+// only a pending set this large (a very wide or deep body) spills to the heap.+enum { WALK_STACK_INLINE = 512 };++// Iterative branching count (pre-order; no cap on the pending set).static int count_branching_iter(TSNode root, const char **types) {- TSNode stack[BRANCHING_STACK_CAP];+ TSNode inline_stack[WALK_STACK_INLINE];+ TSNode *stack = inline_stack;+ int cap = WALK_STACK_INLINE;int top = 0;int count = 0;stack[top++] = root;@@ -647,10 +692,16 @@ static int count_branching_iter(TSNode root, const char **types) {}}uint32_t n = ts_node_child_count(node);- for (int i = (int)n - SKIP_ONE; i >= 0 && top < BRANCHING_STACK_CAP; i--) {+ if (!cbm_walk_stack_reserve((void **)&stack, &cap, top + (int)n, sizeof(TSNode),+ inline_stack, "count_branching")) {+ count = CBM_WALK_METRIC_UNAVAILABLE;+ break;+ }+ for (int i = (int)n - SKIP_ONE; i >= 0; i--) {stack[top++] = ts_node_child(node, (uint32_t)i);}}+ cbm_walk_stack_release(stack, inline_stack);return count;}@@ -723,7 +774,9 @@ void cbm_compute_complexity(TSNode node, const char **branching_types, cbm_complint ldepth;int adepth;};- struct cx_frame stack[BRANCHING_STACK_CAP];+ struct cx_frame inline_stack[WALK_STACK_INLINE];+ struct cx_frame *stack = inline_stack;+ int cap = WALK_STACK_INLINE;int top = 0;stack[top].node = node;stack[top].bdepth = 0;@@ -770,7 +823,16 @@ void cbm_compute_complexity(TSNode node, const char **branching_types, cbm_complchild_l = d;}uint32_t n = ts_node_child_count(f.node);- for (int i = (int)n - SKIP_ONE; i >= 0 && top < BRANCHING_STACK_CAP; i--) {+ if (!cbm_walk_stack_reserve((void **)&stack, &cap, top + (int)n, sizeof(struct cx_frame),+ inline_stack, "compute_complexity")) {+ out->cyclomatic = CBM_WALK_METRIC_UNAVAILABLE;+ out->cognitive = CBM_WALK_METRIC_UNAVAILABLE;+ out->loop_count = CBM_WALK_METRIC_UNAVAILABLE;+ out->loop_depth = CBM_WALK_METRIC_UNAVAILABLE;+ out->max_access_depth = CBM_WALK_METRIC_UNAVAILABLE;+ break;+ }+ for (int i = (int)n - SKIP_ONE; i >= 0; i--) {stack[top].node = ts_node_child(f.node, (uint32_t)i);stack[top].bdepth = child_b;stack[top].ldepth = child_l;@@ -778,6 +840,7 @@ void cbm_compute_complexity(TSNode node, const char **branching_types, cbm_compltop++;}}+ cbm_walk_stack_release(stack, inline_stack);}// --- Enclosing function detection ---diff --git a/internal/cbm/helpers.h b/internal/cbm/helpers.hindex 905de04a8..00e3d30dc 100644--- a/internal/cbm/helpers.h+++ b/internal/cbm/helpers.h@@ -174,7 +174,29 @@ void cbm_kind_in_set_free_cache(void);// Check if node has an ancestor of the given kind, within max_depth levels.bool cbm_has_ancestor_kind(TSNode node, const char *kind, int max_depth);+// Growable pending-node stack for the iterative AST walkers.+//+// No work cap may decide graph content, so walker stacks never truncate: a+// walker starts on a small caller-owned on-stack chunk (`inline_buf`) and spills+// to the heap (through the memory core, class EXTRACT) only for trees whose+// pending set outgrows it. Traversal order is unaffected by where the frames+// live. Ensures room for `need` frames in total; on success `*items`/`*cap`+// describe the (possibly moved) stack and true is returned. On allocation+// failure the stack is left intact, the failure is logged at error level under+// "extract.walk_stack_alloc_failed" (walker=<walker>), and false is returned —+// the caller then reports its documented failure sentinel instead of a+// silently truncated result.+bool cbm_walk_stack_reserve(void **items, int *cap, int need, size_t elem_size,+ const void *inline_buf, const char *walker);+// Release a stack grown by cbm_walk_stack_reserve (no-op for the inline chunk).+void cbm_walk_stack_release(void *items, const void *inline_buf);++// Sentinel reported by the walkers below when their pending stack could not be+// grown (allocation failure): the metric is unavailable, never an undercount.+#define CBM_WALK_METRIC_UNAVAILABLE (-1)+// Count nodes of given kinds in subtree (for complexity metric).+// Returns CBM_WALK_METRIC_UNAVAILABLE on allocation failure.int cbm_count_branching(TSNode node, const char **branching_types);// Per-function structural complexity, computed in a single AST walk.@@ -188,6 +210,7 @@ typedef struct {// Compute the metrics above in one traversal of `node`'s subtree.// `branching_types` is the language's branching node-type set.+// On allocation failure every field is CBM_WALK_METRIC_UNAVAILABLE.void cbm_compute_complexity(TSNode node, const char **branching_types, cbm_complexity_t *out);// Is `kind` a loop construct node type? Language-agnostic curated set (for/while/diff --git a/tests/test_extraction.c b/tests/test_extraction.cindex 6370c090e..6a1dcbdcd 100644--- a/tests/test_extraction.c+++ b/tests/test_extraction.c@@ -16,6 +16,8 @@#include "result_spill.h"#include "pipeline/pass_lsp_cross.h"#include "iris_export_xml.h"+#include "helpers.h" /* cbm_count_branching (walker stack cap) */+#include "lang_specs.h" /* cbm_lang_spec, cbm_ts_language *//* ── Helpers ───────────────────────────────────────────────────── */@@ -5512,6 +5514,101 @@ TEST(walk_defs_wide_file_extracts_every_definition) {PASS();}+/* ── Walker stacks must never cap graph content ──────────────────────+ *+ * The complexity walkers and the body-token sampler used fixed-size pending+ * stacks (4096 / 512 nodes) and silently stopped pushing children once full.+ * Children are pushed last-to-first, so a wide body lost its FIRST statements:+ * a function with 5000 top-level branches under-reported its cyclomatic count,+ * and the token sampler started mid-body. These build bodies wider than either+ * old cap and assert the exact metrics. */+enum { WIDE_STMTS = 5000, WIDE_CALLS = 600, BODY_TOKEN_PREFIX = 100 };++/* C function `wide` whose body holds `n` top-level statements alternating+ * `if (x) g();` and `while (x) g();`. Caller frees. */+static char *build_wide_branch_fn(int n) {+ size_t cap = (size_t)n * 24 + 64;+ char *src = malloc(cap);+ if (!src)+ return NULL;+ size_t pos = (size_t)snprintf(src, cap, "void g(void);\nvoid wide(int x) {\n");+ for (int i = 0; i < n; i++) {+ pos += (size_t)snprintf(src + pos, cap - pos, "%s",+ (i % 2 == 0) ? "if (x) g();\n" : "while (x) g();\n");+ }+ snprintf(src + pos, cap - pos, "}\n");+ return src;+}++TEST(complexity_wide_body_exceeds_old_stack_cap) {+ char *src = build_wide_branch_fn(WIDE_STMTS);+ ASSERT_NOT_NULL(src);+ CBMFileResult *r = extract(src, CBM_LANG_C, "t", "wide.c");+ ASSERT_NOT_NULL(r);+ ASSERT_FALSE(r->has_error);+ const CBMDefinition *d = find_def(r, "wide");+ ASSERT_NOT_NULL(d);+ /* every if/while is one branching node at nesting depth 0 */+ ASSERT_EQ(d->complexity, WIDE_STMTS);+ ASSERT_EQ(d->cognitive, WIDE_STMTS);+ ASSERT_EQ(d->loop_count, WIDE_STMTS / 2);+ ASSERT_EQ(d->loop_depth, 1);+ cbm_free_result(r);+ free(src);+ PASS();+}++TEST(count_branching_wide_body_exceeds_old_stack_cap) {+ char *src = build_wide_branch_fn(WIDE_STMTS);+ ASSERT_NOT_NULL(src);+ TSParser *parser = ts_parser_new();+ ASSERT_NOT_NULL(parser);+ ASSERT_TRUE(ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_C)));+ TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src));+ ASSERT_NOT_NULL(tree);+ int n = cbm_count_branching(ts_tree_root_node(tree),+ cbm_lang_spec(CBM_LANG_C)->branching_node_types);+ ASSERT_EQ(n, WIDE_STMTS);+ ts_tree_delete(tree);+ ts_parser_delete(parser);+ free(src);+ PASS();+}++/* The body-token sampler's contract is "the first BODY_TOKEN_CAP unique+ * identifiers in source order". A body with 600 distinct calls must sample+ * from f0 on, not a window starting mid-body (the capped stack began at f91).+ * Only the source-order prefix f0..f99 is asserted: the sampler's dedupe key+ * (hash | 1) also merges identifiers whose hashes differ only in bit 0, which+ * drops a few later names (f122, f124, ...) -- a separate dedupe property this+ * test deliberately does not pin. */+TEST(body_tokens_wide_body_samples_from_the_start) {+ size_t cap = (size_t)WIDE_CALLS * 16 + 64;+ char *src = malloc(cap);+ ASSERT_NOT_NULL(src);+ size_t pos = (size_t)snprintf(src, cap, "void wide_calls(void) {\n");+ for (int i = 0; i < WIDE_CALLS; i++) {+ pos += (size_t)snprintf(src + pos, cap - pos, "f%d();\n", i);+ }+ snprintf(src + pos, cap - pos, "}\n");+ char expect[BODY_TOKEN_PREFIX * 8];+ size_t ep = 0;+ for (int i = 0; i < BODY_TOKEN_PREFIX; i++) {+ ep += (size_t)snprintf(expect + ep, sizeof(expect) - ep, "f%d ", i);+ }+ CBMFileResult *r = extract(src, CBM_LANG_C, "t", "wide_calls.c");+ ASSERT_NOT_NULL(r);+ ASSERT_FALSE(r->has_error);+ const CBMDefinition *d = find_def(r, "wide_calls");+ ASSERT_NOT_NULL(d);+ ASSERT_NOT_NULL(d->body_tokens);+ ASSERT_GTE(strlen(d->body_tokens), ep);+ ASSERT_MEM_EQ(d->body_tokens, expect, ep);+ cbm_free_result(r);+ free(src);+ PASS();+}+/* ═══════════════════════════════════════════════════════════════════* Perl call-graph noise (#459 follow-up)* ═══════════════════════════════════════════════════════════════════ */@@ -8771,6 +8868,9 @@ SUITE(extraction) {RUN_TEST(complexity_delegation_receivers_not_recursive_issue876);RUN_TEST(complexity_access_depth_and_params);RUN_TEST(walk_defs_wide_file_extracts_every_definition);+ RUN_TEST(complexity_wide_body_exceeds_old_stack_cap);+ RUN_TEST(count_branching_wide_body_exceeds_old_stack_cap);+ RUN_TEST(body_tokens_wide_body_samples_from_the_start);RUN_TEST(extract_c_ifdef_split_brace_fn_recovered_issue961);RUN_TEST(extract_cpp_preproc_signature_gap_issue946);RUN_TEST(extract_cpp_preproc_macro_generated_callable_skipped_issue949);