Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(extract): AST walker stacks grow instead of silently capping content

Martin Vogel7707c9d79182
diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c
index 49543e59a..1e63d26fc 100644
--- a/internal/cbm/extract_defs.c
+++ b/internal/cbm/extract_defs.c
@@ -87,10 +87,18 @@ static bool try_append_ident(const char *source, uint32_t s, int len, uint32_t *
}
/* Walk AST body, collect unique identifier text as space-separated string.
- * Returns arena-allocated string or NULL. */
+ * Contract: the first BT_MAX_IDENTS unique identifiers in pre-order (source
+ * order), bounded by BT_BUF bytes. The ident/byte caps ARE the sampling
+ * contract; the pending stack is not — BT_STACK is only the on-stack first
+ * chunk and spills to the heap, because a full fixed stack dropped a wide
+ * body's FIRST children and sampled a mid-body window instead.
+ * Returns arena-allocated string, or NULL when there are no identifiers or the
+ * stack could not be grown (logged; no truncated sample is ever returned). */
static char *extract_body_ident_tokens(CBMExtractCtx *ctx, TSNode body) {
enum { BT_STACK = 512, BT_BUF = 2048, BT_MAX_IDENTS = 128, BT_SEEN = 256, BT_SEEN_MASK = 255 };
- TSNode bt_stack[BT_STACK];
+ TSNode bt_inline[BT_STACK];
+ TSNode *bt_stack = bt_inline;
+ int bt_cap = BT_STACK;
int bt_top = 0;
bt_stack[bt_top++] = body;
char bt_buf[BT_BUF];
@@ -120,11 +128,17 @@ static char *extract_body_ident_tokens(CBMExtractCtx *ctx, TSNode body) {
}
}
} else {
- for (int i = (int)nc - SKIP_ONE; i >= 0 && bt_top < BT_STACK; i--) {
+ if (!cbm_walk_stack_reserve((void **)&bt_stack, &bt_cap, bt_top + (int)nc,
+ sizeof(TSNode), bt_inline, "body_ident_tokens")) {
+ bt_pos = 0; /* sentinel: no sample rather than a truncated one */
+ break;
+ }
+ for (int i = (int)nc - SKIP_ONE; i >= 0; i--) {
bt_stack[bt_top++] = ts_node_child(nd, (uint32_t)i);
}
}
}
+ cbm_walk_stack_release(bt_stack, bt_inline);
if (bt_pos > 0) {
bt_buf[bt_pos] = '\0';
return cbm_arena_strdup(ctx->arena, bt_buf);
diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c
index 50c75bdf2..4c4d340f0 100644
--- a/internal/cbm/helpers.c
+++ b/internal/cbm/helpers.c
@@ -4,8 +4,10 @@
#include "lang_specs.h"
#include "tree_sitter/api.h" // TSNode, ts_node_*
#include "foundation/constants.h"
-#include "foundation/compat.h" // CBM_TLS
-#include <stdlib.h> // calloc/free for the symbol-set cache
+#include "foundation/compat.h" // CBM_TLS
+#include "foundation/log.h" // cbm_log_error -- walker stack allocation failure
+#include "foundation/mem_core.h" // cbm_alloc/cbm_realloc/cbm_free -- walker stacks
+#include <stdlib.h> // calloc/free for the symbol-set cache
enum {
MIN_ROUTE_LEN = 3,
@@ -25,7 +27,8 @@ enum {
/* Prefix length helper for strncmp with string literals. */
#define SLEN(s) (sizeof(s) - SKIP_ONE)
-#include <stdint.h> // uint32_t
+#include <stdint.h> // uint32_t, SIZE_MAX
+#include <limits.h> // INT_MAX
#include <string.h>
#include <ctype.h>
#include <stdio.h>
@@ -630,10 +633,52 @@ bool cbm_has_ancestor_kind(TSNode node, const char *kind, int max_depth) {
return false;
}
-// Recursive branching count
-#define BRANCHING_STACK_CAP 4096
+bool cbm_walk_stack_reserve(void **items, int *cap, int need, size_t elem_size,
+ const void *inline_buf, const char *walker) {
+ if (need <= *cap) {
+ return true;
+ }
+ size_t new_cap = (size_t)*cap;
+ while (new_cap < (size_t)need) {
+ new_cap *= 2;
+ }
+ void *grown = NULL;
+ if (new_cap <= (size_t)INT_MAX && new_cap <= SIZE_MAX / elem_size) {
+ if (*items == inline_buf) {
+ grown = cbm_alloc(CBM_MEM_CLASS_EXTRACT, new_cap * elem_size);
+ if (grown) {
+ memcpy(grown, inline_buf, (size_t)*cap * elem_size);
+ }
+ } else {
+ grown = cbm_realloc(CBM_MEM_CLASS_EXTRACT, *items, new_cap * elem_size);
+ }
+ }
+ if (!grown) {
+ char pending[24];
+ snprintf(pending, sizeof(pending), "%d", need);
+ cbm_log_error("extract.walk_stack_alloc_failed", "walker", walker, "pending", pending);
+ return false;
+ }
+ *items = grown;
+ *cap = (int)new_cap;
+ return true;
+}
+
+void cbm_walk_stack_release(void *items, const void *inline_buf) {
+ if (items != inline_buf) {
+ cbm_free(CBM_MEM_CLASS_EXTRACT, items);
+ }
+}
+
+// On-stack first chunk of the walker stacks. Normal functions never leave it;
+// only a pending set this large (a very wide or deep body) spills to the heap.
+enum { WALK_STACK_INLINE = 512 };
+
+// Iterative branching count (pre-order; no cap on the pending set).
static int count_branching_iter(TSNode root, const char **types) {
- TSNode stack[BRANCHING_STACK_CAP];
+ TSNode inline_stack[WALK_STACK_INLINE];
+ TSNode *stack = inline_stack;
+ int cap = WALK_STACK_INLINE;
int top = 0;
int count = 0;
stack[top++] = root;
@@ -647,10 +692,16 @@ static int count_branching_iter(TSNode root, const char **types) {
}
}
uint32_t n = ts_node_child_count(node);
- for (int i = (int)n - SKIP_ONE; i >= 0 && top < BRANCHING_STACK_CAP; i--) {
+ if (!cbm_walk_stack_reserve((void **)&stack, &cap, top + (int)n, sizeof(TSNode),
+ inline_stack, "count_branching")) {
+ count = CBM_WALK_METRIC_UNAVAILABLE;
+ break;
+ }
+ for (int i = (int)n - SKIP_ONE; i >= 0; i--) {
stack[top++] = ts_node_child(node, (uint32_t)i);
}
}
+ cbm_walk_stack_release(stack, inline_stack);
return count;
}
@@ -723,7 +774,9 @@ void cbm_compute_complexity(TSNode node, const char **branching_types, cbm_compl
int ldepth;
int adepth;
};
- struct cx_frame stack[BRANCHING_STACK_CAP];
+ struct cx_frame inline_stack[WALK_STACK_INLINE];
+ struct cx_frame *stack = inline_stack;
+ int cap = WALK_STACK_INLINE;
int top = 0;
stack[top].node = node;
stack[top].bdepth = 0;
@@ -770,7 +823,16 @@ void cbm_compute_complexity(TSNode node, const char **branching_types, cbm_compl
child_l = d;
}
uint32_t n = ts_node_child_count(f.node);
- for (int i = (int)n - SKIP_ONE; i >= 0 && top < BRANCHING_STACK_CAP; i--) {
+ if (!cbm_walk_stack_reserve((void **)&stack, &cap, top + (int)n, sizeof(struct cx_frame),
+ inline_stack, "compute_complexity")) {
+ out->cyclomatic = CBM_WALK_METRIC_UNAVAILABLE;
+ out->cognitive = CBM_WALK_METRIC_UNAVAILABLE;
+ out->loop_count = CBM_WALK_METRIC_UNAVAILABLE;
+ out->loop_depth = CBM_WALK_METRIC_UNAVAILABLE;
+ out->max_access_depth = CBM_WALK_METRIC_UNAVAILABLE;
+ break;
+ }
+ for (int i = (int)n - SKIP_ONE; i >= 0; i--) {
stack[top].node = ts_node_child(f.node, (uint32_t)i);
stack[top].bdepth = child_b;
stack[top].ldepth = child_l;
@@ -778,6 +840,7 @@ void cbm_compute_complexity(TSNode node, const char **branching_types, cbm_compl
top++;
}
}
+ cbm_walk_stack_release(stack, inline_stack);
}
// --- Enclosing function detection ---
diff --git a/internal/cbm/helpers.h b/internal/cbm/helpers.h
index 905de04a8..00e3d30dc 100644
--- a/internal/cbm/helpers.h
+++ b/internal/cbm/helpers.h
@@ -174,7 +174,29 @@ void cbm_kind_in_set_free_cache(void);
// Check if node has an ancestor of the given kind, within max_depth levels.
bool cbm_has_ancestor_kind(TSNode node, const char *kind, int max_depth);
+// Growable pending-node stack for the iterative AST walkers.
+//
+// No work cap may decide graph content, so walker stacks never truncate: a
+// walker starts on a small caller-owned on-stack chunk (`inline_buf`) and spills
+// to the heap (through the memory core, class EXTRACT) only for trees whose
+// pending set outgrows it. Traversal order is unaffected by where the frames
+// live. Ensures room for `need` frames in total; on success `*items`/`*cap`
+// describe the (possibly moved) stack and true is returned. On allocation
+// failure the stack is left intact, the failure is logged at error level under
+// "extract.walk_stack_alloc_failed" (walker=<walker>), and false is returned —
+// the caller then reports its documented failure sentinel instead of a
+// silently truncated result.
+bool cbm_walk_stack_reserve(void **items, int *cap, int need, size_t elem_size,
+ const void *inline_buf, const char *walker);
+// Release a stack grown by cbm_walk_stack_reserve (no-op for the inline chunk).
+void cbm_walk_stack_release(void *items, const void *inline_buf);
+
+// Sentinel reported by the walkers below when their pending stack could not be
+// grown (allocation failure): the metric is unavailable, never an undercount.
+#define CBM_WALK_METRIC_UNAVAILABLE (-1)
+
// Count nodes of given kinds in subtree (for complexity metric).
+// Returns CBM_WALK_METRIC_UNAVAILABLE on allocation failure.
int cbm_count_branching(TSNode node, const char **branching_types);
// Per-function structural complexity, computed in a single AST walk.
@@ -188,6 +210,7 @@ typedef struct {
// Compute the metrics above in one traversal of `node`'s subtree.
// `branching_types` is the language's branching node-type set.
+// On allocation failure every field is CBM_WALK_METRIC_UNAVAILABLE.
void cbm_compute_complexity(TSNode node, const char **branching_types, cbm_complexity_t *out);
// Is `kind` a loop construct node type? Language-agnostic curated set (for/while/
diff --git a/tests/test_extraction.c b/tests/test_extraction.c
index 6370c090e..6a1dcbdcd 100644
--- a/tests/test_extraction.c
+++ b/tests/test_extraction.c
@@ -16,6 +16,8 @@
#include "result_spill.h"
#include "pipeline/pass_lsp_cross.h"
#include "iris_export_xml.h"
+#include "helpers.h" /* cbm_count_branching (walker stack cap) */
+#include "lang_specs.h" /* cbm_lang_spec, cbm_ts_language */
/* ── Helpers ───────────────────────────────────────────────────── */
@@ -5512,6 +5514,101 @@ TEST(walk_defs_wide_file_extracts_every_definition) {
PASS();
}
+/* ── Walker stacks must never cap graph content ──────────────────────
+ *
+ * The complexity walkers and the body-token sampler used fixed-size pending
+ * stacks (4096 / 512 nodes) and silently stopped pushing children once full.
+ * Children are pushed last-to-first, so a wide body lost its FIRST statements:
+ * a function with 5000 top-level branches under-reported its cyclomatic count,
+ * and the token sampler started mid-body. These build bodies wider than either
+ * old cap and assert the exact metrics. */
+enum { WIDE_STMTS = 5000, WIDE_CALLS = 600, BODY_TOKEN_PREFIX = 100 };
+
+/* C function `wide` whose body holds `n` top-level statements alternating
+ * `if (x) g();` and `while (x) g();`. Caller frees. */
+static char *build_wide_branch_fn(int n) {
+ size_t cap = (size_t)n * 24 + 64;
+ char *src = malloc(cap);
+ if (!src)
+ return NULL;
+ size_t pos = (size_t)snprintf(src, cap, "void g(void);\nvoid wide(int x) {\n");
+ for (int i = 0; i < n; i++) {
+ pos += (size_t)snprintf(src + pos, cap - pos, "%s",
+ (i % 2 == 0) ? "if (x) g();\n" : "while (x) g();\n");
+ }
+ snprintf(src + pos, cap - pos, "}\n");
+ return src;
+}
+
+TEST(complexity_wide_body_exceeds_old_stack_cap) {
+ char *src = build_wide_branch_fn(WIDE_STMTS);
+ ASSERT_NOT_NULL(src);
+ CBMFileResult *r = extract(src, CBM_LANG_C, "t", "wide.c");
+ ASSERT_NOT_NULL(r);
+ ASSERT_FALSE(r->has_error);
+ const CBMDefinition *d = find_def(r, "wide");
+ ASSERT_NOT_NULL(d);
+ /* every if/while is one branching node at nesting depth 0 */
+ ASSERT_EQ(d->complexity, WIDE_STMTS);
+ ASSERT_EQ(d->cognitive, WIDE_STMTS);
+ ASSERT_EQ(d->loop_count, WIDE_STMTS / 2);
+ ASSERT_EQ(d->loop_depth, 1);
+ cbm_free_result(r);
+ free(src);
+ PASS();
+}
+
+TEST(count_branching_wide_body_exceeds_old_stack_cap) {
+ char *src = build_wide_branch_fn(WIDE_STMTS);
+ ASSERT_NOT_NULL(src);
+ TSParser *parser = ts_parser_new();
+ ASSERT_NOT_NULL(parser);
+ ASSERT_TRUE(ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_C)));
+ TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src));
+ ASSERT_NOT_NULL(tree);
+ int n = cbm_count_branching(ts_tree_root_node(tree),
+ cbm_lang_spec(CBM_LANG_C)->branching_node_types);
+ ASSERT_EQ(n, WIDE_STMTS);
+ ts_tree_delete(tree);
+ ts_parser_delete(parser);
+ free(src);
+ PASS();
+}
+
+/* The body-token sampler's contract is "the first BODY_TOKEN_CAP unique
+ * identifiers in source order". A body with 600 distinct calls must sample
+ * from f0 on, not a window starting mid-body (the capped stack began at f91).
+ * Only the source-order prefix f0..f99 is asserted: the sampler's dedupe key
+ * (hash | 1) also merges identifiers whose hashes differ only in bit 0, which
+ * drops a few later names (f122, f124, ...) -- a separate dedupe property this
+ * test deliberately does not pin. */
+TEST(body_tokens_wide_body_samples_from_the_start) {
+ size_t cap = (size_t)WIDE_CALLS * 16 + 64;
+ char *src = malloc(cap);
+ ASSERT_NOT_NULL(src);
+ size_t pos = (size_t)snprintf(src, cap, "void wide_calls(void) {\n");
+ for (int i = 0; i < WIDE_CALLS; i++) {
+ pos += (size_t)snprintf(src + pos, cap - pos, "f%d();\n", i);
+ }
+ snprintf(src + pos, cap - pos, "}\n");
+ char expect[BODY_TOKEN_PREFIX * 8];
+ size_t ep = 0;
+ for (int i = 0; i < BODY_TOKEN_PREFIX; i++) {
+ ep += (size_t)snprintf(expect + ep, sizeof(expect) - ep, "f%d ", i);
+ }
+ CBMFileResult *r = extract(src, CBM_LANG_C, "t", "wide_calls.c");
+ ASSERT_NOT_NULL(r);
+ ASSERT_FALSE(r->has_error);
+ const CBMDefinition *d = find_def(r, "wide_calls");
+ ASSERT_NOT_NULL(d);
+ ASSERT_NOT_NULL(d->body_tokens);
+ ASSERT_GTE(strlen(d->body_tokens), ep);
+ ASSERT_MEM_EQ(d->body_tokens, expect, ep);
+ cbm_free_result(r);
+ free(src);
+ PASS();
+}
+
/* ═══════════════════════════════════════════════════════════════════
* Perl call-graph noise (#459 follow-up)
* ═══════════════════════════════════════════════════════════════════ */
@@ -8771,6 +8868,9 @@ SUITE(extraction) {
RUN_TEST(complexity_delegation_receivers_not_recursive_issue876);
RUN_TEST(complexity_access_depth_and_params);
RUN_TEST(walk_defs_wide_file_extracts_every_definition);
+ RUN_TEST(complexity_wide_body_exceeds_old_stack_cap);
+ RUN_TEST(count_branching_wide_body_exceeds_old_stack_cap);
+ RUN_TEST(body_tokens_wide_body_samples_from_the_start);
RUN_TEST(extract_c_ifdef_split_brace_fn_recovered_issue961);
RUN_TEST(extract_cpp_preproc_signature_gap_issue946);
RUN_TEST(extract_cpp_preproc_macro_generated_callable_skipped_issue949);