Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(pipeline): keep both nodes when a file and a directory share a stem

Martin Vogelf96001f4ed40
diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt
index fe3a7c883..b52f1d30a 100644
--- a/scripts/memory-core-baseline.txt
+++ b/scripts/memory-core-baseline.txt
@@ -74,7 +74,7 @@ src/pipeline/pass_semantic.c 3
src/pipeline/pass_tests.c 3
src/pipeline/pass_usages.c 4
src/pipeline/path_alias.c 40
-src/pipeline/pipeline.c 134
+src/pipeline/pipeline.c 132
src/pipeline/pipeline_delta.c 19
src/pipeline/pipeline_incremental.c 140
src/pipeline/registry.c 17
diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c
index eb414a649..aecd91d99 100644
--- a/src/pipeline/pass_definitions.c
+++ b/src/pipeline/pass_definitions.c
@@ -424,9 +424,15 @@ static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const
size_t props_size = sizeof(stack);
char *props = pd_props_buf(def, stack, &props_size);
build_def_props(props, props_size, def);
- int64_t node_id = cbm_gbuf_upsert_node(
- ctx->gbuf, def->label ? def->label : "Function", def->name, def->qualified_name,
- def->file_path ? def->file_path : rel, (int)def->start_line, (int)def->end_line, props);
+ const char *def_rel = def->file_path ? def->file_path : rel;
+ char marked[CBM_SZ_4K];
+ const char *qn = def->label && strcmp(def->label, "Module") == 0
+ ? cbm_stem_marks_module_qn(ctx->stem_marks, def->qualified_name, def_rel,
+ lang, marked, sizeof(marked))
+ : def->qualified_name;
+ int64_t node_id =
+ cbm_gbuf_upsert_node(ctx->gbuf, def->label ? def->label : "Function", def->name, qn,
+ def_rel, (int)def->start_line, (int)def->end_line, props);
if (props != stack) {
cbm_free(CBM_MEM_CLASS_GBUF_STRING, props);
}
diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c
index 5d64bc079..2c0a9023a 100644
--- a/src/pipeline/pass_parallel.c
+++ b/src/pipeline/pass_parallel.c
@@ -760,16 +760,21 @@ static char *pp_props_buf(const CBMDefinition *def, char *stack, size_t *size) {
}
/* Insert one definition node (and its route if present) into the local gbuf. */
-static void insert_def_into_gbuf(extract_worker_state_t *ws, const cbm_file_info_t *fi,
- CBMDefinition *def) {
+static void insert_def_into_gbuf(extract_worker_state_t *ws, const cbm_stem_marks_t *marks,
+ const cbm_file_info_t *fi, CBMDefinition *def) {
char stack[CBM_SZ_2K];
size_t props_size = sizeof(stack);
char *props = pp_props_buf(def, stack, &props_size);
build_def_props(props, props_size, def);
+ const char *rel = def->file_path ? def->file_path : fi->rel_path;
+ char marked[CBM_SZ_4K];
+ const char *qn = def->label && strcmp(def->label, "Module") == 0
+ ? cbm_stem_marks_module_qn(marks, def->qualified_name, rel, fi->language,
+ marked, sizeof(marked))
+ : def->qualified_name;
int64_t func_id =
- cbm_gbuf_upsert_node(ws->local_gbuf, def->label ? def->label : "Function", def->name,
- def->qualified_name, def->file_path ? def->file_path : fi->rel_path,
- (int)def->start_line, (int)def->end_line, props);
+ cbm_gbuf_upsert_node(ws->local_gbuf, def->label ? def->label : "Function", def->name, qn,
+ rel, (int)def->start_line, (int)def->end_line, props);
if (props != stack) {
cbm_free(CBM_MEM_CLASS_GBUF_STRING, props);
}
@@ -1404,7 +1409,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
for (int d = 0; d < result->defs.count; d++) {
CBMDefinition *def = &result->defs.items[d];
if (def->qualified_name && def->name) {
- insert_def_into_gbuf(ws, fi, def);
+ insert_def_into_gbuf(ws, ec->pctx ? ec->pctx->stem_marks : NULL, fi, def);
}
}
@@ -1779,7 +1784,12 @@ static int register_and_link_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *d
cbm_registry_add_lang(ctx->registry, def->name, def->qualified_name, def->label, lang);
(*reg_entries)++;
}
- const cbm_gbuf_node_t *def_node = cbm_gbuf_find_by_qn(ctx->gbuf, def->qualified_name);
+ char marked[CBM_SZ_4K];
+ const char *qn = strcmp(def->label, "Module") == 0
+ ? cbm_stem_marks_module_qn(ctx->stem_marks, def->qualified_name,
+ def->file_path, lang, marked, sizeof(marked))
+ : def->qualified_name;
+ const cbm_gbuf_node_t *def_node = cbm_gbuf_find_by_qn(ctx->gbuf, qn);
if (file_node_id > 0 && def_node) {
cbm_gbuf_insert_edge(ctx->gbuf, file_node_id, def_node->id, "DEFINES", "{}");
edges++;
diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c
index 988175a7e..badc27954 100644
--- a/src/pipeline/pass_pkgmap.c
+++ b/src/pipeline/pass_pkgmap.c
@@ -2241,6 +2241,23 @@ static bool kustomize_file_label(const char *label) {
return label && (strcmp(label, "Module") == 0 || strcmp(label, "File") == 0);
}
+/* The Folder of a directory entry: plain, or `proj.pkg.base/` when a file
+ * beside it shares the stem (cbm_stem_marks_t). */
+static const cbm_gbuf_node_t *kustomize_folder(const cbm_pipeline_ctx_t *ctx, const char *path) {
+ char *fqn = cbm_pipeline_fqn_folder(ctx->project_name, path);
+ if (!fqn) {
+ return NULL;
+ }
+ const cbm_gbuf_node_t *n = cbm_gbuf_find_by_qn(ctx->gbuf, fqn);
+ if (!n || !n->label || strcmp(n->label, "Folder") != 0) {
+ char marked[PKGMAP_PATH_BUF];
+ int len = snprintf(marked, sizeof(marked), "%s/", fqn);
+ n = len > 0 && (size_t)len < sizeof(marked) ? cbm_gbuf_find_by_qn(ctx->gbuf, marked) : NULL;
+ }
+ free(fqn);
+ return n && n->label && strcmp(n->label, "Folder") == 0 ? n : NULL;
+}
+
/* A kustomization entry is a path relative to the kustomization's own
* directory -- a manifest file or a directory -- or a remote base; it never
* names a symbol. Resolve it exactly or not at all: the name-guess strategies
@@ -2264,15 +2281,12 @@ static const cbm_gbuf_node_t *resolve_kustomize_entry(const cbm_pipeline_ctx_t *
char *qn = cbm_pipeline_fqn_module(ctx->project_name, path);
const cbm_gbuf_node_t *n = qn ? cbm_gbuf_find_by_qn(ctx->gbuf, qn) : NULL;
free(qn);
- if (n && kustomize_file_label(n->label)) {
+ /* The Module must be the entry's own file: a directory entry `base`
+ * beside `base.yaml` shares its module QN but names the directory. */
+ if (n && kustomize_file_label(n->label) && n->file_path && strcmp(n->file_path, path) == 0) {
found = n;
} else {
- char *fqn = cbm_pipeline_fqn_folder(ctx->project_name, path);
- n = fqn ? cbm_gbuf_find_by_qn(ctx->gbuf, fqn) : NULL;
- free(fqn);
- if (n && n->label && strcmp(n->label, "Folder") == 0) {
- found = n;
- }
+ found = kustomize_folder(ctx, path);
}
free(path);
return found;
diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c
index 662337aa8..002b5a46c 100644
--- a/src/pipeline/pipeline.c
+++ b/src/pipeline/pipeline.c
@@ -29,6 +29,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 7 };
#include "store/store.h"
#include "macro_table.h"
#include "arena.h"
+#include "helpers.h" /* cbm_fqn_module_source_lang */
#include "discover/discover.h"
#include "discover/userconfig.h"
#include "foundation/platform.h"
@@ -1458,13 +1459,75 @@ static void free_seen_dir_key(const char *key, void *val, void *ud) {
/* ── Pass 1: Structure ──────────────────────────────────────────── */
+/* True for languages whose module QN is the CONTAINING DIRECTORY (Java/Go
+ * package). MUST match cbm_lang_module_is_dir() (internal/cbm/helpers.c). */
+static bool stem_module_is_dir(CBMLanguage lang) {
+ return lang == CBM_LANG_JAVA || lang == CBM_LANG_GO;
+}
+
+/* Each file's Module QN, computed as extraction computes it. */
+void cbm_stem_marks_init(cbm_stem_marks_t *m, const char *project, const cbm_file_info_t *files,
+ int file_count) {
+ cbm_arena_init(&m->arena);
+ m->module_qns = cbm_ht_create(CBM_SZ_256);
+ m->dir_module_qns = cbm_ht_create(CBM_SZ_64);
+ for (int i = 0; i < file_count; i++) {
+ if (!files[i].rel_path) {
+ continue;
+ }
+ char *qn =
+ cbm_fqn_module_source_lang(&m->arena, project, files[i].rel_path, files[i].language);
+ if (qn) {
+ cbm_ht_set(stem_module_is_dir(files[i].language) ? m->dir_module_qns : m->module_qns,
+ qn, qn);
+ }
+ }
+}
+
+void cbm_stem_marks_free(cbm_stem_marks_t *m) {
+ cbm_ht_free(m->module_qns);
+ cbm_ht_free(m->dir_module_qns);
+ cbm_arena_destroy(&m->arena);
+}
+
+/* The extension names the yielding Module's file ("sh" for genproto.sh);
+ * "file" when there is none. */
+static const char *stem_mark_suffix(const char *rel_path) {
+ const char *base = strrchr(rel_path, '/');
+ base = base ? base + SKIP_ONE : rel_path;
+ const char *dot = strrchr(base, '.');
+ return dot && dot != base && dot[SKIP_ONE] ? dot + SKIP_ONE : "file";
+}
+
+const char *cbm_stem_marks_module_qn(const cbm_stem_marks_t *m, const char *qn,
+ const char *rel_path, CBMLanguage lang, char *buf,
+ size_t buf_size) {
+ if (!m || !qn || !rel_path || stem_module_is_dir(lang) || !cbm_ht_get(m->dir_module_qns, qn)) {
+ return qn;
+ }
+ int n = snprintf(buf, buf_size, "%s#%s", qn, stem_mark_suffix(rel_path));
+ return n > 0 && (size_t)n < buf_size ? buf : qn;
+}
+
+const char *cbm_stem_marks_folder_qn(const cbm_stem_marks_t *m, const char *project,
+ const char *rel_dir, char *buf, size_t buf_size) {
+ char *qn = cbm_pipeline_fqn_folder(project, rel_dir);
+ bool mark = qn && m && cbm_ht_get(m->module_qns, qn) && !cbm_ht_get(m->dir_module_qns, qn);
+ int n = qn ? snprintf(buf, buf_size, "%s%s", qn, mark ? "/" : "") : -1;
+ free(qn);
+ return n >= 0 && (size_t)n < buf_size ? buf : NULL;
+}
+
/* Create Project, Folder/Package, and File nodes in the graph buffer. */
/* Walk directory chain upward, creating Folder nodes and CONTAINS_FOLDER edges. */
-static void create_folder_chain(cbm_pipeline_t *p, const char *dir, CBMHashTable *seen_dirs) {
+static void create_folder_chain(cbm_pipeline_t *p, const cbm_stem_marks_t *marks, const char *dir,
+ CBMHashTable *seen_dirs) {
char *walk = strdup(dir);
while (walk[0] != '\0' && !cbm_ht_get(seen_dirs, walk)) {
cbm_ht_set(seen_dirs, strdup(walk), intptr_to_ptr(SKIP_ONE));
- char *folder_qn = cbm_pipeline_fqn_folder(p->project_name, walk);
+ char folder_buf[CBM_SZ_8K];
+ const char *folder_qn =
+ cbm_stem_marks_folder_qn(marks, p->project_name, walk, folder_buf, sizeof(folder_buf));
const char *dir_base = strrchr(walk, '/');
dir_base = dir_base ? dir_base + SKIP_ONE : walk;
cbm_gbuf_upsert_node(p->gbuf, "Folder", dir_base, folder_qn, walk, 0, 0, "{}");
@@ -1478,20 +1541,17 @@ static void create_folder_chain(cbm_pipeline_t *p, const char *dir, CBMHashTable
pdir = strdup("");
}
const char *pqn;
- char *pqn_heap = NULL;
+ char pqn_buf[CBM_SZ_8K];
if (pdir[0] == '\0') {
pqn = p->branch_qn ? p->branch_qn : p->project_name;
} else {
- pqn_heap = cbm_pipeline_fqn_folder(p->project_name, pdir);
- pqn = pqn_heap;
+ pqn = cbm_stem_marks_folder_qn(marks, p->project_name, pdir, pqn_buf, sizeof(pqn_buf));
}
- const cbm_gbuf_node_t *fn = cbm_gbuf_find_by_qn(p->gbuf, folder_qn);
- const cbm_gbuf_node_t *pn = cbm_gbuf_find_by_qn(p->gbuf, pqn);
+ const cbm_gbuf_node_t *fn = folder_qn ? cbm_gbuf_find_by_qn(p->gbuf, folder_qn) : NULL;
+ const cbm_gbuf_node_t *pn = pqn ? cbm_gbuf_find_by_qn(p->gbuf, pqn) : NULL;
if (fn && pn) {
cbm_gbuf_insert_edge(p->gbuf, pn->id, fn->id, "CONTAINS_FOLDER", "{}");
}
- free(folder_qn);
- free(pqn_heap);
char *up = strrchr(walk, '/');
if (up) {
*up = '\0';
@@ -1503,7 +1563,8 @@ static void create_folder_chain(cbm_pipeline_t *p, const char *dir, CBMHashTable
free(walk);
}
-static int pass_structure(cbm_pipeline_t *p, const cbm_file_info_t *files, int file_count) {
+static int pass_structure(cbm_pipeline_t *p, const cbm_stem_marks_t *marks,
+ const cbm_file_info_t *files, int file_count) {
cbm_log_info("pass.start", "pass", "structure", "files", itoa_buf(file_count));
/* Project node */
@@ -1562,27 +1623,26 @@ static int pass_structure(cbm_pipeline_t *p, const cbm_file_info_t *files, int f
}
const char *parent_qn;
- char *parent_qn_heap = NULL;
+ char parent_buf[CBM_SZ_8K];
if (dir[0] == '\0') {
parent_qn = branch_qn;
} else {
- parent_qn_heap = cbm_pipeline_fqn_folder(p->project_name, dir);
- parent_qn = parent_qn_heap;
+ parent_qn = cbm_stem_marks_folder_qn(marks, p->project_name, dir, parent_buf,
+ sizeof(parent_buf));
}
/* Walk up directory chain, creating Folder nodes */
- create_folder_chain(p, dir, seen_dirs);
+ create_folder_chain(p, marks, dir, seen_dirs);
/* Now create the CONTAINS_FILE edge */
const cbm_gbuf_node_t *fnode = cbm_gbuf_find_by_qn(p->gbuf, file_qn);
- const cbm_gbuf_node_t *pnode = cbm_gbuf_find_by_qn(p->gbuf, parent_qn);
+ const cbm_gbuf_node_t *pnode = parent_qn ? cbm_gbuf_find_by_qn(p->gbuf, parent_qn) : NULL;
if (fnode && pnode) {
cbm_gbuf_insert_edge(p->gbuf, pnode->id, fnode->id, "CONTAINS_FILE", "{}");
}
free(file_qn);
free(dir);
- free(parent_qn_heap);
}
/* Free seen_dirs keys */
@@ -3707,12 +3767,12 @@ static int run_post_extraction(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
#define MIN_FILES_FOR_PARALLEL 50
/* Run structure + extraction passes (parallel or sequential). */
-static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
- const cbm_file_info_t *files, int file_count) {
+static int run_structure_and_extraction(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
+ const cbm_file_info_t *files, int file_count) {
struct timespec t;
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
CBM_PROF_START(t_struct);
- pass_structure(p, files, file_count);
+ pass_structure(p, ctx->stem_marks, files, file_count);
CBM_PROF_END_N("pipeline", "pass_structure", t_struct, file_count);
cbm_log_info("pass.timing", "pass", "structure", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
pipeline_phase_mark("structure");
@@ -3740,6 +3800,19 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
return rc;
}
+/* The stem collisions decide Folder QNs (structure) and Module QNs
+ * (extraction); both phases read the one set. */
+static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
+ const cbm_file_info_t *files, int file_count) {
+ cbm_stem_marks_t marks;
+ cbm_stem_marks_init(&marks, p->project_name, files, file_count);
+ ctx->stem_marks = &marks;
+ int rc = run_structure_and_extraction(p, ctx, files, file_count);
+ ctx->stem_marks = NULL;
+ cbm_stem_marks_free(&marks);
+ return rc;
+}
+
static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
if (!p) {
return CBM_NOT_FOUND;
diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c
index 64c39dea0..c5f4ffafb 100644
--- a/src/pipeline/pipeline_incremental.c
+++ b/src/pipeline/pipeline_incremental.c
@@ -21,6 +21,7 @@ enum { INCR_RING_BUF = 4, INCR_RING_MASK = 3, INCR_TS_BUF = 24 };
#include "sqlite3.h"
#include "yyjson/yyjson.h"
#include "pipeline/pipeline_internal.h"
+#include "helpers.h" /* cbm_fqn_module */
#include "pipeline/doc_links.h"
#include "store/store.h"
#include "graph_buffer/graph_buffer.h"
@@ -2508,6 +2509,67 @@ static const char *closure_k8s_decline(cbm_store_t *store, const char *project,
return NULL;
}
+/* rel's directory prefix of len bytes as the plain Folder QN, in the arena. */
+static char *closure_prefix_folder_qn(CBMArena *a, const char *project, const char *rel,
+ size_t len) {
+ char *dir = cbm_arena_strndup(a, rel, len);
+ return dir ? cbm_fqn_folder(a, project, dir) : NULL;
+}
+
+/* A deleted file that took part in a stem collision (cbm_stem_marks_t)
+ * changes which side carries the mark: its own Module QN was a Folder's, or
+ * a directory above it -- which it may empty, or leave without its last
+ * Java/Go file -- holds another file's Module QN. Only a full rebuild
+ * re-spells the survivors. Both sides are read from the files before the
+ * deletion (discovered plus deleted); a deleted file's language is unknown,
+ * so it counts as a stem file either way. */
+static const char *closure_stem_mark_decline(const char *project, const cbm_file_info_t *files,
+ int file_count, char **deleted, int n_deleted) {
+ if (n_deleted <= 0) {
+ return NULL;
+ }
+ CBMArena a;
+ cbm_arena_init(&a);
+ CBMHashTable *folders = cbm_ht_create(CBM_SZ_256); /* plain Folder QNs */
+ CBMHashTable *stems = cbm_ht_create(CBM_SZ_256); /* Module QNs outside Java/Go */
+ for (int i = 0; i < file_count + n_deleted; i++) {
+ const char *rel = i < file_count ? files[i].rel_path : deleted[i - file_count];
+ if (!rel) {
+ continue;
+ }
+ bool dir_module = i < file_count &&
+ (files[i].language == CBM_LANG_JAVA || files[i].language == CBM_LANG_GO);
+ char *qn = dir_module ? NULL : cbm_fqn_module(&a, project, rel);
+ if (qn) {
+ cbm_ht_set(stems, qn, qn);
+ }
+ for (const char *s = strchr(rel, '/'); s; s = strchr(s + SKIP_ONE, '/')) {
+ char *fqn = closure_prefix_folder_qn(&a, project, rel, (size_t)(s - rel));
+ if (fqn) {
+ cbm_ht_set(folders, fqn, fqn);
+ }
+ }
+ }
+ const char *decline = NULL;
+ for (int i = 0; i < n_deleted && !decline; i++) {
+ const char *rel = deleted[i];
+ char *own = cbm_fqn_module(&a, project, rel);
+ if (own && cbm_ht_get(folders, own)) {
+ decline = "stem_mark_changed";
+ }
+ for (const char *s = strchr(rel, '/'); s && !decline; s = strchr(s + SKIP_ONE, '/')) {
+ char *fqn = closure_prefix_folder_qn(&a, project, rel, (size_t)(s - rel));
+ if (fqn && cbm_ht_get(stems, fqn)) {
+ decline = "stem_mark_changed";
+ }
+ }
+ }
+ cbm_ht_free(folders);
+ cbm_ht_free(stems);
+ cbm_arena_destroy(&a);
+ return decline;
+}
+
/* A Python file whose namespace metadata changed, or is unreadable, declines
* the closure before any hash cutoff: its importers' bindings follow the
* namespace order. A deleted file passes NULL for fresh_json. */
@@ -2647,6 +2709,11 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p
if (decline) {
goto done;
}
+ decline =
+ closure_stem_mark_decline(project, files, file_count, changed_paths + n_changed, n_deleted);
+ if (decline) {
+ goto done;
+ }
/* Load the previous generation's surfaces and probe the changed files'
* fresh ones. Missing rows fail closed. */
@@ -2943,6 +3010,10 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
int manifest_count = 0;
cbm_coverage_row_t *cov = NULL;
int cov_n = 0;
+ /* The repair re-extracts files: their Module QNs follow the stem
+ * collisions of the whole discovered set, as in a full run. */
+ cbm_stem_marks_t stem_marks;
+ cbm_stem_marks_init(&stem_marks, project, files, file_count);
cbm_doclink_scope_t *doc_base = NULL;
int doc_base_count = 0;
cbm_doc_link_row_t *doc_fresh = NULL;
@@ -3159,6 +3230,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
.doc_link_base_count = doc_base_count,
.build_decl_files = decl_files,
.build_decl_count = decl_count,
+ .stem_marks = &stem_marks,
};
for (int i = 0; i < ci; i++) {
char *file_qn = cbm_pipeline_fqn_compute(project, changed_files[i].rel_path, "__file__");
@@ -3424,6 +3496,7 @@ out:
}
cbm_delta_free_snapshot(snapshot, snapshot_count);
cbm_arena_destroy(&decl_arena);
+ cbm_stem_marks_free(&stem_marks);
surface_name_set_free(stale_surface_set);
cbm_doclinks_free_scopes(doc_base, doc_base_count);
cbm_doclinks_free_rows(doc_fresh, doc_fresh_count);
diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h
index b526e5ff7..5d8d9241c 100644
--- a/src/pipeline/pipeline_internal.h
+++ b/src/pipeline/pipeline_internal.h
@@ -284,6 +284,34 @@ void cbm_pxc_test_import_probe_fail_after(int successful_allocations);
/* cbm_route_canon_path: cbm.h (route_path.c). */
+/* A file and the directory beside it share a stem (`pkg/foo.sh` beside
+ * `pkg/foo/`): the file's Module QN equals the directory's Folder QN, and the
+ * graph buffer keeps only the Folder. One side yields:
+ * - the Folder, spelled `proj.pkg.foo/` (cbm_stem_marks_folder_qn);
+ * - but a Java/Go package directory keeps its plain QN -- it IS their module,
+ * and their imports resolve to it -- so there the file's Module yields,
+ * spelled `proj.pkg.foo#sh` (cbm_stem_marks_module_qn).
+ * Symbol QNs never change. Built from the discovered files once per run,
+ * read-only while extraction workers consult it. */
+typedef struct {
+ CBMArena arena; /* owns the QN keys of both tables */
+ CBMHashTable *module_qns; /* Module QNs of files outside Java/Go */
+ CBMHashTable *dir_module_qns; /* Module QNs of Java/Go files: their package dirs */
+} cbm_stem_marks_t;
+
+void cbm_stem_marks_init(cbm_stem_marks_t *m, const char *project, const cbm_file_info_t *files,
+ int file_count);
+void cbm_stem_marks_free(cbm_stem_marks_t *m);
+/* rel_dir's Folder QN, marked when it must yield, written into buf; NULL
+ * when it does not fit. */
+const char *cbm_stem_marks_folder_qn(const cbm_stem_marks_t *m, const char *project,
+ const char *rel_dir, char *buf, size_t buf_size);
+/* The QN a file's Module definition takes: `qn#ext` when it must yield,
+ * written into buf; qn itself otherwise (and when m is NULL). */
+const char *cbm_stem_marks_module_qn(const cbm_stem_marks_t *m, const char *qn,
+ const char *rel_path, CBMLanguage lang, char *buf,
+ size_t buf_size);
+
/* True when a graph node is a structural directory container (Folder/Project)
* rather than a code node. In a directory-based-module language (Java/Go, see
* cbm_lang_module_is_dir) a file's module QN equals its directory QN, so an
@@ -366,6 +394,10 @@ typedef struct {
char **excluded_dirs;
int excluded_count;
+ /* Stem collisions of the discovered files: which Module definitions take
+ * `qn#ext`. Borrowed for the extraction phase; NULL elsewhere. */
+ const cbm_stem_marks_t *stem_marks;
+
/* Sequential cross-LSP registry arena. The lsp_cross pass builds its
* shared per-language registries here; resolved_calls entries may BORROW
* strings owned by these registries, and the later calls pass still
@@ -1401,8 +1433,14 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr
* carry the p of its diagrams' kind family; a non-static file-scope
* C/C++/CUDA/ObjC function in a platform file or directory (foo_win.c,
* src/unix/) keeps its path QN and carries a `variant_key` property
- * (its platform-neutral QN) that relates its platform variants. */
-enum { CBM_SEMANTIC_INDEX_VERSION = 7 };
+ * (its platform-neutral QN) that relates its platform variants.
+ * 8: a file and the directory beside it with one stem (`pkg/foo.sh`,
+ * `pkg/foo/`) no longer share a QN (cbm_stem_marks_t): the Folder is
+ * spelled `proj.pkg.foo/`, or, where the directory is a Java/Go package,
+ * the file's Module is `proj.pkg.foo#sh`. The Module the Folder used to
+ * swallow is a node again. An index built before holds neither, so it
+ * rebuilds once on upgrade. */
+enum { CBM_SEMANTIC_INDEX_VERSION = 8 };
typedef struct {
cbm_gbuf_t *gbuf;
diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c
index 7aecf1cef..434ee1ba7 100644
--- a/tests/test_pipeline.c
+++ b/tests/test_pipeline.c
@@ -4536,6 +4536,249 @@ TEST(pipeline_closure_repair_new_file_declines_to_full) {
PASS();
}
+static int stem_index(const char *repo, const char *db) {
+ cbm_pipeline_t *p = cbm_pipeline_new(repo, db, CBM_MODE_FULL);
+ if (!p) {
+ return -1;
+ }
+ (void)cbm_pipeline_set_project_name(p, "stem");
+ cbm_pipeline_set_persistence(p, false);
+ int rc = cbm_pipeline_run(p);
+ cbm_pipeline_free(p);
+ return rc;
+}
+
+static void stem_unlink(const char *tmp, const char *rel) {
+ char path[512];
+ snprintf(path, sizeof(path), "%s/%s", tmp, rel);
+ (void)cbm_unlink(path);
+}
+
+/* Every collision shape of cbm_stem_marks_t, plus `fillers` plain scripts. */
+static void stem_repo(const char *tmp, int fillers) {
+ write_temp_file(tmp, "genproto.sh", "#!/bin/sh\ngen_all() {\n echo all\n}\n");
+ write_temp_file(tmp, "genproto/one.sh", "#!/bin/sh\ngen_one() {\n echo one\n}\n");
+ write_temp_file(tmp, "rs/foo.rs", "pub mod bar;\npub fn top() {}\n");
+ write_temp_file(tmp, "rs/foo/bar.rs", "pub fn inner() {}\n");
+ write_temp_file(tmp, "c/foo.h", "int foo_add(int a, int b);\n");
+ write_temp_file(tmp, "c/foo.c", "#include \"foo.h\"\nint foo_add(int a, int b) { return a + b; }\n");
+ write_temp_file(tmp, "c/foo/x.c", "int foo_x(void) { return 1; }\n");
+ write_temp_file(tmp, "ts/foo.test.ts", "export function stemProbeA(): number { return 1; }\n");
+ write_temp_file(tmp, "ts/foo/test/a.ts", "export function stemProbeB(): number { return 2; }\n");
+ write_temp_file(tmp, "go.mod", "module example.com/stem\n\ngo 1.21\n");
+ write_temp_file(tmp, "goapi/api.sh", "#!/bin/sh\nrun_api() {\n echo api\n}\n");
+ write_temp_file(tmp, "goapi/api/server.go", "package api\n\nfunc Serve() {}\n");
+ write_temp_file(tmp, "goapi/main/main.go",
+ "package main\n\nimport \"example.com/stem/goapi/api\"\n\n"
+ "func main() { api.Serve() }\n");
+ for (int i = 0; i < fillers; i++) {
+ char name[64];
+ char body[128];
+ snprintf(name, sizeof(name), "fill/f%02d.sh", i);
+ snprintf(body, sizeof(body), "#!/bin/sh\nfill_%02d() {\n echo %d\n}\n", i, i);
+ write_temp_file(tmp, name, body);
+ }
+}
+
+/* A file and the directory beside it with one stem (genproto.sh, genproto/)
+ * shared a QN, and the graph buffer kept only the Folder: the file's Module
+ * was lost. Now one side yields (cbm_stem_marks_t). The Folder is spelled
+ * `stem.genproto/` -- unless the directory is a Go package: it IS that
+ * package's module and Go imports land on it, so the file's Module yields as
+ * `stem.goapi.api#sh`. Rust's foo.rs beside foo/ and C's foo.h/foo.c stem pair
+ * keep their Module QNs; a dotted stem (foo.test.ts, foo/test/) collides by QN
+ * too. Run small (sequential extraction) and past MIN_FILES_FOR_PARALLEL
+ * (parallel workers), which insert Modules on separate paths. */
+TEST(pipeline_stem_folder_collision_keeps_both_nodes) {
+ static const char *const checks[][2] = {
+ {"SELECT COUNT(*) FROM nodes WHERE label='Module' AND qualified_name='stem.genproto' AND "
+ "file_path='genproto.sh'",
+ "genproto.sh Module"},
+ {"SELECT COUNT(*) FROM nodes WHERE label='Folder' AND qualified_name='stem.genproto/' AND "
+ "file_path='genproto'",
+ "genproto/ Folder marked"},
+ {"SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
+ "t.id=e.target_id WHERE e.type='CONTAINS_FILE' AND s.qualified_name='stem.genproto/' AND "
+ "t.file_path='genproto/one.sh'",
+ "marked Folder contains its file"},
+ {"SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
+ "t.id=e.target_id WHERE e.type='DEFINES' AND s.label='File' AND "
+ "s.file_path='genproto.sh' AND t.label='Module' AND t.qualified_name='stem.genproto'",
+ "genproto.sh defines its Module"},
+ {"SELECT COUNT(*) FROM nodes WHERE label='Module' AND qualified_name='stem.rs.foo' AND "
+ "file_path='rs/foo.rs'",
+ "Rust foo.rs Module"},
+ {"SELECT COUNT(*) FROM nodes WHERE label='Module' AND qualified_name='stem.rs.foo.bar' AND "
+ "file_path='rs/foo/bar.rs'",
+ "Rust foo/bar.rs Module"},
+ {"SELECT COUNT(*) FROM nodes WHERE label='Folder' AND qualified_name='stem.rs.foo/'",
+ "rs/foo/ Folder marked"},
+ {"SELECT COUNT(*) FROM nodes WHERE label='Module' AND qualified_name='stem.c.foo'",
+ "C stem pair Module"},
+ {"SELECT COUNT(*) FROM nodes WHERE label='Folder' AND qualified_name='stem.c.foo/'",
+ "c/foo/ Folder marked"},
+ {"SELECT COUNT(*) FROM nodes WHERE label='Module' AND qualified_name='stem.ts.foo.test' AND "
+ "file_path='ts/foo.test.ts'",
+ "dotted foo.test.ts Module"},
+ {"SELECT COUNT(*) FROM nodes WHERE label='Folder' AND qualified_name='stem.ts.foo.test/'",
+ "ts/foo/test/ Folder marked"},
+ {"SELECT COUNT(*) FROM nodes WHERE label='Folder' AND qualified_name='stem.goapi.api'",
+ "Go package Folder keeps its QN"},
+ {"SELECT COUNT(*) FROM nodes WHERE label='Module' AND qualified_name='stem.goapi.api#sh' "
+ "AND file_path='goapi/api.sh'",
+ "api.sh Module beside the Go package"},
+ {"SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
+ "t.id=e.target_id WHERE e.type='DEFINES' AND s.label='File' AND "
+ "s.file_path='goapi/api.sh' AND t.qualified_name='stem.goapi.api#sh'",
+ "api.sh defines its Module"},
+ {"SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
+ "t.id=e.target_id WHERE e.type='IMPORTS' AND s.file_path='goapi/main/main.go' AND "
+ "t.label='Folder' AND t.qualified_name='stem.goapi.api'",
+ "Go import lands on the package Folder"},
+ };
+ enum { STEM_CHECKS = sizeof(checks) / sizeof(checks[0]) };
+ static const int fillers[] = {0, 60};
+ int got[2][STEM_CHECKS];
+ int rcs[2];
+ for (int run = 0; run < 2; run++) {
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_stemdir_XXXXXX");
+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
+ stem_repo(tmp, fillers[run]);
+ char db[512];
+ snprintf(db, sizeof(db), "%s/stem.db", tmp);
+ rcs[run] = stem_index(tmp, db);
+ for (int c = 0; c < STEM_CHECKS; c++) {
+ got[run][c] = k8s_inc_count(db, checks[c][0]);
+ }
+ th_rmtree(tmp);
+ }
+ int wrong = 0;
+ for (int run = 0; run < 2; run++) {
+ ASSERT_EQ(rcs[run], 0);
+ for (int c = 0; c < STEM_CHECKS; c++) {
+ if (got[run][c] != 1) {
+ printf(" fillers=%d: %s: %d\n", fillers[run], checks[c][1], got[run][c]);
+ wrong++;
+ }
+ }
+ }
+ ASSERT_EQ(wrong, 0);
+ PASS();
+}
+
+/* A kustomization names a directory (`dashboards`) and the manifest beside it
+ * with the same stem (`dashboards.yaml`), as kubernetes/k8s.io does. The
+ * file's Module, once swallowed by the Folder, is a node now: the directory
+ * entry must still land on the (marked) Folder, the file entry on the file. */
+TEST(pipeline_kustomize_entry_beside_same_stem_file) {
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_stemkust_XXXXXX");
+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
+ write_temp_file(tmp, "monitoring/kustomization.yaml",
+ "apiVersion: kustomize.config.k8s.io/v1beta1\n"
+ "kind: Kustomization\n"
+ "resources:\n"
+ " - dashboards\n"
+ " - dashboards.yaml\n");
+ write_temp_file(tmp, "monitoring/dashboards.yaml",
+ "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: board\n");
+ write_temp_file(tmp, "monitoring/dashboards/kustomization.yaml",
+ "apiVersion: kustomize.config.k8s.io/v1beta1\n"
+ "kind: Kustomization\n"
+ "resources:\n"
+ " - cm.yaml\n");
+ write_temp_file(tmp, "monitoring/dashboards/cm.yaml",
+ "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: inner\n");
+ char db[512];
+ snprintf(db, sizeof(db), "%s/stemkust.db", tmp);
+ int rc = stem_index(tmp, db);
+ int to_dir = k8s_inc_count(
+ db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
+ "t.id=e.target_id WHERE e.type='IMPORTS' AND "
+ "s.file_path='monitoring/kustomization.yaml' AND t.label='Folder' AND "
+ "t.qualified_name='stem.monitoring.dashboards/'");
+ int to_file = k8s_inc_count(
+ db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
+ "t.id=e.target_id WHERE e.type='IMPORTS' AND "
+ "s.file_path='monitoring/kustomization.yaml' AND t.label='Module' AND "
+ "t.file_path='monitoring/dashboards.yaml'");
+ th_rmtree(tmp);
+ ASSERT_EQ(rc, 0);
+ ASSERT_EQ(to_dir, 1);
+ ASSERT_EQ(to_file, 1);
+ PASS();
+}
+
+/* Deleting a file that takes part in a stem collision changes which side
+ * carries the mark, and only a full rebuild re-spells the survivors, so the
+ * planner declines the closure repair: deleting genproto.sh frees the
+ * Folder's plain QN; deleting a Go package's only Go file leaves a plain
+ * directory, whose Folder then yields to api.sh's Module. A deletion away
+ * from every collision stays a closure repair. */
+TEST(incremental_stem_mark_deletion_rebuilds) {
+ char tmp[256];
+ snprintf(tmp, sizeof(tmp), "/tmp/cbm_stemdel_XXXXXX");
+ ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
+ closure_probe_repo(tmp);
+ write_temp_file(tmp, "genproto.sh", "#!/bin/sh\ngen_all() {\n echo all\n}\n");
+ write_temp_file(tmp, "genproto/one.sh", "#!/bin/sh\ngen_one() {\n echo one\n}\n");
+ write_temp_file(tmp, "goapi/api.sh", "#!/bin/sh\nrun_api() {\n echo api\n}\n");
+ write_temp_file(tmp, "goapi/api/server.go", "package api\n\nfunc Serve() {}\n");
+ write_temp_file(tmp, "goapi/api/run.sh", "#!/bin/sh\nrun_it() {\n echo run\n}\n");
+ write_temp_file(tmp, "misc/other.sh", "#!/bin/sh\nother() {\n echo other\n}\n");
+ char db[512];
+ snprintf(db, sizeof(db), "%s/stemdel.db", tmp);
+ static const char marked_dir[] =
+ "SELECT COUNT(*) FROM nodes WHERE label='Folder' AND qualified_name='stem.genproto/'";
+ static const char plain_dir[] =
+ "SELECT COUNT(*) FROM nodes WHERE label='Folder' AND qualified_name='stem.genproto'";
+ static const char go_side[] =
+ "SELECT COUNT(*) FROM nodes WHERE label='Module' AND qualified_name='stem.goapi.api#sh'";
+ static const char api_dir[] =
+ "SELECT COUNT(*) FROM nodes WHERE label='Folder' AND qualified_name='stem.goapi.api/'";
+ static const char api_mod[] = "SELECT COUNT(*) FROM nodes WHERE label='Module' AND "
+ "qualified_name='stem.goapi.api' AND file_path='goapi/api.sh'";
+
+ int full_rc = stem_index(tmp, db);
+ int full_marked = k8s_inc_count(db, marked_dir);
+ int full_go_side = k8s_inc_count(db, go_side);
+
+ stem_unlink(tmp, "misc/other.sh");
+ int other_rc = stem_index(tmp, db);
+ cbm_incremental_route_t other_route = cbm_pipeline_incremental_test_last_route();
+
+ stem_unlink(tmp, "genproto.sh");
+ int gen_rc = stem_index(tmp, db);
+ cbm_incremental_route_t gen_route = cbm_pipeline_incremental_test_last_route();
+ int gen_marked = k8s_inc_count(db, marked_dir);
+ int gen_plain = k8s_inc_count(db, plain_dir);
+
+ stem_unlink(tmp, "goapi/api/server.go");
+ int go_rc = stem_index(tmp, db);
+ cbm_incremental_route_t go_route = cbm_pipeline_incremental_test_last_route();
+ int go_side_after = k8s_inc_count(db, go_side);
+ int api_dir_after = k8s_inc_count(db, api_dir);
+ int api_mod_after = k8s_inc_count(db, api_mod);
+ th_rmtree(tmp);
+
+ ASSERT_EQ(full_rc, 0);
+ ASSERT_EQ(full_marked, 1);
+ ASSERT_EQ(full_go_side, 1);
+ ASSERT_EQ(other_rc, 0);
+ ASSERT_EQ(other_route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
+ ASSERT_EQ(gen_rc, 0);
+ ASSERT_EQ(gen_route, CBM_INCREMENTAL_ROUTE_FORCED_FULL);
+ ASSERT_EQ(gen_marked, 0);
+ ASSERT_EQ(gen_plain, 1);
+ ASSERT_EQ(go_rc, 0);
+ ASSERT_EQ(go_route, CBM_INCREMENTAL_ROUTE_FORCED_FULL);
+ ASSERT_EQ(go_side_after, 0);
+ ASSERT_EQ(api_dir_after, 1);
+ ASSERT_EQ(api_mod_after, 1);
+ PASS();
+}
+
TEST(pipeline_closure_repair_budget_declines_to_full) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_closure_budget_XXXXXX");
@@ -22722,6 +22965,8 @@ SUITE(pipeline) {
RUN_TEST(pipeline_sql_dump_graph_matches_the_full_parse_issue1735);
RUN_TEST(pipeline_spill_resolves_namespace_imports_like_memory);
RUN_TEST(pipeline_structure_nodes);
+ RUN_TEST(pipeline_stem_folder_collision_keeps_both_nodes);
+ RUN_TEST(pipeline_kustomize_entry_beside_same_stem_file);
RUN_TEST(pipeline_committed_counts_match_persisted);
RUN_TEST(pipeline_adr_survives_full_reindex);
RUN_TEST(pipeline_export_error_snapshot_on_artifact_failure);
@@ -23093,6 +23338,7 @@ SUITE(pipeline_semantic_manifest_repro) {
RUN_TEST(pipeline_closure_repair_removed_def_drops_dependent_edge);
RUN_TEST(pipeline_closure_repair_added_name_declines_to_full);
RUN_TEST(pipeline_closure_repair_new_file_declines_to_full);
+ RUN_TEST(incremental_stem_mark_deletion_rebuilds);
RUN_TEST(pipeline_closure_repair_budget_declines_to_full);
RUN_TEST(pipeline_incremental_tsconfig_alias_change_matches_fresh_full);
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API