Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(pipeline): cap surface callbacks at the worker budget

Martin Vogel1d7ce9ba6ac0
diff --git a/src/pipeline/lsp_surface.c b/src/pipeline/lsp_surface.c
index 5b9e283b6..66f6f3b25 100644
--- a/src/pipeline/lsp_surface.c
+++ b/src/pipeline/lsp_surface.c
@@ -483,10 +483,12 @@ typedef struct {
const CBMLSPDef *all_defs;
const int *def_starts;
cbm_lsp_surface_row_t *rows;
+ int file_count;
+ _Atomic int next_file;
_Atomic bool failed;
} surface_job_t;
-static void surface_row_one(int i, void *arg) {
+static void surface_row_build(int i, void *arg) {
surface_job_t *job = (surface_job_t *)arg;
if (atomic_load_explicit(&job->failed, memory_order_relaxed)) {
return;
@@ -526,6 +528,45 @@ static void surface_row_one(int i, void *arg) {
}
}
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+static cbm_lsp_surface_row_observer_fn g_row_observer;
+static void *g_row_observer_ctx;
+
+void cbm_lsp_surface_test_set_row_observer(cbm_lsp_surface_row_observer_fn observer, void *ctx) {
+ g_row_observer = observer;
+ g_row_observer_ctx = ctx;
+}
+#endif
+
+static void surface_row_one(int i, void *arg) {
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+ if (g_row_observer) {
+ g_row_observer(i, true, g_row_observer_ctx);
+ }
+#endif
+ surface_row_build(i, arg);
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+ if (g_row_observer) {
+ g_row_observer(i, false, g_row_observer_ctx);
+ }
+#endif
+}
+
+/* The pool also runs work on its caller. Dispatch at most one coarse job per
+ * allowed participant, each stealing files, so the total active row builders
+ * cannot exceed the pipeline budget even when the caller takes a job. */
+static void surface_rows_worker(int worker, void *arg) {
+ (void)worker;
+ surface_job_t *job = arg;
+ for (;;) {
+ int i = atomic_fetch_add_explicit(&job->next_file, 1, memory_order_relaxed);
+ if (i >= job->file_count) {
+ break;
+ }
+ surface_row_one(i, job);
+ }
+}
+
int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *project,
CBMFileResult **cache, const cbm_file_info_t *files, int file_count,
const CBMLSPDef *all_defs, const int *def_starts,
@@ -547,10 +588,15 @@ int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *projec
.all_defs = all_defs,
.def_starts = def_starts,
.rows = rows,
+ .file_count = file_count,
};
+ atomic_init(&job.next_file, 0);
atomic_init(&job.failed, false);
int workers = cbm_pipeline_worker_count(ctx ? ctx->pipeline : NULL);
- cbm_parallel_for(file_count, surface_row_one, &job,
+ if (workers > file_count) {
+ workers = file_count;
+ }
+ cbm_parallel_for(workers, surface_rows_worker, &job,
(cbm_parallel_for_opts_t){.max_workers = workers, .force_pthreads = false});
if (atomic_load_explicit(&job.failed, memory_order_relaxed)) {
cbm_store_free_lsp_surfaces(rows, file_count); /* untouched rows are all NULL */
diff --git a/src/pipeline/lsp_surface.h b/src/pipeline/lsp_surface.h
index dc198bb65..1a73b3427 100644
--- a/src/pipeline/lsp_surface.h
+++ b/src/pipeline/lsp_surface.h
@@ -68,4 +68,11 @@ int cbm_lsp_surface_python_namespace_from_json(CBMArena *arena, const char *json
*/
int cbm_lsp_surface_python_change(const char *old_json, const char *new_json);
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+/* Bracket each file callback, including skipped/error rows. Install only with
+ * no surface build running; clear after the observed build returns. */
+typedef void (*cbm_lsp_surface_row_observer_fn)(int index, bool entering, void *ctx);
+void cbm_lsp_surface_test_set_row_observer(cbm_lsp_surface_row_observer_fn observer, void *ctx);
+#endif
+
#endif /* CBM_PIPELINE_LSP_SURFACE_H */
diff --git a/src/pipeline/worker_pool.c b/src/pipeline/worker_pool.c
index 2943fd746..96611bf8d 100644
--- a/src/pipeline/worker_pool.c
+++ b/src/pipeline/worker_pool.c
@@ -22,9 +22,28 @@ enum { WP_TRUE = 1, WP_MIN = 1, WP_STEP = 1 };
/* CBM_WORKER_STACK_SIZE (8 MB per worker) lives in worker_pool.h. */
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+static cbm_parallel_spawn_observer_fn g_spawn_observer;
+static void *g_spawn_observer_ctx;
+
+void cbm_parallel_test_set_spawn_observer(cbm_parallel_spawn_observer_fn observer, void *ctx) {
+ g_spawn_observer = observer;
+ g_spawn_observer_ctx = ctx;
+}
+
+static void observe_spawn(int count, int helpers) {
+ if (g_spawn_observer) {
+ g_spawn_observer(count, helpers, g_spawn_observer_ctx);
+ }
+}
+#else
+#define observe_spawn(count, helpers) ((void)0)
+#endif
+
/* ── Serial fallback ─────────────────────────────────────────────── */
static void run_serial(int count, cbm_parallel_fn fn, void *ctx) {
+ observe_spawn(count, 0);
for (int i = 0; i < count; i++) {
fn(i, ctx);
}
@@ -103,6 +122,7 @@ static void run_pthreads_counted(int count, cbm_parallel_fn fn, void *ctx, int n
}
started++;
}
+ observe_spawn(count, started);
int main_done = 0;
uint64_t main_ops_before = cbm_memev_thread_ops();
while (WP_TRUE) {
@@ -164,6 +184,7 @@ static void run_pthreads(int count, cbm_parallel_fn fn, void *ctx, int nworkers)
}
}
+ observe_spawn(count, nworkers);
/* Main thread also participates */
while (WP_TRUE) {
int idx = atomic_fetch_add_explicit(&next_idx, WP_STEP, memory_order_relaxed);
diff --git a/src/pipeline/worker_pool.h b/src/pipeline/worker_pool.h
index 91697b961..49b3efd1f 100644
--- a/src/pipeline/worker_pool.h
+++ b/src/pipeline/worker_pool.h
@@ -36,4 +36,11 @@ typedef struct {
* If count <= 1 or workers <= 1, runs single-threaded. */
void cbm_parallel_for(int count, cbm_parallel_fn fn, void *ctx, cbm_parallel_for_opts_t opts);
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+/* Observe the completed spawn attempt before the caller takes work. Install
+ * only while no dispatch is running; clear after the observed dispatch joins. */
+typedef void (*cbm_parallel_spawn_observer_fn)(int count, int helpers, void *ctx);
+void cbm_parallel_test_set_spawn_observer(cbm_parallel_spawn_observer_fn observer, void *ctx);
+#endif
+
#endif /* CBM_WORKER_POOL_H */
diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c
index 1c7544631..9a364e941 100644
--- a/tests/test_pipeline.c
+++ b/tests/test_pipeline.c
@@ -13,6 +13,7 @@
#include "lsp/c_lsp.h"
#include "lsp/py_lsp.h"
#include "pipeline/lsp_surface.h"
+#include "pipeline/worker_pool.h"
#include "pipeline/pass_lsp_cross.h"
#include "pipeline/pipeline_internal.h"
#include "pipeline/helm_template.h"
@@ -164,6 +165,129 @@ TEST(pipeline_background_worker_policy_preserves_overrides) {
PASS();
}
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+enum { SURFACE_BUDGET_FILES = 5 };
+
+typedef struct {
+ atomic_int expected;
+ atomic_int arrivals;
+ atomic_int live;
+ atomic_int peak;
+ atomic_bool released;
+ atomic_int visits[SURFACE_BUDGET_FILES];
+ int helpers;
+} surface_budget_probe_t;
+
+static void surface_budget_spawned(int count, int helpers, void *arg) {
+ surface_budget_probe_t *probe = arg;
+ probe->helpers = helpers;
+ int participants = helpers + 1;
+ atomic_store(&probe->expected, count < participants ? count : participants);
+}
+
+static void surface_budget_row(int index, bool entering, void *arg) {
+ surface_budget_probe_t *probe = arg;
+ if (!entering) {
+ atomic_fetch_sub(&probe->live, 1);
+ return;
+ }
+ atomic_fetch_add(&probe->visits[index], 1);
+ int live = atomic_fetch_add(&probe->live, 1) + 1;
+ int peak = atomic_load(&probe->peak);
+ while (peak < live && !atomic_compare_exchange_weak(&probe->peak, &peak, live)) {}
+ atomic_fetch_add(&probe->arrivals, 1);
+ /* No timing window: hold the first wave until every possible participant
+ * has entered. The post-spawn observation accounts for partial thread
+ * creation and serial fallback, so neither can strand this gate. The test
+ * runner's watchdog bounds a broken dispatch without making time a verdict. */
+ while (!atomic_load(&probe->released)) {
+ int expected = atomic_load(&probe->expected);
+ if (expected > 0 && atomic_load(&probe->arrivals) >= expected) {
+ atomic_store(&probe->released, true);
+ }
+ }
+}
+
+TEST(pipeline_surface_respects_total_worker_budget) {
+ CBMFileResult results[SURFACE_BUDGET_FILES] = {0};
+ CBMFileResult *cache[SURFACE_BUDGET_FILES];
+ cbm_file_info_t files[SURFACE_BUDGET_FILES] = {
+ {.rel_path = "a.go", .language = CBM_LANG_GO},
+ {.rel_path = "b.go", .language = CBM_LANG_GO},
+ {.rel_path = "c.go", .language = CBM_LANG_GO},
+ {.rel_path = "d.go", .language = CBM_LANG_GO},
+ {.rel_path = "e.go", .language = CBM_LANG_GO},
+ };
+ surface_budget_probe_t probe = {0};
+ for (int i = 0; i < SURFACE_BUDGET_FILES; i++) {
+ cache[i] = &results[i];
+ atomic_init(&probe.visits[i], 0);
+ }
+ cache[1] = NULL; /* The skipped row must not change stable file ordering. */
+ atomic_init(&probe.expected, 0);
+ atomic_init(&probe.arrivals, 0);
+ atomic_init(&probe.live, 0);
+ atomic_init(&probe.peak, 0);
+ atomic_init(&probe.released, false);
+ const char *old_workers = getenv("CBM_WORKERS");
+ char *saved_workers = old_workers ? strdup(old_workers) : NULL;
+ const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD");
+ char *saved_single = old_single ? strdup(old_single) : NULL;
+ cbm_setenv("CBM_WORKERS", "2", 1);
+ cbm_unsetenv("CBM_INDEX_SINGLE_THREAD");
+ cbm_pipeline_t *pipeline = cbm_pipeline_new("/some/path", NULL, CBM_MODE_FULL);
+ cbm_pipeline_set_background(pipeline, true);
+ cbm_pipeline_ctx_t ctx = {.pipeline = pipeline};
+ cbm_lsp_surface_row_t *rows = NULL;
+ int count = 0;
+ cbm_parallel_test_set_spawn_observer(surface_budget_spawned, &probe);
+ cbm_lsp_surface_test_set_row_observer(surface_budget_row, &probe);
+ int rc = cbm_lsp_surface_build_rows(&ctx, "budget", cache, files, SURFACE_BUDGET_FILES, NULL,
+ NULL, &rows, &count);
+ int parallel_peak = atomic_load(&probe.peak);
+ int parallel_helpers = probe.helpers;
+ atomic_store(&probe.expected, 0);
+ atomic_store(&probe.arrivals, 0);
+ atomic_store(&probe.peak, 0);
+ atomic_store(&probe.released, false);
+ cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1);
+ cbm_lsp_surface_row_t *serial_rows = NULL;
+ int serial_count = 0;
+ int serial_rc = cbm_lsp_surface_build_rows(&ctx, "budget", cache, files, SURFACE_BUDGET_FILES,
+ NULL, NULL, &serial_rows, &serial_count);
+ cbm_lsp_surface_test_set_row_observer(NULL, NULL);
+ cbm_parallel_test_set_spawn_observer(NULL, NULL);
+ bool same = rc == 0 && serial_rc == 0 && rows && serial_rows &&
+ count == SURFACE_BUDGET_FILES - 1 && count == serial_count;
+ for (int i = 0; same && i < count; i++) {
+ same = strcmp(rows[i].rel_path, files[i < 1 ? i : i + 1].rel_path) == 0 &&
+ strcmp(rows[i].defs_json, serial_rows[i].defs_json) == 0 &&
+ strcmp(rows[i].surface_sha, serial_rows[i].surface_sha) == 0;
+ }
+ cbm_store_free_lsp_surfaces(rows, count);
+ cbm_store_free_lsp_surfaces(serial_rows, serial_count);
+ cbm_pipeline_free(pipeline);
+ saved_workers ? cbm_setenv("CBM_WORKERS", saved_workers, 1) : cbm_unsetenv("CBM_WORKERS");
+ saved_single ? cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1)
+ : cbm_unsetenv("CBM_INDEX_SINGLE_THREAD");
+ free(saved_workers);
+ free(saved_single);
+
+ ASSERT_EQ(rc, 0);
+ ASSERT_EQ(serial_rc, 0);
+ ASSERT_TRUE(same);
+ ASSERT_GTE(parallel_helpers, 1); /* A serial setup is not a concurrency repro. */
+ ASSERT_EQ(probe.helpers, 0);
+ ASSERT_EQ(atomic_load(&probe.live), 0);
+ for (int i = 0; i < SURFACE_BUDGET_FILES; i++) {
+ ASSERT_EQ(atomic_load(&probe.visits[i]), 2);
+ }
+ ASSERT_EQ(atomic_load(&probe.peak), 1);
+ ASSERT_EQ(parallel_peak, 2);
+ PASS();
+}
+#endif
+
TEST(pipeline_background_policy_preserves_index_results) {
ASSERT_EQ(setup_test_repo(), 0);
/* 64 fillers plus setup_test_repo's three files exceed MIN_FILES_FOR_PARALLEL
@@ -27721,6 +27845,9 @@ SUITE(pipeline) {
/* Lifecycle */
RUN_TEST(pipeline_create_free);
RUN_TEST(pipeline_background_worker_policy_preserves_overrides);
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+ RUN_TEST(pipeline_surface_respects_total_worker_budget);
+#endif
RUN_TEST(pipeline_background_policy_preserves_index_results);
RUN_TEST(pipeline_null_repo);
RUN_TEST(pipeline_free_null);