Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(pipeline): cap surface callbacks at the worker budget
Martin Vogel1d7ce9ba6ac0
diff --git a/src/pipeline/lsp_surface.c b/src/pipeline/lsp_surface.cindex 5b9e283b6..66f6f3b25 100644--- a/src/pipeline/lsp_surface.c+++ b/src/pipeline/lsp_surface.c@@ -483,10 +483,12 @@ typedef struct {const CBMLSPDef *all_defs;const int *def_starts;cbm_lsp_surface_row_t *rows;+ int file_count;+ _Atomic int next_file;_Atomic bool failed;} surface_job_t;-static void surface_row_one(int i, void *arg) {+static void surface_row_build(int i, void *arg) {surface_job_t *job = (surface_job_t *)arg;if (atomic_load_explicit(&job->failed, memory_order_relaxed)) {return;@@ -526,6 +528,45 @@ static void surface_row_one(int i, void *arg) {}}+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS+static cbm_lsp_surface_row_observer_fn g_row_observer;+static void *g_row_observer_ctx;++void cbm_lsp_surface_test_set_row_observer(cbm_lsp_surface_row_observer_fn observer, void *ctx) {+ g_row_observer = observer;+ g_row_observer_ctx = ctx;+}+#endif++static void surface_row_one(int i, void *arg) {+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS+ if (g_row_observer) {+ g_row_observer(i, true, g_row_observer_ctx);+ }+#endif+ surface_row_build(i, arg);+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS+ if (g_row_observer) {+ g_row_observer(i, false, g_row_observer_ctx);+ }+#endif+}++/* The pool also runs work on its caller. Dispatch at most one coarse job per+ * allowed participant, each stealing files, so the total active row builders+ * cannot exceed the pipeline budget even when the caller takes a job. */+static void surface_rows_worker(int worker, void *arg) {+ (void)worker;+ surface_job_t *job = arg;+ for (;;) {+ int i = atomic_fetch_add_explicit(&job->next_file, 1, memory_order_relaxed);+ if (i >= job->file_count) {+ break;+ }+ surface_row_one(i, job);+ }+}+int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *project,CBMFileResult **cache, const cbm_file_info_t *files, int file_count,const CBMLSPDef *all_defs, const int *def_starts,@@ -547,10 +588,15 @@ int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *projec.all_defs = all_defs,.def_starts = def_starts,.rows = rows,+ .file_count = file_count,};+ atomic_init(&job.next_file, 0);atomic_init(&job.failed, false);int workers = cbm_pipeline_worker_count(ctx ? ctx->pipeline : NULL);- cbm_parallel_for(file_count, surface_row_one, &job,+ if (workers > file_count) {+ workers = file_count;+ }+ cbm_parallel_for(workers, surface_rows_worker, &job,(cbm_parallel_for_opts_t){.max_workers = workers, .force_pthreads = false});if (atomic_load_explicit(&job.failed, memory_order_relaxed)) {cbm_store_free_lsp_surfaces(rows, file_count); /* untouched rows are all NULL */diff --git a/src/pipeline/lsp_surface.h b/src/pipeline/lsp_surface.hindex dc198bb65..1a73b3427 100644--- a/src/pipeline/lsp_surface.h+++ b/src/pipeline/lsp_surface.h@@ -68,4 +68,11 @@ int cbm_lsp_surface_python_namespace_from_json(CBMArena *arena, const char *json*/int cbm_lsp_surface_python_change(const char *old_json, const char *new_json);+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS+/* Bracket each file callback, including skipped/error rows. Install only with+ * no surface build running; clear after the observed build returns. */+typedef void (*cbm_lsp_surface_row_observer_fn)(int index, bool entering, void *ctx);+void cbm_lsp_surface_test_set_row_observer(cbm_lsp_surface_row_observer_fn observer, void *ctx);+#endif+#endif /* CBM_PIPELINE_LSP_SURFACE_H */diff --git a/src/pipeline/worker_pool.c b/src/pipeline/worker_pool.cindex 2943fd746..96611bf8d 100644--- a/src/pipeline/worker_pool.c+++ b/src/pipeline/worker_pool.c@@ -22,9 +22,28 @@ enum { WP_TRUE = 1, WP_MIN = 1, WP_STEP = 1 };/* CBM_WORKER_STACK_SIZE (8 MB per worker) lives in worker_pool.h. */+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS+static cbm_parallel_spawn_observer_fn g_spawn_observer;+static void *g_spawn_observer_ctx;++void cbm_parallel_test_set_spawn_observer(cbm_parallel_spawn_observer_fn observer, void *ctx) {+ g_spawn_observer = observer;+ g_spawn_observer_ctx = ctx;+}++static void observe_spawn(int count, int helpers) {+ if (g_spawn_observer) {+ g_spawn_observer(count, helpers, g_spawn_observer_ctx);+ }+}+#else+#define observe_spawn(count, helpers) ((void)0)+#endif+/* ── Serial fallback ─────────────────────────────────────────────── */static void run_serial(int count, cbm_parallel_fn fn, void *ctx) {+ observe_spawn(count, 0);for (int i = 0; i < count; i++) {fn(i, ctx);}@@ -103,6 +122,7 @@ static void run_pthreads_counted(int count, cbm_parallel_fn fn, void *ctx, int n}started++;}+ observe_spawn(count, started);int main_done = 0;uint64_t main_ops_before = cbm_memev_thread_ops();while (WP_TRUE) {@@ -164,6 +184,7 @@ static void run_pthreads(int count, cbm_parallel_fn fn, void *ctx, int nworkers)}}+ observe_spawn(count, nworkers);/* Main thread also participates */while (WP_TRUE) {int idx = atomic_fetch_add_explicit(&next_idx, WP_STEP, memory_order_relaxed);diff --git a/src/pipeline/worker_pool.h b/src/pipeline/worker_pool.hindex 91697b961..49b3efd1f 100644--- a/src/pipeline/worker_pool.h+++ b/src/pipeline/worker_pool.h@@ -36,4 +36,11 @@ typedef struct {* If count <= 1 or workers <= 1, runs single-threaded. */void cbm_parallel_for(int count, cbm_parallel_fn fn, void *ctx, cbm_parallel_for_opts_t opts);+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS+/* Observe the completed spawn attempt before the caller takes work. Install+ * only while no dispatch is running; clear after the observed dispatch joins. */+typedef void (*cbm_parallel_spawn_observer_fn)(int count, int helpers, void *ctx);+void cbm_parallel_test_set_spawn_observer(cbm_parallel_spawn_observer_fn observer, void *ctx);+#endif+#endif /* CBM_WORKER_POOL_H */diff --git a/tests/test_pipeline.c b/tests/test_pipeline.cindex 1c7544631..9a364e941 100644--- a/tests/test_pipeline.c+++ b/tests/test_pipeline.c@@ -13,6 +13,7 @@#include "lsp/c_lsp.h"#include "lsp/py_lsp.h"#include "pipeline/lsp_surface.h"+#include "pipeline/worker_pool.h"#include "pipeline/pass_lsp_cross.h"#include "pipeline/pipeline_internal.h"#include "pipeline/helm_template.h"@@ -164,6 +165,129 @@ TEST(pipeline_background_worker_policy_preserves_overrides) {PASS();}+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS+enum { SURFACE_BUDGET_FILES = 5 };++typedef struct {+ atomic_int expected;+ atomic_int arrivals;+ atomic_int live;+ atomic_int peak;+ atomic_bool released;+ atomic_int visits[SURFACE_BUDGET_FILES];+ int helpers;+} surface_budget_probe_t;++static void surface_budget_spawned(int count, int helpers, void *arg) {+ surface_budget_probe_t *probe = arg;+ probe->helpers = helpers;+ int participants = helpers + 1;+ atomic_store(&probe->expected, count < participants ? count : participants);+}++static void surface_budget_row(int index, bool entering, void *arg) {+ surface_budget_probe_t *probe = arg;+ if (!entering) {+ atomic_fetch_sub(&probe->live, 1);+ return;+ }+ atomic_fetch_add(&probe->visits[index], 1);+ int live = atomic_fetch_add(&probe->live, 1) + 1;+ int peak = atomic_load(&probe->peak);+ while (peak < live && !atomic_compare_exchange_weak(&probe->peak, &peak, live)) {}+ atomic_fetch_add(&probe->arrivals, 1);+ /* No timing window: hold the first wave until every possible participant+ * has entered. The post-spawn observation accounts for partial thread+ * creation and serial fallback, so neither can strand this gate. The test+ * runner's watchdog bounds a broken dispatch without making time a verdict. */+ while (!atomic_load(&probe->released)) {+ int expected = atomic_load(&probe->expected);+ if (expected > 0 && atomic_load(&probe->arrivals) >= expected) {+ atomic_store(&probe->released, true);+ }+ }+}++TEST(pipeline_surface_respects_total_worker_budget) {+ CBMFileResult results[SURFACE_BUDGET_FILES] = {0};+ CBMFileResult *cache[SURFACE_BUDGET_FILES];+ cbm_file_info_t files[SURFACE_BUDGET_FILES] = {+ {.rel_path = "a.go", .language = CBM_LANG_GO},+ {.rel_path = "b.go", .language = CBM_LANG_GO},+ {.rel_path = "c.go", .language = CBM_LANG_GO},+ {.rel_path = "d.go", .language = CBM_LANG_GO},+ {.rel_path = "e.go", .language = CBM_LANG_GO},+ };+ surface_budget_probe_t probe = {0};+ for (int i = 0; i < SURFACE_BUDGET_FILES; i++) {+ cache[i] = &results[i];+ atomic_init(&probe.visits[i], 0);+ }+ cache[1] = NULL; /* The skipped row must not change stable file ordering. */+ atomic_init(&probe.expected, 0);+ atomic_init(&probe.arrivals, 0);+ atomic_init(&probe.live, 0);+ atomic_init(&probe.peak, 0);+ atomic_init(&probe.released, false);+ const char *old_workers = getenv("CBM_WORKERS");+ char *saved_workers = old_workers ? strdup(old_workers) : NULL;+ const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD");+ char *saved_single = old_single ? strdup(old_single) : NULL;+ cbm_setenv("CBM_WORKERS", "2", 1);+ cbm_unsetenv("CBM_INDEX_SINGLE_THREAD");+ cbm_pipeline_t *pipeline = cbm_pipeline_new("/some/path", NULL, CBM_MODE_FULL);+ cbm_pipeline_set_background(pipeline, true);+ cbm_pipeline_ctx_t ctx = {.pipeline = pipeline};+ cbm_lsp_surface_row_t *rows = NULL;+ int count = 0;+ cbm_parallel_test_set_spawn_observer(surface_budget_spawned, &probe);+ cbm_lsp_surface_test_set_row_observer(surface_budget_row, &probe);+ int rc = cbm_lsp_surface_build_rows(&ctx, "budget", cache, files, SURFACE_BUDGET_FILES, NULL,+ NULL, &rows, &count);+ int parallel_peak = atomic_load(&probe.peak);+ int parallel_helpers = probe.helpers;+ atomic_store(&probe.expected, 0);+ atomic_store(&probe.arrivals, 0);+ atomic_store(&probe.peak, 0);+ atomic_store(&probe.released, false);+ cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1);+ cbm_lsp_surface_row_t *serial_rows = NULL;+ int serial_count = 0;+ int serial_rc = cbm_lsp_surface_build_rows(&ctx, "budget", cache, files, SURFACE_BUDGET_FILES,+ NULL, NULL, &serial_rows, &serial_count);+ cbm_lsp_surface_test_set_row_observer(NULL, NULL);+ cbm_parallel_test_set_spawn_observer(NULL, NULL);+ bool same = rc == 0 && serial_rc == 0 && rows && serial_rows &&+ count == SURFACE_BUDGET_FILES - 1 && count == serial_count;+ for (int i = 0; same && i < count; i++) {+ same = strcmp(rows[i].rel_path, files[i < 1 ? i : i + 1].rel_path) == 0 &&+ strcmp(rows[i].defs_json, serial_rows[i].defs_json) == 0 &&+ strcmp(rows[i].surface_sha, serial_rows[i].surface_sha) == 0;+ }+ cbm_store_free_lsp_surfaces(rows, count);+ cbm_store_free_lsp_surfaces(serial_rows, serial_count);+ cbm_pipeline_free(pipeline);+ saved_workers ? cbm_setenv("CBM_WORKERS", saved_workers, 1) : cbm_unsetenv("CBM_WORKERS");+ saved_single ? cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1)+ : cbm_unsetenv("CBM_INDEX_SINGLE_THREAD");+ free(saved_workers);+ free(saved_single);++ ASSERT_EQ(rc, 0);+ ASSERT_EQ(serial_rc, 0);+ ASSERT_TRUE(same);+ ASSERT_GTE(parallel_helpers, 1); /* A serial setup is not a concurrency repro. */+ ASSERT_EQ(probe.helpers, 0);+ ASSERT_EQ(atomic_load(&probe.live), 0);+ for (int i = 0; i < SURFACE_BUDGET_FILES; i++) {+ ASSERT_EQ(atomic_load(&probe.visits[i]), 2);+ }+ ASSERT_EQ(atomic_load(&probe.peak), 1);+ ASSERT_EQ(parallel_peak, 2);+ PASS();+}+#endif+TEST(pipeline_background_policy_preserves_index_results) {ASSERT_EQ(setup_test_repo(), 0);/* 64 fillers plus setup_test_repo's three files exceed MIN_FILES_FOR_PARALLEL@@ -27721,6 +27845,9 @@ SUITE(pipeline) {/* Lifecycle */RUN_TEST(pipeline_create_free);RUN_TEST(pipeline_background_worker_policy_preserves_overrides);+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS+ RUN_TEST(pipeline_surface_respects_total_worker_budget);+#endifRUN_TEST(pipeline_background_policy_preserves_index_results);RUN_TEST(pipeline_null_repo);RUN_TEST(pipeline_free_null);