Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix: preserve CPU headroom during background indexing

Matt Van Horn2443e92bece2
diff --git a/src/daemon/application.c b/src/daemon/application.c
index 22c62029b..91b5937ec 100644
--- a/src/daemon/application.c
+++ b/src/daemon/application.c
@@ -2199,10 +2199,10 @@ static char *application_auto_index_args(cbm_daemon_application_t *application,
return NULL;
}
yyjson_mut_doc_set_root(document, root);
- char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) &&
- application_index_args_add_policy(application, document, root)
- ? yyjson_mut_write(document, 0, NULL)
- : NULL;
+ bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) &&
+ application_index_args_add_policy(application, document, root) &&
+ yyjson_mut_obj_add_bool(document, root, "_background", true);
+ char *args = encoded ? yyjson_mut_write(document, 0, NULL) : NULL;
yyjson_mut_doc_free(document);
return args;
}
@@ -3894,7 +3894,8 @@ static int application_background_index(cbm_daemon_application_t *application,
}
yyjson_mut_doc_set_root(document, root);
bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) &&
- application_index_args_add_policy(application, document, root);
+ application_index_args_add_policy(application, document, root) &&
+ yyjson_mut_obj_add_bool(document, root, "_background", true);
char *default_project = cbm_project_name_from_path(canonical_root);
bool custom_project =
project_name[0] && (!default_project || strcmp(default_project, project_name) != 0);
diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c
index c877d5110..0d15ac017 100644
--- a/src/mcp/mcp.c
+++ b/src/mcp/mcp.c
@@ -11109,7 +11109,8 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p
yyjson_mut_val *root = yyjson_mut_obj(doc);
yyjson_mut_doc_set_root(doc, root);
if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) ||
- !cbm_mcp_index_policy_add_to_args(doc, root, &policy)) {
+ !cbm_mcp_index_policy_add_to_args(doc, root, &policy) ||
+ !yyjson_mut_obj_add_bool(doc, root, "_background", true)) {
yyjson_mut_doc_free(doc);
return NULL;
}
@@ -11498,6 +11499,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
char *repo_path = cbm_mcp_get_string_arg(args, "repo_path");
char *mode_str = cbm_mcp_get_string_arg(args, "mode");
char *name_override = cbm_mcp_get_string_arg(args, "name");
+ bool background = cbm_mcp_get_bool_arg(args, "_background");
cbm_normalize_path_sep(repo_path);
if (!repo_path) {
@@ -11660,6 +11662,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
free(repo_path);
return cbm_mcp_text_result("failed to create pipeline", true);
}
+ cbm_pipeline_set_background(p, background);
if (name_override && name_override[0] && !cbm_pipeline_set_project_name(p, name_override)) {
cbm_pipeline_free(p);
mcp_project_mutation_end(srv, mutation_project);
@@ -18106,6 +18109,7 @@ static void *autoindex_thread(void *arg) {
cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed");
return NULL;
}
+ cbm_pipeline_set_background(p, true);
/* Block until any concurrent pipeline finishes */
cbm_pipeline_lock();
diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c
index ba41b877c..f2aeb47be 100644
--- a/src/pipeline/pipeline.c
+++ b/src/pipeline/pipeline.c
@@ -71,6 +71,18 @@ static inline void *intptr_to_ptr(intptr_t v) {
* Atomic spinlock: 0 = free, 1 = locked. */
static atomic_int g_pipeline_busy = 0;
+#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
+static atomic_int g_last_worker_count = 0;
+
+void cbm_pipeline_worker_count_test_reset(void) {
+ atomic_store(&g_last_worker_count, 0);
+}
+
+int cbm_pipeline_worker_count_test_last(void) {
+ return atomic_load(&g_last_worker_count);
+}
+#endif
+
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
static atomic_bool g_persist_test_fail_after_stage_dump = false;
static atomic_bool g_persist_test_cancel_after_predump = false;
@@ -192,6 +204,7 @@ struct cbm_pipeline {
bool persistence; /* write .codebase-memory/graph.db.zst after indexing */
cbm_index_resource_policy_t resource_policy;
cbm_index_resource_violation_t resource_violation;
+ bool background; /* reserve CPU headroom when no user is waiting */
/* Snapshot of the artifact export failure of THIS run (set only by
* export_after_publish failure, zeroed at run start, cleared on success).
@@ -400,6 +413,12 @@ const char *cbm_pipeline_export_error(const cbm_pipeline_t *p) {
return p ? p->export_error : "";
}
+void cbm_pipeline_set_background(cbm_pipeline_t *p, bool background) {
+ if (p) {
+ p->background = background;
+ }
+}
+
bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) {
if (!p || !name || !name[0]) {
return false;
@@ -613,12 +632,18 @@ void cbm_pipeline_set_committed_counts(cbm_pipeline_t *p, int nodes, int edges)
* crasher; a parallel re-run would race the marker. Honour that override
* everywhere the worker count drives the parallel/sequential decision, so the
* whole extraction phase collapses to the deterministic sequential path. */
-static int effective_worker_count(bool initial) {
+int cbm_pipeline_worker_count(const cbm_pipeline_t *p) {
const char *st = getenv("CBM_INDEX_SINGLE_THREAD");
+ int workers;
if (st && st[0] == '1') {
- return 1;
+ workers = 1;
+ } else {
+ workers = cbm_default_worker_count(!p || !p->background);
}
- return cbm_default_worker_count(initial);
+#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
+ atomic_store(&g_last_worker_count, workers);
+#endif
+ return workers;
}
/* Resolve the DB path for this pipeline. Caller must free(). */
@@ -2590,7 +2615,7 @@ static int run_githistory(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) {
gh_compute_arg_t gh_arg = {.repo_path = ctx->repo_path, .result = &gh_result};
if (p->mode != CBM_MODE_FAST) {
- if (effective_worker_count(true) > SKIP_ONE) {
+ if (cbm_pipeline_worker_count(p) > SKIP_ONE) {
if (cbm_thread_create(&gh_thread, 0, gh_compute_thread_fn, &gh_arg) == 0) {
gh_threaded = true;
}
@@ -2690,7 +2715,7 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
return CBM_NOT_FOUND;
}
- int worker_count = effective_worker_count(true);
+ int worker_count = cbm_pipeline_worker_count(p);
CBM_PROF_START(t_extract_total);
int rc = (worker_count > SKIP_ONE && file_count > MIN_FILES_FOR_PARALLEL)
? run_parallel_pipeline(p, ctx, files, file_count, worker_count, &t)
diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h
index c1d571ecd..e52513673 100644
--- a/src/pipeline/pipeline.h
+++ b/src/pipeline/pipeline.h
@@ -59,6 +59,18 @@ void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resourc
/* Copy the exact discovery violation from the most recent run. */
void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p,
cbm_index_resource_violation_t *violation);
+/* Mark work that runs without a waiting user. Background pipelines reserve
+ * CPU headroom; foreground pipelines retain the initial-index all-core policy. */
+void cbm_pipeline_set_background(cbm_pipeline_t *p, bool background);
+
+/* Resolve the worker policy for this pipeline, including environment and
+ * crash-recovery overrides. */
+int cbm_pipeline_worker_count(const cbm_pipeline_t *p);
+
+#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
+void cbm_pipeline_worker_count_test_reset(void);
+int cbm_pipeline_worker_count_test_last(void);
+#endif
/* Snapshot of the artifact export failure of the last cbm_pipeline_run, or ""
* when the run succeeded / did not reach post-publish export. Used to
diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c
index 154518658..4e7d61dc3 100644
--- a/src/pipeline/pipeline_incremental.c
+++ b/src/pipeline/pipeline_incremental.c
@@ -1212,7 +1212,7 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed
* a full build takes, which is what makes its output converge. */
#define MIN_FILES_FOR_PARALLEL_INCR 50
- int worker_count = cbm_default_worker_count(true);
+ int worker_count = cbm_pipeline_worker_count(ctx->pipeline);
bool use_parallel =
closure != NULL || (worker_count > SKIP_ONE && ci > MIN_FILES_FOR_PARALLEL_INCR);
@@ -1590,7 +1590,7 @@ static int closure_probe_surfaces(cbm_pipeline_t *p, const char *project,
_Atomic int64_t probe_ids;
atomic_init(&probe_ids, cbm_gbuf_next_id(probe_gbuf));
rc = cbm_parallel_extract(&probe_ctx, probe_files, probe_count, cache, &probe_ids,
- cbm_default_worker_count(true));
+ cbm_pipeline_worker_count(p));
}
if (rc == 0) {
char **def_modules = (char **)calloc((size_t)probe_count, sizeof(char *));
@@ -2061,7 +2061,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
elig[elig_count++] = row;
}
}
- int workers = cbm_default_worker_count(true);
+ int workers = cbm_pipeline_worker_count(p);
if (workers < 1) {
workers = 1;
}
diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c
index 7f4c9cc28..50abe5a8c 100644
--- a/tests/test_daemon_application.c
+++ b/tests/test_daemon_application.c
@@ -1375,6 +1375,7 @@ typedef struct {
* itself instead, which is the edge a reader on another thread needs. */
atomic_bool args_captured[APP_FAKE_MAX_ATTEMPTS];
size_t memory_budgets[APP_FAKE_MAX_ATTEMPTS];
+ bool background_requests[APP_FAKE_MAX_ATTEMPTS];
} app_fake_worker_context_t;
typedef struct {
@@ -1442,6 +1443,8 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem
args_json ? args_json : "");
atomic_store(&context->args_captured[worker->attempt], true);
context->memory_budgets[worker->attempt] = memory_budget_bytes;
+ context->background_requests[worker->attempt] =
+ cbm_mcp_get_bool_arg(args_json, "_background");
if (marker_file) {
(void)snprintf(context->marker_paths[worker->attempt], APP_TEST_PATH_CAP, "%s",
marker_file);
@@ -2202,6 +2205,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) {
ASSERT_TRUE(first_initialized);
ASSERT_TRUE(first_owned);
ASSERT_TRUE(auto_policy_propagated);
+ ASSERT_TRUE(fake.background_requests[0]);
ASSERT_TRUE(second_initialized);
ASSERT_TRUE(coalesced);
ASSERT_TRUE(restricted_disconnect_kept_job);
@@ -3634,6 +3638,7 @@ TEST(daemon_application_request_cancel_preserves_persistent_watch_and_session) {
ASSERT_TRUE(thread_started);
ASSERT_TRUE(subscribed);
ASSERT_TRUE(worker_started);
+ ASSERT_FALSE(fixture.fake.background_requests[0]);
ASSERT_TRUE(request_returned);
ASSERT_TRUE(thread_joined);
ASSERT_EQ(request.status, CBM_DAEMON_RUNTIME_APPLICATION_CANCELLED);
@@ -3931,6 +3936,7 @@ TEST(daemon_application_final_cancel_drains_admitted_watcher_job) {
ASSERT_TRUE(fixture_ready);
ASSERT_TRUE(thread_started);
ASSERT_TRUE(worker_started);
+ ASSERT_TRUE(fixture.fake.background_requests[0]);
ASSERT_TRUE(job_active);
ASSERT_EQ(watches_after_cancel, 0);
ASSERT_TRUE(thread_joined);
diff --git a/tests/test_mcp.c b/tests/test_mcp.c
index 22af972af..34563e0c9 100644
--- a/tests/test_mcp.c
+++ b/tests/test_mcp.c
@@ -20006,6 +20006,73 @@ TEST(autoindex_limit_guards_git_root_issue713) {
PASS();
}
+TEST(mcp_auto_index_in_process_uses_background_worker_policy) {
+ char cache[256];
+ char repo[512];
+ (void)snprintf(cache, sizeof(cache), "%s/cbm-autoindex-workers-XXXXXX", cbm_tmpdir());
+ bool cache_ready = cbm_mkdtemp(cache) != NULL;
+ (void)snprintf(repo, sizeof(repo), "%s/repo", cache);
+ char source[640];
+ (void)snprintf(source, sizeof(source), "%s/main.py", repo);
+ bool repo_ready = cache_ready && th_mkdir_p(repo) == 0 &&
+ th_write_file(source, "def background_index():\n return True\n") == 0;
+
+ mcp_test_env_backup_t environment[] = {
+ {.name = "CBM_CACHE_DIR"},
+ {.name = "CBM_WORKERS"},
+ {.name = "CBM_INDEX_SINGLE_THREAD"},
+ };
+ bool environment_saved = true;
+ for (size_t i = 0; i < sizeof(environment) / sizeof(environment[0]); i++) {
+ const char *value = getenv(environment[i].name);
+ environment[i].present = value != NULL;
+ environment[i].value = value ? strdup(value) : NULL;
+ environment_saved = environment_saved && (!value || environment[i].value);
+ }
+ bool environment_ready =
+ environment_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0 &&
+ cbm_unsetenv("CBM_WORKERS") == 0 && cbm_unsetenv("CBM_INDEX_SINGLE_THREAD") == 0;
+
+ char old_cwd[CBM_SZ_4K] = {0};
+ bool cwd_ready = repo_ready && environment_ready && cbm_getcwd(old_cwd, sizeof(old_cwd)) &&
+ cbm_chdir(repo) == 0;
+ cbm_config_t *config = cwd_ready ? cbm_config_open(cache) : NULL;
+ bool config_ready = config && cbm_config_set(config, CBM_CONFIG_AUTO_INDEX, "true") == 0 &&
+ cbm_config_set(config, CBM_CONFIG_AUTO_WATCH, "false") == 0;
+ cbm_pipeline_worker_count_test_reset();
+ cbm_mcp_server_t *server = config_ready ? cbm_mcp_server_new(NULL) : NULL;
+ char *response = NULL;
+ if (server) {
+ cbm_mcp_server_set_config(server, config);
+ response = cbm_mcp_server_handle(
+ server, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{}}");
+ cbm_mcp_server_free(server); /* joins the automatic index thread */
+ }
+ int selected_workers = cbm_pipeline_worker_count_test_last();
+ bool server_ready = server != NULL;
+ bool response_ready = response != NULL;
+
+ free(response);
+ cbm_config_close(config);
+ if (cwd_ready) {
+ (void)cbm_chdir(old_cwd);
+ }
+ mcp_test_restore_env(environment, sizeof(environment) / sizeof(environment[0]));
+ bool cleaned = !cache_ready || th_rmtree(cache) == 0;
+
+ ASSERT_TRUE(cache_ready);
+ ASSERT_TRUE(repo_ready);
+ ASSERT_TRUE(environment_saved);
+ ASSERT_TRUE(environment_ready);
+ ASSERT_TRUE(cwd_ready);
+ ASSERT_TRUE(config_ready);
+ ASSERT_TRUE(server_ready);
+ ASSERT_TRUE(response_ready);
+ ASSERT_EQ(selected_workers, cbm_default_worker_count(false));
+ ASSERT_TRUE(cleaned);
+ PASS();
+}
+
/* ══════════════════════════════════════════════════════════════════
* #853 — auto_watch=false must ALSO gate the SUPERVISED fresh-index
* watcher registration (keystone × #849 merge interaction)
@@ -21361,6 +21428,7 @@ SUITE(mcp) {
RUN_TEST(autoindex_limit_guards_non_git_root_issue713);
RUN_TEST(autoindex_limit_admits_non_git_root_under_limit_issue713);
RUN_TEST(autoindex_limit_guards_git_root_issue713);
+ RUN_TEST(mcp_auto_index_in_process_uses_background_worker_policy);
}
/* Kept separate so daemon-coordination regressions can be iterated without
diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c
index 56be6b88e..32f4b1a4c 100644
--- a/tests/test_pipeline.c
+++ b/tests/test_pipeline.c
@@ -117,6 +117,84 @@ TEST(pipeline_create_free) {
PASS();
}
+TEST(pipeline_background_worker_policy_preserves_overrides) {
+ const char *old_workers = getenv("CBM_WORKERS");
+ char *saved_workers = old_workers ? strdup(old_workers) : NULL;
+ const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD");
+ char *saved_single = old_single ? strdup(old_single) : NULL;
+ cbm_unsetenv("CBM_WORKERS");
+ cbm_unsetenv("CBM_INDEX_SINGLE_THREAD");
+
+ cbm_pipeline_t *foreground = cbm_pipeline_new("/some/path", NULL, CBM_MODE_FULL);
+ cbm_pipeline_t *background = cbm_pipeline_new("/some/path", NULL, CBM_MODE_FULL);
+ ASSERT_NOT_NULL(foreground);
+ ASSERT_NOT_NULL(background);
+ cbm_pipeline_set_background(background, true);
+
+ int foreground_default = cbm_pipeline_worker_count(foreground);
+ int background_default = cbm_pipeline_worker_count(background);
+ bool defaults_match = foreground_default == cbm_default_worker_count(true) &&
+ background_default == cbm_default_worker_count(false) &&
+ foreground_default >= 1 && background_default >= 1;
+
+ cbm_setenv("CBM_WORKERS", "3", 1);
+ bool env_override = cbm_pipeline_worker_count(foreground) == 3 &&
+ cbm_pipeline_worker_count(background) == 3;
+
+ cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1);
+ bool recovery_override = cbm_pipeline_worker_count(foreground) == 1 &&
+ cbm_pipeline_worker_count(background) == 1;
+
+ cbm_pipeline_free(foreground);
+ cbm_pipeline_free(background);
+ saved_workers ? cbm_setenv("CBM_WORKERS", saved_workers, 1) : cbm_unsetenv("CBM_WORKERS");
+ saved_single ? cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1)
+ : cbm_unsetenv("CBM_INDEX_SINGLE_THREAD");
+ free(saved_workers);
+ free(saved_single);
+
+ ASSERT_TRUE(defaults_match);
+ ASSERT_TRUE(env_override);
+ ASSERT_TRUE(recovery_override);
+ PASS();
+}
+
+TEST(pipeline_background_policy_preserves_index_results) {
+ ASSERT_EQ(setup_test_repo(), 0);
+ char foreground_db[512];
+ char background_db[512];
+ (void)snprintf(foreground_db, sizeof(foreground_db), "%s/foreground.db", g_tmpdir);
+ (void)snprintf(background_db, sizeof(background_db), "%s/background.db", g_tmpdir);
+
+ cbm_pipeline_t *foreground =
+ cbm_pipeline_new(g_tmpdir, foreground_db, CBM_MODE_FULL);
+ cbm_pipeline_t *background =
+ cbm_pipeline_new(g_tmpdir, background_db, CBM_MODE_FULL);
+ ASSERT_NOT_NULL(foreground);
+ ASSERT_NOT_NULL(background);
+ cbm_pipeline_set_background(background, true);
+
+ int foreground_rc = cbm_pipeline_run(foreground);
+ int background_rc = cbm_pipeline_run(background);
+ int foreground_nodes = -1;
+ int foreground_edges = -1;
+ int background_nodes = -1;
+ int background_edges = -1;
+ cbm_pipeline_get_committed_counts(foreground, &foreground_nodes, &foreground_edges);
+ cbm_pipeline_get_committed_counts(background, &background_nodes, &background_edges);
+
+ cbm_pipeline_free(foreground);
+ cbm_pipeline_free(background);
+ teardown_test_repo();
+
+ ASSERT_EQ(foreground_rc, 0);
+ ASSERT_EQ(background_rc, 0);
+ ASSERT_GT(foreground_nodes, 0);
+ ASSERT_EQ(background_nodes, foreground_nodes);
+ ASSERT_EQ(background_edges, foreground_edges);
+ PASS();
+}
+
TEST(pipeline_null_repo) {
cbm_pipeline_t *p = cbm_pipeline_new(NULL, NULL, CBM_MODE_FULL);
ASSERT_NULL(p);
@@ -16284,6 +16362,8 @@ SUITE(pipeline) {
RUN_TEST(pipeline_lock_release_allows_contender);
/* Lifecycle */
RUN_TEST(pipeline_create_free);
+ RUN_TEST(pipeline_background_worker_policy_preserves_overrides);
+ RUN_TEST(pipeline_background_policy_preserves_index_results);
RUN_TEST(pipeline_null_repo);
RUN_TEST(pipeline_free_null);
RUN_TEST(pipeline_cancel);