Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix: preserve CPU headroom during background indexing
Matt Van Horn2443e92bece2
diff --git a/src/daemon/application.c b/src/daemon/application.cindex 22c62029b..91b5937ec 100644--- a/src/daemon/application.c+++ b/src/daemon/application.c@@ -2199,10 +2199,10 @@ static char *application_auto_index_args(cbm_daemon_application_t *application,return NULL;}yyjson_mut_doc_set_root(document, root);- char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) &&- application_index_args_add_policy(application, document, root)- ? yyjson_mut_write(document, 0, NULL)- : NULL;+ bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) &&+ application_index_args_add_policy(application, document, root) &&+ yyjson_mut_obj_add_bool(document, root, "_background", true);+ char *args = encoded ? yyjson_mut_write(document, 0, NULL) : NULL;yyjson_mut_doc_free(document);return args;}@@ -3894,7 +3894,8 @@ static int application_background_index(cbm_daemon_application_t *application,}yyjson_mut_doc_set_root(document, root);bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) &&- application_index_args_add_policy(application, document, root);+ application_index_args_add_policy(application, document, root) &&+ yyjson_mut_obj_add_bool(document, root, "_background", true);char *default_project = cbm_project_name_from_path(canonical_root);bool custom_project =project_name[0] && (!default_project || strcmp(default_project, project_name) != 0);diff --git a/src/mcp/mcp.c b/src/mcp/mcp.cindex c877d5110..0d15ac017 100644--- a/src/mcp/mcp.c+++ b/src/mcp/mcp.c@@ -11109,7 +11109,8 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_pyyjson_mut_val *root = yyjson_mut_obj(doc);yyjson_mut_doc_set_root(doc, root);if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) ||- !cbm_mcp_index_policy_add_to_args(doc, root, &policy)) {+ !cbm_mcp_index_policy_add_to_args(doc, root, &policy) ||+ !yyjson_mut_obj_add_bool(doc, root, "_background", true)) {yyjson_mut_doc_free(doc);return NULL;}@@ -11498,6 +11499,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {char *repo_path = cbm_mcp_get_string_arg(args, "repo_path");char *mode_str = cbm_mcp_get_string_arg(args, "mode");char *name_override = cbm_mcp_get_string_arg(args, "name");+ bool background = cbm_mcp_get_bool_arg(args, "_background");cbm_normalize_path_sep(repo_path);if (!repo_path) {@@ -11660,6 +11662,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {free(repo_path);return cbm_mcp_text_result("failed to create pipeline", true);}+ cbm_pipeline_set_background(p, background);if (name_override && name_override[0] && !cbm_pipeline_set_project_name(p, name_override)) {cbm_pipeline_free(p);mcp_project_mutation_end(srv, mutation_project);@@ -18106,6 +18109,7 @@ static void *autoindex_thread(void *arg) {cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed");return NULL;}+ cbm_pipeline_set_background(p, true);/* Block until any concurrent pipeline finishes */cbm_pipeline_lock();diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.cindex ba41b877c..f2aeb47be 100644--- a/src/pipeline/pipeline.c+++ b/src/pipeline/pipeline.c@@ -71,6 +71,18 @@ static inline void *intptr_to_ptr(intptr_t v) {* Atomic spinlock: 0 = free, 1 = locked. */static atomic_int g_pipeline_busy = 0;+#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API+static atomic_int g_last_worker_count = 0;++void cbm_pipeline_worker_count_test_reset(void) {+ atomic_store(&g_last_worker_count, 0);+}++int cbm_pipeline_worker_count_test_last(void) {+ return atomic_load(&g_last_worker_count);+}+#endif+#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_APIstatic atomic_bool g_persist_test_fail_after_stage_dump = false;static atomic_bool g_persist_test_cancel_after_predump = false;@@ -192,6 +204,7 @@ struct cbm_pipeline {bool persistence; /* write .codebase-memory/graph.db.zst after indexing */cbm_index_resource_policy_t resource_policy;cbm_index_resource_violation_t resource_violation;+ bool background; /* reserve CPU headroom when no user is waiting *//* Snapshot of the artifact export failure of THIS run (set only by* export_after_publish failure, zeroed at run start, cleared on success).@@ -400,6 +413,12 @@ const char *cbm_pipeline_export_error(const cbm_pipeline_t *p) {return p ? p->export_error : "";}+void cbm_pipeline_set_background(cbm_pipeline_t *p, bool background) {+ if (p) {+ p->background = background;+ }+}+bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) {if (!p || !name || !name[0]) {return false;@@ -613,12 +632,18 @@ void cbm_pipeline_set_committed_counts(cbm_pipeline_t *p, int nodes, int edges)* crasher; a parallel re-run would race the marker. Honour that override* everywhere the worker count drives the parallel/sequential decision, so the* whole extraction phase collapses to the deterministic sequential path. */-static int effective_worker_count(bool initial) {+int cbm_pipeline_worker_count(const cbm_pipeline_t *p) {const char *st = getenv("CBM_INDEX_SINGLE_THREAD");+ int workers;if (st && st[0] == '1') {- return 1;+ workers = 1;+ } else {+ workers = cbm_default_worker_count(!p || !p->background);}- return cbm_default_worker_count(initial);+#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API+ atomic_store(&g_last_worker_count, workers);+#endif+ return workers;}/* Resolve the DB path for this pipeline. Caller must free(). */@@ -2590,7 +2615,7 @@ static int run_githistory(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) {gh_compute_arg_t gh_arg = {.repo_path = ctx->repo_path, .result = &gh_result};if (p->mode != CBM_MODE_FAST) {- if (effective_worker_count(true) > SKIP_ONE) {+ if (cbm_pipeline_worker_count(p) > SKIP_ONE) {if (cbm_thread_create(&gh_thread, 0, gh_compute_thread_fn, &gh_arg) == 0) {gh_threaded = true;}@@ -2690,7 +2715,7 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,return CBM_NOT_FOUND;}- int worker_count = effective_worker_count(true);+ int worker_count = cbm_pipeline_worker_count(p);CBM_PROF_START(t_extract_total);int rc = (worker_count > SKIP_ONE && file_count > MIN_FILES_FOR_PARALLEL)? run_parallel_pipeline(p, ctx, files, file_count, worker_count, &t)diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.hindex c1d571ecd..e52513673 100644--- a/src/pipeline/pipeline.h+++ b/src/pipeline/pipeline.h@@ -59,6 +59,18 @@ void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resourc/* Copy the exact discovery violation from the most recent run. */void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p,cbm_index_resource_violation_t *violation);+/* Mark work that runs without a waiting user. Background pipelines reserve+ * CPU headroom; foreground pipelines retain the initial-index all-core policy. */+void cbm_pipeline_set_background(cbm_pipeline_t *p, bool background);++/* Resolve the worker policy for this pipeline, including environment and+ * crash-recovery overrides. */+int cbm_pipeline_worker_count(const cbm_pipeline_t *p);++#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API+void cbm_pipeline_worker_count_test_reset(void);+int cbm_pipeline_worker_count_test_last(void);+#endif/* Snapshot of the artifact export failure of the last cbm_pipeline_run, or ""* when the run succeeded / did not reach post-publish export. Used todiff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.cindex 154518658..4e7d61dc3 100644--- a/src/pipeline/pipeline_incremental.c+++ b/src/pipeline/pipeline_incremental.c@@ -1212,7 +1212,7 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed* a full build takes, which is what makes its output converge. */#define MIN_FILES_FOR_PARALLEL_INCR 50- int worker_count = cbm_default_worker_count(true);+ int worker_count = cbm_pipeline_worker_count(ctx->pipeline);bool use_parallel =closure != NULL || (worker_count > SKIP_ONE && ci > MIN_FILES_FOR_PARALLEL_INCR);@@ -1590,7 +1590,7 @@ static int closure_probe_surfaces(cbm_pipeline_t *p, const char *project,_Atomic int64_t probe_ids;atomic_init(&probe_ids, cbm_gbuf_next_id(probe_gbuf));rc = cbm_parallel_extract(&probe_ctx, probe_files, probe_count, cache, &probe_ids,- cbm_default_worker_count(true));+ cbm_pipeline_worker_count(p));}if (rc == 0) {char **def_modules = (char **)calloc((size_t)probe_count, sizeof(char *));@@ -2061,7 +2061,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const charelig[elig_count++] = row;}}- int workers = cbm_default_worker_count(true);+ int workers = cbm_pipeline_worker_count(p);if (workers < 1) {workers = 1;}diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.cindex 7f4c9cc28..50abe5a8c 100644--- a/tests/test_daemon_application.c+++ b/tests/test_daemon_application.c@@ -1375,6 +1375,7 @@ typedef struct {* itself instead, which is the edge a reader on another thread needs. */atomic_bool args_captured[APP_FAKE_MAX_ATTEMPTS];size_t memory_budgets[APP_FAKE_MAX_ATTEMPTS];+ bool background_requests[APP_FAKE_MAX_ATTEMPTS];} app_fake_worker_context_t;typedef struct {@@ -1442,6 +1443,8 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t memargs_json ? args_json : "");atomic_store(&context->args_captured[worker->attempt], true);context->memory_budgets[worker->attempt] = memory_budget_bytes;+ context->background_requests[worker->attempt] =+ cbm_mcp_get_bool_arg(args_json, "_background");if (marker_file) {(void)snprintf(context->marker_paths[worker->attempt], APP_TEST_PATH_CAP, "%s",marker_file);@@ -2202,6 +2205,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) {ASSERT_TRUE(first_initialized);ASSERT_TRUE(first_owned);ASSERT_TRUE(auto_policy_propagated);+ ASSERT_TRUE(fake.background_requests[0]);ASSERT_TRUE(second_initialized);ASSERT_TRUE(coalesced);ASSERT_TRUE(restricted_disconnect_kept_job);@@ -3634,6 +3638,7 @@ TEST(daemon_application_request_cancel_preserves_persistent_watch_and_session) {ASSERT_TRUE(thread_started);ASSERT_TRUE(subscribed);ASSERT_TRUE(worker_started);+ ASSERT_FALSE(fixture.fake.background_requests[0]);ASSERT_TRUE(request_returned);ASSERT_TRUE(thread_joined);ASSERT_EQ(request.status, CBM_DAEMON_RUNTIME_APPLICATION_CANCELLED);@@ -3931,6 +3936,7 @@ TEST(daemon_application_final_cancel_drains_admitted_watcher_job) {ASSERT_TRUE(fixture_ready);ASSERT_TRUE(thread_started);ASSERT_TRUE(worker_started);+ ASSERT_TRUE(fixture.fake.background_requests[0]);ASSERT_TRUE(job_active);ASSERT_EQ(watches_after_cancel, 0);ASSERT_TRUE(thread_joined);diff --git a/tests/test_mcp.c b/tests/test_mcp.cindex 22af972af..34563e0c9 100644--- a/tests/test_mcp.c+++ b/tests/test_mcp.c@@ -20006,6 +20006,73 @@ TEST(autoindex_limit_guards_git_root_issue713) {PASS();}+TEST(mcp_auto_index_in_process_uses_background_worker_policy) {+ char cache[256];+ char repo[512];+ (void)snprintf(cache, sizeof(cache), "%s/cbm-autoindex-workers-XXXXXX", cbm_tmpdir());+ bool cache_ready = cbm_mkdtemp(cache) != NULL;+ (void)snprintf(repo, sizeof(repo), "%s/repo", cache);+ char source[640];+ (void)snprintf(source, sizeof(source), "%s/main.py", repo);+ bool repo_ready = cache_ready && th_mkdir_p(repo) == 0 &&+ th_write_file(source, "def background_index():\n return True\n") == 0;++ mcp_test_env_backup_t environment[] = {+ {.name = "CBM_CACHE_DIR"},+ {.name = "CBM_WORKERS"},+ {.name = "CBM_INDEX_SINGLE_THREAD"},+ };+ bool environment_saved = true;+ for (size_t i = 0; i < sizeof(environment) / sizeof(environment[0]); i++) {+ const char *value = getenv(environment[i].name);+ environment[i].present = value != NULL;+ environment[i].value = value ? strdup(value) : NULL;+ environment_saved = environment_saved && (!value || environment[i].value);+ }+ bool environment_ready =+ environment_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0 &&+ cbm_unsetenv("CBM_WORKERS") == 0 && cbm_unsetenv("CBM_INDEX_SINGLE_THREAD") == 0;++ char old_cwd[CBM_SZ_4K] = {0};+ bool cwd_ready = repo_ready && environment_ready && cbm_getcwd(old_cwd, sizeof(old_cwd)) &&+ cbm_chdir(repo) == 0;+ cbm_config_t *config = cwd_ready ? cbm_config_open(cache) : NULL;+ bool config_ready = config && cbm_config_set(config, CBM_CONFIG_AUTO_INDEX, "true") == 0 &&+ cbm_config_set(config, CBM_CONFIG_AUTO_WATCH, "false") == 0;+ cbm_pipeline_worker_count_test_reset();+ cbm_mcp_server_t *server = config_ready ? cbm_mcp_server_new(NULL) : NULL;+ char *response = NULL;+ if (server) {+ cbm_mcp_server_set_config(server, config);+ response = cbm_mcp_server_handle(+ server, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{}}");+ cbm_mcp_server_free(server); /* joins the automatic index thread */+ }+ int selected_workers = cbm_pipeline_worker_count_test_last();+ bool server_ready = server != NULL;+ bool response_ready = response != NULL;++ free(response);+ cbm_config_close(config);+ if (cwd_ready) {+ (void)cbm_chdir(old_cwd);+ }+ mcp_test_restore_env(environment, sizeof(environment) / sizeof(environment[0]));+ bool cleaned = !cache_ready || th_rmtree(cache) == 0;++ ASSERT_TRUE(cache_ready);+ ASSERT_TRUE(repo_ready);+ ASSERT_TRUE(environment_saved);+ ASSERT_TRUE(environment_ready);+ ASSERT_TRUE(cwd_ready);+ ASSERT_TRUE(config_ready);+ ASSERT_TRUE(server_ready);+ ASSERT_TRUE(response_ready);+ ASSERT_EQ(selected_workers, cbm_default_worker_count(false));+ ASSERT_TRUE(cleaned);+ PASS();+}+/* ══════════════════════════════════════════════════════════════════* #853 — auto_watch=false must ALSO gate the SUPERVISED fresh-index* watcher registration (keystone × #849 merge interaction)@@ -21361,6 +21428,7 @@ SUITE(mcp) {RUN_TEST(autoindex_limit_guards_non_git_root_issue713);RUN_TEST(autoindex_limit_admits_non_git_root_under_limit_issue713);RUN_TEST(autoindex_limit_guards_git_root_issue713);+ RUN_TEST(mcp_auto_index_in_process_uses_background_worker_policy);}/* Kept separate so daemon-coordination regressions can be iterated withoutdiff --git a/tests/test_pipeline.c b/tests/test_pipeline.cindex 56be6b88e..32f4b1a4c 100644--- a/tests/test_pipeline.c+++ b/tests/test_pipeline.c@@ -117,6 +117,84 @@ TEST(pipeline_create_free) {PASS();}+TEST(pipeline_background_worker_policy_preserves_overrides) {+ const char *old_workers = getenv("CBM_WORKERS");+ char *saved_workers = old_workers ? strdup(old_workers) : NULL;+ const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD");+ char *saved_single = old_single ? strdup(old_single) : NULL;+ cbm_unsetenv("CBM_WORKERS");+ cbm_unsetenv("CBM_INDEX_SINGLE_THREAD");++ cbm_pipeline_t *foreground = cbm_pipeline_new("/some/path", NULL, CBM_MODE_FULL);+ cbm_pipeline_t *background = cbm_pipeline_new("/some/path", NULL, CBM_MODE_FULL);+ ASSERT_NOT_NULL(foreground);+ ASSERT_NOT_NULL(background);+ cbm_pipeline_set_background(background, true);++ int foreground_default = cbm_pipeline_worker_count(foreground);+ int background_default = cbm_pipeline_worker_count(background);+ bool defaults_match = foreground_default == cbm_default_worker_count(true) &&+ background_default == cbm_default_worker_count(false) &&+ foreground_default >= 1 && background_default >= 1;++ cbm_setenv("CBM_WORKERS", "3", 1);+ bool env_override = cbm_pipeline_worker_count(foreground) == 3 &&+ cbm_pipeline_worker_count(background) == 3;++ cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1);+ bool recovery_override = cbm_pipeline_worker_count(foreground) == 1 &&+ cbm_pipeline_worker_count(background) == 1;++ cbm_pipeline_free(foreground);+ cbm_pipeline_free(background);+ saved_workers ? cbm_setenv("CBM_WORKERS", saved_workers, 1) : cbm_unsetenv("CBM_WORKERS");+ saved_single ? cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1)+ : cbm_unsetenv("CBM_INDEX_SINGLE_THREAD");+ free(saved_workers);+ free(saved_single);++ ASSERT_TRUE(defaults_match);+ ASSERT_TRUE(env_override);+ ASSERT_TRUE(recovery_override);+ PASS();+}++TEST(pipeline_background_policy_preserves_index_results) {+ ASSERT_EQ(setup_test_repo(), 0);+ char foreground_db[512];+ char background_db[512];+ (void)snprintf(foreground_db, sizeof(foreground_db), "%s/foreground.db", g_tmpdir);+ (void)snprintf(background_db, sizeof(background_db), "%s/background.db", g_tmpdir);++ cbm_pipeline_t *foreground =+ cbm_pipeline_new(g_tmpdir, foreground_db, CBM_MODE_FULL);+ cbm_pipeline_t *background =+ cbm_pipeline_new(g_tmpdir, background_db, CBM_MODE_FULL);+ ASSERT_NOT_NULL(foreground);+ ASSERT_NOT_NULL(background);+ cbm_pipeline_set_background(background, true);++ int foreground_rc = cbm_pipeline_run(foreground);+ int background_rc = cbm_pipeline_run(background);+ int foreground_nodes = -1;+ int foreground_edges = -1;+ int background_nodes = -1;+ int background_edges = -1;+ cbm_pipeline_get_committed_counts(foreground, &foreground_nodes, &foreground_edges);+ cbm_pipeline_get_committed_counts(background, &background_nodes, &background_edges);++ cbm_pipeline_free(foreground);+ cbm_pipeline_free(background);+ teardown_test_repo();++ ASSERT_EQ(foreground_rc, 0);+ ASSERT_EQ(background_rc, 0);+ ASSERT_GT(foreground_nodes, 0);+ ASSERT_EQ(background_nodes, foreground_nodes);+ ASSERT_EQ(background_edges, foreground_edges);+ PASS();+}+TEST(pipeline_null_repo) {cbm_pipeline_t *p = cbm_pipeline_new(NULL, NULL, CBM_MODE_FULL);ASSERT_NULL(p);@@ -16284,6 +16362,8 @@ SUITE(pipeline) {RUN_TEST(pipeline_lock_release_allows_contender);/* Lifecycle */RUN_TEST(pipeline_create_free);+ RUN_TEST(pipeline_background_worker_policy_preserves_overrides);+ RUN_TEST(pipeline_background_policy_preserves_index_results);RUN_TEST(pipeline_null_repo);RUN_TEST(pipeline_free_null);RUN_TEST(pipeline_cancel);