Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(ingestion): bound the parse pool instead of sizing it from the host's cores (#1410)

Raghav Chamadiya5244ca4a4f6d
diff --git a/docs/reference/CONFIG.md b/docs/reference/CONFIG.md
index 0d07f325a..462223016 100644
--- a/docs/reference/CONFIG.md
+++ b/docs/reference/CONFIG.md
@@ -669,6 +669,7 @@ Anonymous usage telemetry is **enabled by default** (opt-out).
| `REPOWISE_GIT_WINDOW_ANCHOR` | Set to `head` to anchor git "now" to the latest commit instead of wall-clock time |
| `REPOWISE_SKIP_EDITOR_SETUP` | Truthy value stops `init` writing to your machine-wide editor config: the Claude Code / Claude Desktop MCP entry, the Claude Code hooks, and the distill rewrite-hook offer. Same switch as `init --no-editor-setup` ([CLI_REFERENCE.md](CLI_REFERENCE.md#repowise-init-path)); the env var is the one to use for CI, sandboxes, and benchmark runs that index many repos. Project-local files (`.repowise/mcp.json`, `CLAUDE.md`, Codex config) are written either way |
| `REPOWISE_CHANGELOG` | Override the changelog source used by the "what's new" check |
+| `REPOWISE_PARSE_WORKERS` | How many processes parse files during indexing. Defaults to your CPU count capped at 8, and never exceeds the number of files to parse. Each worker is a separate interpreter holding roughly 50 MB, so lower it on a memory-constrained machine; raising it above 8 is not measurably faster |
---
diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
index a917fce02..17fbde11f 100644
--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py
+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
@@ -153,10 +153,69 @@ def _emit_traversal_summary(
# its picklable args are known to work under it.
_MP_SPAWN = multiprocessing.get_context("spawn")
+# Upper bound on the parse pool, independent of how many cores the host has.
+#
+# Every worker is a fresh interpreter under ``spawn`` that imports the repowise
+# stack and lazily builds an ``ASTParser`` holding compiled tree-sitter
+# Language/Query objects. That costs a flat ~50 MB of *private* memory per
+# worker — measured at 51.0 MB/worker on PowerToys (5,766 files, 29.5 MB of
+# source) and 49.2 MB/worker on hugo (2,130 files, 8.8 MB), so it is a
+# per-worker constant and not a function of repo size. Sizing the pool from
+# ``cpu_count`` therefore made peak memory a function of the *host*: 32 workers
+# held 1.57 GB where 8 hold 0.46 GB.
+#
+# Nothing was bought with it. With 16 cores genuinely free, 8 workers parsed
+# PowerToys in 5.54s/4.75s against 16 workers' 6.79s/5.23s (interleaved pairs):
+# past this point the per-worker import and grammar build is paid again without
+# amortizing over enough files to repay it, so more workers is slightly slower
+# *and* multiplies the memory. The same reasoning caps the betweenness pool
+# (see ``ingestion/graph/_betweenness.py``).
+_MAX_PARSE_WORKERS = 8
+
+# Escape hatch for hosts this default judges wrong in either direction.
+_PARSE_WORKERS_ENV = "REPOWISE_PARSE_WORKERS"
+
# Module-level process-local parser cache (one per worker process).
_WORKER_PARSER: Any = None
+def parse_pool_workers(pending: int) -> int:
+ """How many workers to ask the parse pool for, to parse *pending* files.
+
+ Stated once so the init and resume paths cannot drift apart, and so the
+ bound can be asserted on directly. Never exceeds *pending* — a repo with
+ three changed files has no use for eight interpreters.
+
+ ``REPOWISE_PARSE_WORKERS`` overrides the cap in both directions. A set but
+ non-positive value is ignored with a warning rather than failing an
+ otherwise healthy parse; an empty value means "unset", which is what an
+ ``export REPOWISE_PARSE_WORKERS=`` from an undefined shell variable
+ produces, so it is not worth warning about.
+ """
+ if pending <= 0:
+ return 1
+
+ override = os.environ.get(_PARSE_WORKERS_ENV)
+ if override:
+ try:
+ requested = int(override)
+ except ValueError:
+ requested = 0
+ if requested > 0:
+ return min(requested, pending)
+ logger.warning("invalid_parse_workers_env", value=override)
+
+ # ``process_cpu_count`` (3.13+) honours the CPU affinity mask, so a process
+ # pinned to fewer CPUs than the host has stops sizing its pool from the
+ # host's core count. It does *not* read cgroup CPU quotas: a container run
+ # with ``--cpus=2`` (CFS quota, no cpuset) still sees the full host mask,
+ # and the cap below is what bounds that case. Falls back to ``cpu_count``
+ # on 3.11/3.12, and when affinity is unavailable and it returns None.
+ affinity_aware = getattr(os, "process_cpu_count", None)
+ cpus = (affinity_aware() if affinity_aware else None) or os.cpu_count() or 4
+ return max(1, min(cpus, _MAX_PARSE_WORKERS, pending))
+
+
def _parse_one(path_and_fi_and_bytes: tuple) -> Any:
"""Worker function for ProcessPoolExecutor parsing.
@@ -369,7 +428,7 @@ async def _run_ingestion(
# entirely (an incremental update re-ingests the whole repo for one
# changed file). Misses parse exactly as before.
parse_cache, cached_hits, to_parse = _split_cached(repo_path, fi_and_bytes, progress)
- workers = max(1, min(os.cpu_count() or 4, len(to_parse) or 1))
+ workers = parse_pool_workers(len(to_parse))
parse_results: list[Any] = []
@@ -636,7 +695,7 @@ async def reparse_for_resume(
loop = asyncio.get_running_loop()
parse_cache, cached_hits, to_parse = _split_cached(repo_path, fi_and_bytes, progress)
- workers = max(1, min(os.cpu_count() or 4, len(to_parse) or 1))
+ workers = parse_pool_workers(len(to_parse))
parse_results: list[Any] = []
if to_parse:
diff --git a/tests/unit/pipeline/test_parse_pool_workers.py b/tests/unit/pipeline/test_parse_pool_workers.py
new file mode 100644
index 000000000..622c2711b
--- /dev/null
+++ b/tests/unit/pipeline/test_parse_pool_workers.py
@@ -0,0 +1,172 @@
+"""The parse pool's worker count is bounded (issue #1394).
+
+Every worker is a fresh spawned interpreter holding ~50 MB of private memory,
+so sizing the pool from the host's core count made peak memory a function of
+the machine rather than the repo.
+"""
+
+import pytest
+
+from repowise.core.pipeline.phases import ingestion as ing
+
+
+class TestParsePoolWorkers:
+ """The cap, the floor, and the env override."""
+
+ def test_the_cap_is_eight(self):
+ """Pin the literal.
+
+ Every other assertion here compares the helper's output to
+ ``_MAX_PARSE_WORKERS``, which a revert of the constant would satisfy
+ while reintroducing the 32-worker fleet this exists to prevent.
+ """
+ assert ing._MAX_PARSE_WORKERS == 8
+
+ def test_cap_applies_to_host_core_count(self, monkeypatch):
+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)
+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)
+ assert ing.parse_pool_workers(5000) == 8
+
+ def test_below_cap_is_left_alone(self, monkeypatch):
+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 4)
+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 4, raising=False)
+ assert ing.parse_pool_workers(5000) == 4
+
+ def test_never_exceeds_the_files_to_parse(self, monkeypatch):
+ """A three-file update has no use for eight interpreters."""
+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)
+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)
+ assert ing.parse_pool_workers(3) == 3
+
+ def test_empty_work_list_degrades_to_one(self):
+ assert ing.parse_pool_workers(0) == 1
+ assert ing.parse_pool_workers(-1) == 1
+
+ def test_unknown_core_count_degrades_to_a_default(self, monkeypatch):
+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
+ monkeypatch.setattr(ing.os, "cpu_count", lambda: None)
+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: None, raising=False)
+ assert ing.parse_pool_workers(5000) == 4
+
+ def test_affinity_aware_count_wins_over_host_count(self, monkeypatch):
+ """A container pinned to 2 CPUs must not size its pool from the host."""
+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)
+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 2, raising=False)
+ assert ing.parse_pool_workers(5000) == 2
+
+ def test_missing_process_cpu_count_falls_back(self, monkeypatch):
+ """Python 3.11/3.12 have no ``process_cpu_count``."""
+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
+ monkeypatch.delattr(ing.os, "process_cpu_count", raising=False)
+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 3)
+ assert ing.parse_pool_workers(5000) == 3
+
+ @pytest.mark.parametrize("value,expected", [("2", 2), ("16", 16), ("1", 1)])
+ def test_env_override_wins_in_both_directions(self, monkeypatch, value, expected):
+ monkeypatch.setenv(ing._PARSE_WORKERS_ENV, value)
+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)
+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)
+ assert ing.parse_pool_workers(5000) == expected
+
+ def test_env_override_still_bounded_by_pending_files(self, monkeypatch):
+ monkeypatch.setenv(ing._PARSE_WORKERS_ENV, "64")
+ assert ing.parse_pool_workers(5) == 5
+
+ @pytest.mark.parametrize("value", ["nonsense", "0", "-4", "3.5"])
+ def test_unusable_env_value_is_ignored_not_fatal(self, monkeypatch, value):
+ """A bad env value must never fail an otherwise healthy parse."""
+ monkeypatch.setenv(ing._PARSE_WORKERS_ENV, value)
+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)
+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)
+ assert ing.parse_pool_workers(5000) == 8
+
+ def test_unusable_env_value_says_so(self, monkeypatch):
+ """The warning is the only feedback a misconfigured user gets."""
+ monkeypatch.setenv(ing._PARSE_WORKERS_ENV, "-4")
+ warned: list[tuple] = []
+ monkeypatch.setattr(
+ ing.logger, "warning", lambda event, **kw: warned.append((event, kw))
+ )
+ ing.parse_pool_workers(5000)
+ assert warned and warned[0][0] == "invalid_parse_workers_env"
+ assert warned[0][1]["value"] == "-4"
+
+ def test_empty_env_value_means_unset(self, monkeypatch):
+ """``export REPOWISE_PARSE_WORKERS=`` from an undefined shell variable
+ is not a misconfiguration worth warning about."""
+ monkeypatch.setenv(ing._PARSE_WORKERS_ENV, "")
+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)
+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)
+ warned: list[tuple] = []
+ monkeypatch.setattr(
+ ing.logger, "warning", lambda event, **kw: warned.append((event, kw))
+ )
+ assert ing.parse_pool_workers(5000) == 8
+ assert not warned
+
+ def test_affinity_unavailable_falls_back_to_host_count(self, monkeypatch):
+ """``process_cpu_count`` exists but returns None when there is no mask."""
+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: None, raising=False)
+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 3)
+ assert ing.parse_pool_workers(5000) == 3
+
+
+class TestBothCallSitesAreBounded:
+ """The bound has to reach the executor on the init *and* resume paths.
+
+ Grepping the source is what this guards: the two paths carried a duplicated
+ ``min(os.cpu_count(), ...)`` expression and only one would have been noticed
+ if a future edit reverted it.
+ """
+
+ @pytest.mark.parametrize("func", ["_run_ingestion", "reparse_for_resume"])
+ def test_call_site_uses_the_shared_helper(self, func):
+ import inspect
+
+ target = getattr(ing, func)
+ src = inspect.getsource(target)
+ assert "parse_pool_workers(" in src, f"{func} must size its pool via the shared helper"
+ assert "os.cpu_count()" not in src, f"{func} must not size its pool from the host core count"
+
+ async def test_the_bound_reaches_the_real_executor(self, tmp_path, monkeypatch):
+ """Record what ``ProcessPoolExecutor`` is really constructed with.
+
+ The source-text assertions above cannot catch a regression spelled a
+ different way; this pins the argument itself on a real ingestion run.
+ """
+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 64)
+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 64, raising=False)
+
+ for i in range(12):
+ (tmp_path / f"mod_{i}.py").write_text(f"def f_{i}(x):\n return x + {i}\n")
+
+ seen: list[int | None] = []
+ real = ing.ProcessPoolExecutor
+
+ def recording(*args, **kwargs):
+ seen.append(kwargs.get("max_workers"))
+ # Honour the bound under test but keep the pool tiny: this is a
+ # unit test, not a benchmark, and spawning eight interpreters to
+ # parse twelve files would dominate the suite's runtime.
+ kwargs["max_workers"] = 1
+ return real(*args, **kwargs)
+
+ monkeypatch.setattr(ing, "ProcessPoolExecutor", recording)
+
+ parsed, *_ = await ing._run_ingestion(
+ tmp_path,
+ exclude_patterns=None,
+ skip_tests=False,
+ skip_infra=False,
+ progress=None,
+ )
+
+ assert parsed, "ingestion parsed nothing, so the run proves nothing"
+ assert seen, "no process pool was constructed, so the bound was never exercised"
+ assert seen == [8], f"expected the capped count to reach the executor, got {seen}"