Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(ingestion): bound the parse pool instead of sizing it from the host's cores (#1410)
Raghav Chamadiya5244ca4a4f6d
diff --git a/docs/reference/CONFIG.md b/docs/reference/CONFIG.mdindex 0d07f325a..462223016 100644--- a/docs/reference/CONFIG.md+++ b/docs/reference/CONFIG.md@@ -669,6 +669,7 @@ Anonymous usage telemetry is **enabled by default** (opt-out).| `REPOWISE_GIT_WINDOW_ANCHOR` | Set to `head` to anchor git "now" to the latest commit instead of wall-clock time || `REPOWISE_SKIP_EDITOR_SETUP` | Truthy value stops `init` writing to your machine-wide editor config: the Claude Code / Claude Desktop MCP entry, the Claude Code hooks, and the distill rewrite-hook offer. Same switch as `init --no-editor-setup` ([CLI_REFERENCE.md](CLI_REFERENCE.md#repowise-init-path)); the env var is the one to use for CI, sandboxes, and benchmark runs that index many repos. Project-local files (`.repowise/mcp.json`, `CLAUDE.md`, Codex config) are written either way || `REPOWISE_CHANGELOG` | Override the changelog source used by the "what's new" check |+| `REPOWISE_PARSE_WORKERS` | How many processes parse files during indexing. Defaults to your CPU count capped at 8, and never exceeds the number of files to parse. Each worker is a separate interpreter holding roughly 50 MB, so lower it on a memory-constrained machine; raising it above 8 is not measurably faster |---diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.pyindex a917fce02..17fbde11f 100644--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py@@ -153,10 +153,69 @@ def _emit_traversal_summary(# its picklable args are known to work under it._MP_SPAWN = multiprocessing.get_context("spawn")+# Upper bound on the parse pool, independent of how many cores the host has.+#+# Every worker is a fresh interpreter under ``spawn`` that imports the repowise+# stack and lazily builds an ``ASTParser`` holding compiled tree-sitter+# Language/Query objects. That costs a flat ~50 MB of *private* memory per+# worker — measured at 51.0 MB/worker on PowerToys (5,766 files, 29.5 MB of+# source) and 49.2 MB/worker on hugo (2,130 files, 8.8 MB), so it is a+# per-worker constant and not a function of repo size. Sizing the pool from+# ``cpu_count`` therefore made peak memory a function of the *host*: 32 workers+# held 1.57 GB where 8 hold 0.46 GB.+#+# Nothing was bought with it. With 16 cores genuinely free, 8 workers parsed+# PowerToys in 5.54s/4.75s against 16 workers' 6.79s/5.23s (interleaved pairs):+# past this point the per-worker import and grammar build is paid again without+# amortizing over enough files to repay it, so more workers is slightly slower+# *and* multiplies the memory. The same reasoning caps the betweenness pool+# (see ``ingestion/graph/_betweenness.py``).+_MAX_PARSE_WORKERS = 8++# Escape hatch for hosts this default judges wrong in either direction.+_PARSE_WORKERS_ENV = "REPOWISE_PARSE_WORKERS"+# Module-level process-local parser cache (one per worker process)._WORKER_PARSER: Any = None+def parse_pool_workers(pending: int) -> int:+ """How many workers to ask the parse pool for, to parse *pending* files.++ Stated once so the init and resume paths cannot drift apart, and so the+ bound can be asserted on directly. Never exceeds *pending* — a repo with+ three changed files has no use for eight interpreters.++ ``REPOWISE_PARSE_WORKERS`` overrides the cap in both directions. A set but+ non-positive value is ignored with a warning rather than failing an+ otherwise healthy parse; an empty value means "unset", which is what an+ ``export REPOWISE_PARSE_WORKERS=`` from an undefined shell variable+ produces, so it is not worth warning about.+ """+ if pending <= 0:+ return 1++ override = os.environ.get(_PARSE_WORKERS_ENV)+ if override:+ try:+ requested = int(override)+ except ValueError:+ requested = 0+ if requested > 0:+ return min(requested, pending)+ logger.warning("invalid_parse_workers_env", value=override)++ # ``process_cpu_count`` (3.13+) honours the CPU affinity mask, so a process+ # pinned to fewer CPUs than the host has stops sizing its pool from the+ # host's core count. It does *not* read cgroup CPU quotas: a container run+ # with ``--cpus=2`` (CFS quota, no cpuset) still sees the full host mask,+ # and the cap below is what bounds that case. Falls back to ``cpu_count``+ # on 3.11/3.12, and when affinity is unavailable and it returns None.+ affinity_aware = getattr(os, "process_cpu_count", None)+ cpus = (affinity_aware() if affinity_aware else None) or os.cpu_count() or 4+ return max(1, min(cpus, _MAX_PARSE_WORKERS, pending))++def _parse_one(path_and_fi_and_bytes: tuple) -> Any:"""Worker function for ProcessPoolExecutor parsing.@@ -369,7 +428,7 @@ async def _run_ingestion(# entirely (an incremental update re-ingests the whole repo for one# changed file). Misses parse exactly as before.parse_cache, cached_hits, to_parse = _split_cached(repo_path, fi_and_bytes, progress)- workers = max(1, min(os.cpu_count() or 4, len(to_parse) or 1))+ workers = parse_pool_workers(len(to_parse))parse_results: list[Any] = []@@ -636,7 +695,7 @@ async def reparse_for_resume(loop = asyncio.get_running_loop()parse_cache, cached_hits, to_parse = _split_cached(repo_path, fi_and_bytes, progress)- workers = max(1, min(os.cpu_count() or 4, len(to_parse) or 1))+ workers = parse_pool_workers(len(to_parse))parse_results: list[Any] = []if to_parse:diff --git a/tests/unit/pipeline/test_parse_pool_workers.py b/tests/unit/pipeline/test_parse_pool_workers.pynew file mode 100644index 000000000..622c2711b--- /dev/null+++ b/tests/unit/pipeline/test_parse_pool_workers.py@@ -0,0 +1,172 @@+"""The parse pool's worker count is bounded (issue #1394).++Every worker is a fresh spawned interpreter holding ~50 MB of private memory,+so sizing the pool from the host's core count made peak memory a function of+the machine rather than the repo.+"""++import pytest++from repowise.core.pipeline.phases import ingestion as ing+++class TestParsePoolWorkers:+ """The cap, the floor, and the env override."""++ def test_the_cap_is_eight(self):+ """Pin the literal.++ Every other assertion here compares the helper's output to+ ``_MAX_PARSE_WORKERS``, which a revert of the constant would satisfy+ while reintroducing the 32-worker fleet this exists to prevent.+ """+ assert ing._MAX_PARSE_WORKERS == 8++ def test_cap_applies_to_host_core_count(self, monkeypatch):+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)+ assert ing.parse_pool_workers(5000) == 8++ def test_below_cap_is_left_alone(self, monkeypatch):+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 4)+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 4, raising=False)+ assert ing.parse_pool_workers(5000) == 4++ def test_never_exceeds_the_files_to_parse(self, monkeypatch):+ """A three-file update has no use for eight interpreters."""+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)+ assert ing.parse_pool_workers(3) == 3++ def test_empty_work_list_degrades_to_one(self):+ assert ing.parse_pool_workers(0) == 1+ assert ing.parse_pool_workers(-1) == 1++ def test_unknown_core_count_degrades_to_a_default(self, monkeypatch):+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)+ monkeypatch.setattr(ing.os, "cpu_count", lambda: None)+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: None, raising=False)+ assert ing.parse_pool_workers(5000) == 4++ def test_affinity_aware_count_wins_over_host_count(self, monkeypatch):+ """A container pinned to 2 CPUs must not size its pool from the host."""+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 2, raising=False)+ assert ing.parse_pool_workers(5000) == 2++ def test_missing_process_cpu_count_falls_back(self, monkeypatch):+ """Python 3.11/3.12 have no ``process_cpu_count``."""+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)+ monkeypatch.delattr(ing.os, "process_cpu_count", raising=False)+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 3)+ assert ing.parse_pool_workers(5000) == 3++ @pytest.mark.parametrize("value,expected", [("2", 2), ("16", 16), ("1", 1)])+ def test_env_override_wins_in_both_directions(self, monkeypatch, value, expected):+ monkeypatch.setenv(ing._PARSE_WORKERS_ENV, value)+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)+ assert ing.parse_pool_workers(5000) == expected++ def test_env_override_still_bounded_by_pending_files(self, monkeypatch):+ monkeypatch.setenv(ing._PARSE_WORKERS_ENV, "64")+ assert ing.parse_pool_workers(5) == 5++ @pytest.mark.parametrize("value", ["nonsense", "0", "-4", "3.5"])+ def test_unusable_env_value_is_ignored_not_fatal(self, monkeypatch, value):+ """A bad env value must never fail an otherwise healthy parse."""+ monkeypatch.setenv(ing._PARSE_WORKERS_ENV, value)+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)+ assert ing.parse_pool_workers(5000) == 8++ def test_unusable_env_value_says_so(self, monkeypatch):+ """The warning is the only feedback a misconfigured user gets."""+ monkeypatch.setenv(ing._PARSE_WORKERS_ENV, "-4")+ warned: list[tuple] = []+ monkeypatch.setattr(+ ing.logger, "warning", lambda event, **kw: warned.append((event, kw))+ )+ ing.parse_pool_workers(5000)+ assert warned and warned[0][0] == "invalid_parse_workers_env"+ assert warned[0][1]["value"] == "-4"++ def test_empty_env_value_means_unset(self, monkeypatch):+ """``export REPOWISE_PARSE_WORKERS=`` from an undefined shell variable+ is not a misconfiguration worth warning about."""+ monkeypatch.setenv(ing._PARSE_WORKERS_ENV, "")+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)+ warned: list[tuple] = []+ monkeypatch.setattr(+ ing.logger, "warning", lambda event, **kw: warned.append((event, kw))+ )+ assert ing.parse_pool_workers(5000) == 8+ assert not warned++ def test_affinity_unavailable_falls_back_to_host_count(self, monkeypatch):+ """``process_cpu_count`` exists but returns None when there is no mask."""+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: None, raising=False)+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 3)+ assert ing.parse_pool_workers(5000) == 3+++class TestBothCallSitesAreBounded:+ """The bound has to reach the executor on the init *and* resume paths.++ Grepping the source is what this guards: the two paths carried a duplicated+ ``min(os.cpu_count(), ...)`` expression and only one would have been noticed+ if a future edit reverted it.+ """++ @pytest.mark.parametrize("func", ["_run_ingestion", "reparse_for_resume"])+ def test_call_site_uses_the_shared_helper(self, func):+ import inspect++ target = getattr(ing, func)+ src = inspect.getsource(target)+ assert "parse_pool_workers(" in src, f"{func} must size its pool via the shared helper"+ assert "os.cpu_count()" not in src, f"{func} must not size its pool from the host core count"++ async def test_the_bound_reaches_the_real_executor(self, tmp_path, monkeypatch):+ """Record what ``ProcessPoolExecutor`` is really constructed with.++ The source-text assertions above cannot catch a regression spelled a+ different way; this pins the argument itself on a real ingestion run.+ """+ monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)+ monkeypatch.setattr(ing.os, "cpu_count", lambda: 64)+ monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 64, raising=False)++ for i in range(12):+ (tmp_path / f"mod_{i}.py").write_text(f"def f_{i}(x):\n return x + {i}\n")++ seen: list[int | None] = []+ real = ing.ProcessPoolExecutor++ def recording(*args, **kwargs):+ seen.append(kwargs.get("max_workers"))+ # Honour the bound under test but keep the pool tiny: this is a+ # unit test, not a benchmark, and spawning eight interpreters to+ # parse twelve files would dominate the suite's runtime.+ kwargs["max_workers"] = 1+ return real(*args, **kwargs)++ monkeypatch.setattr(ing, "ProcessPoolExecutor", recording)++ parsed, *_ = await ing._run_ingestion(+ tmp_path,+ exclude_patterns=None,+ skip_tests=False,+ skip_infra=False,+ progress=None,+ )++ assert parsed, "ingestion parsed nothing, so the run proves nothing"+ assert seen, "no process pool was constructed, so the bound was never exercised"+ assert seen == [8], f"expected the capped count to reach the executor, got {seen}"