Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(core): use spawn for parse and betweenness process pools (#679)
Raghav Chamadiya8737c9d255e6
diff --git a/packages/core/src/repowise/core/ingestion/graph/_betweenness.py b/packages/core/src/repowise/core/ingestion/graph/_betweenness.pyindex e2585c9e2..4a87ac289 100644--- a/packages/core/src/repowise/core/ingestion/graph/_betweenness.py+++ b/packages/core/src/repowise/core/ingestion/graph/_betweenness.py@@ -21,6 +21,7 @@ are unavailable.from __future__ import annotations+import multiprocessingimport osfrom concurrent.futures import ProcessPoolExecutor@@ -29,6 +30,14 @@ import structloglog = structlog.get_logger(__name__)+# ``spawn`` rather than the POSIX default ``fork``: graph metrics run after+# ingestion, and in a workspace init an earlier repo's doc generation may have+# already imported LanceDB (whose ``lance`` core is not fork-safe and installs+# an ``os.register_at_fork`` hook that can deadlock the child). Clean spawned+# workers never inherit that runtime. Same rationale as the parse pool in+# ``pipeline/phases/ingestion.py`` (issue #678).+_MP_SPAWN = multiprocessing.get_context("spawn")+# Parallelize only when the estimated Brandes cost (~nodes x edges) is large# enough to amortize process startup (~2-3s for pool spawn + imports).# Measured rates on the repowise graph: file subgraph (1.8k x 5.2k ~ 1e7)@@ -138,6 +147,7 @@ def betweenness_centrality_fast(try:with ProcessPoolExecutor(max_workers=workers,+ mp_context=_MP_SPAWN,initializer=_init_worker,initargs=(n, edges, g.is_directed()),) as pool:diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.pyindex 2f28e6f45..d4a020ebe 100644--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py@@ -8,6 +8,7 @@ from __future__ import annotationsimport asyncioimport contextlib+import multiprocessingimport osimport timefrom concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor@@ -60,6 +61,17 @@ async def _timed_step(# Process-pool worker (module-level — must be picklable)# ---------------------------------------------------------------------------+# Force ``spawn`` for the parse pool instead of the POSIX default ``fork``.+# A workspace init generates docs for one repo (which imports and connects+# LanceDB) before ingesting the next, and ``lance`` is not fork-safe — it+# registers an ``os.register_at_fork`` hook that resets its native async+# runtime in the child but can still deadlock. Forking the parse workers+# after LanceDB is live is exactly what hung multi-repo inits at "Parsing+# files" (issue #678). ``spawn`` starts clean workers that never inherit the+# runtime; it is already the default on Windows/macOS, so ``_parse_one`` and+# its picklable args are known to work under it.+_MP_SPAWN = multiprocessing.get_context("spawn")+# Module-level process-local parser cache (one per worker process)._WORKER_PARSER: Any = None@@ -262,7 +274,7 @@ async def _run_ingestion(if to_parse:try:- with ProcessPoolExecutor(max_workers=workers) as pool:+ with ProcessPoolExecutor(max_workers=workers, mp_context=_MP_SPAWN) as pool:tasks = [loop.run_in_executor(pool, _parse_one, item) for _idx, item, _h in to_parse]@@ -551,7 +563,7 @@ async def reparse_for_resume(if to_parse:try:- with ProcessPoolExecutor(max_workers=workers) as pool:+ with ProcessPoolExecutor(max_workers=workers, mp_context=_MP_SPAWN) as pool:tasks = [loop.run_in_executor(pool, _parse_one, item) for _idx, item, _h in to_parse]