Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(core): use spawn for parse and betweenness process pools (#679)

Raghav Chamadiya8737c9d255e6
diff --git a/packages/core/src/repowise/core/ingestion/graph/_betweenness.py b/packages/core/src/repowise/core/ingestion/graph/_betweenness.py
index e2585c9e2..4a87ac289 100644
--- a/packages/core/src/repowise/core/ingestion/graph/_betweenness.py
+++ b/packages/core/src/repowise/core/ingestion/graph/_betweenness.py
@@ -21,6 +21,7 @@ are unavailable.
from __future__ import annotations
+import multiprocessing
import os
from concurrent.futures import ProcessPoolExecutor
@@ -29,6 +30,14 @@ import structlog
log = structlog.get_logger(__name__)
+# ``spawn`` rather than the POSIX default ``fork``: graph metrics run after
+# ingestion, and in a workspace init an earlier repo's doc generation may have
+# already imported LanceDB (whose ``lance`` core is not fork-safe and installs
+# an ``os.register_at_fork`` hook that can deadlock the child). Clean spawned
+# workers never inherit that runtime. Same rationale as the parse pool in
+# ``pipeline/phases/ingestion.py`` (issue #678).
+_MP_SPAWN = multiprocessing.get_context("spawn")
+
# Parallelize only when the estimated Brandes cost (~nodes x edges) is large
# enough to amortize process startup (~2-3s for pool spawn + imports).
# Measured rates on the repowise graph: file subgraph (1.8k x 5.2k ~ 1e7)
@@ -138,6 +147,7 @@ def betweenness_centrality_fast(
try:
with ProcessPoolExecutor(
max_workers=workers,
+ mp_context=_MP_SPAWN,
initializer=_init_worker,
initargs=(n, edges, g.is_directed()),
) as pool:
diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
index 2f28e6f45..d4a020ebe 100644
--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py
+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import asyncio
import contextlib
+import multiprocessing
import os
import time
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
@@ -60,6 +61,17 @@ async def _timed_step(
# Process-pool worker (module-level — must be picklable)
# ---------------------------------------------------------------------------
+# Force ``spawn`` for the parse pool instead of the POSIX default ``fork``.
+# A workspace init generates docs for one repo (which imports and connects
+# LanceDB) before ingesting the next, and ``lance`` is not fork-safe — it
+# registers an ``os.register_at_fork`` hook that resets its native async
+# runtime in the child but can still deadlock. Forking the parse workers
+# after LanceDB is live is exactly what hung multi-repo inits at "Parsing
+# files" (issue #678). ``spawn`` starts clean workers that never inherit the
+# runtime; it is already the default on Windows/macOS, so ``_parse_one`` and
+# its picklable args are known to work under it.
+_MP_SPAWN = multiprocessing.get_context("spawn")
+
# Module-level process-local parser cache (one per worker process).
_WORKER_PARSER: Any = None
@@ -262,7 +274,7 @@ async def _run_ingestion(
if to_parse:
try:
- with ProcessPoolExecutor(max_workers=workers) as pool:
+ with ProcessPoolExecutor(max_workers=workers, mp_context=_MP_SPAWN) as pool:
tasks = [
loop.run_in_executor(pool, _parse_one, item) for _idx, item, _h in to_parse
]
@@ -551,7 +563,7 @@ async def reparse_for_resume(
if to_parse:
try:
- with ProcessPoolExecutor(max_workers=workers) as pool:
+ with ProcessPoolExecutor(max_workers=workers, mp_context=_MP_SPAWN) as pool:
tasks = [
loop.run_in_executor(pool, _parse_one, item) for _idx, item, _h in to_parse
]