Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(pipeline): keep SQLAlchemy out of spawned parse workers (#2715)

Raghav Chamadiya3728dc4579e4
diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.py
index 233a1752d..5f6896eac 100644
--- a/packages/core/src/repowise/core/generation/page_generator/core.py
+++ b/packages/core/src/repowise/core/generation/page_generator/core.py
@@ -23,7 +23,6 @@ import jinja2
import structlog
from repowise.core.ingestion.models import ParsedFile, RepoStructure
-from repowise.core.persistence.vector_store import FILE_VOCABULARY_KEY
from repowise.core.providers.llm.base import BaseProvider, CacheHint, GeneratedResponse
from ..context.evidence import (
@@ -339,6 +338,9 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):
_attach_file_provenance(page, ctx)
# Embedded for search, not rendered (see file_page.j2).
if ctx.file_vocabulary:
+ # Local: persistence pulls SQLAlchemy into every parse worker.
+ from repowise.core.persistence.vector_store import FILE_VOCABULARY_KEY
+
page.metadata[FILE_VOCABULARY_KEY] = ctx.file_vocabulary
return page
diff --git a/tests/unit/pipeline/test_parse_pool_workers.py b/tests/unit/pipeline/test_parse_pool_workers.py
index 622c2711b..99d89825d 100644
--- a/tests/unit/pipeline/test_parse_pool_workers.py
+++ b/tests/unit/pipeline/test_parse_pool_workers.py
@@ -5,6 +5,9 @@ so sizing the pool from the host's core count made peak memory a function of
the machine rather than the repo.
"""
+import subprocess
+import sys
+
import pytest
from repowise.core.pipeline.phases import ingestion as ing
@@ -170,3 +173,23 @@ class TestBothCallSitesAreBounded:
assert parsed, "ingestion parsed nothing, so the run proves nothing"
assert seen, "no process pool was constructed, so the bound was never exercised"
assert seen == [8], f"expected the capped count to reach the executor, got {seen}"
+
+
+class TestWorkerImportWeight:
+ """A spawned worker re-imports this module, so its imports are per-worker cost.
+
+ A string constant imported from the persistence package once pulled
+ SQLAlchemy into every worker: +31 MB and +209 modules each.
+ """
+
+ def test_worker_module_does_not_load_persistence(self):
+ script = (
+ "import sys, repowise.core.pipeline.phases.ingestion; "
+ "print('\\n'.join(sorted(m for m in sys.modules if m == 'sqlalchemy' "
+ "or m.startswith(('sqlalchemy.', 'repowise.core.persistence')))))"
+ )
+ done = subprocess.run(
+ [sys.executable, "-c", script], capture_output=True, text=True, check=False
+ )
+ assert done.returncode == 0, done.stderr
+ assert done.stdout.split() == [], f"parse worker imports: {done.stdout.split()[:10]}"