Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(pipeline): keep SQLAlchemy out of spawned parse workers (#2715)
Raghav Chamadiya3728dc4579e4
diff --git a/packages/core/src/repowise/core/generation/page_generator/core.py b/packages/core/src/repowise/core/generation/page_generator/core.pyindex 233a1752d..5f6896eac 100644--- a/packages/core/src/repowise/core/generation/page_generator/core.py+++ b/packages/core/src/repowise/core/generation/page_generator/core.py@@ -23,7 +23,6 @@ import jinja2import structlogfrom repowise.core.ingestion.models import ParsedFile, RepoStructure-from repowise.core.persistence.vector_store import FILE_VOCABULARY_KEYfrom repowise.core.providers.llm.base import BaseProvider, CacheHint, GeneratedResponsefrom ..context.evidence import (@@ -339,6 +338,9 @@ class PageGenerator(PerTypeGenerationMixin, StructuralRenderMixin):_attach_file_provenance(page, ctx)# Embedded for search, not rendered (see file_page.j2).if ctx.file_vocabulary:+ # Local: persistence pulls SQLAlchemy into every parse worker.+ from repowise.core.persistence.vector_store import FILE_VOCABULARY_KEY+page.metadata[FILE_VOCABULARY_KEY] = ctx.file_vocabularyreturn pagediff --git a/tests/unit/pipeline/test_parse_pool_workers.py b/tests/unit/pipeline/test_parse_pool_workers.pyindex 622c2711b..99d89825d 100644--- a/tests/unit/pipeline/test_parse_pool_workers.py+++ b/tests/unit/pipeline/test_parse_pool_workers.py@@ -5,6 +5,9 @@ so sizing the pool from the host's core count made peak memory a function ofthe machine rather than the repo."""+import subprocess+import sys+import pytestfrom repowise.core.pipeline.phases import ingestion as ing@@ -170,3 +173,23 @@ class TestBothCallSitesAreBounded:assert parsed, "ingestion parsed nothing, so the run proves nothing"assert seen, "no process pool was constructed, so the bound was never exercised"assert seen == [8], f"expected the capped count to reach the executor, got {seen}"+++class TestWorkerImportWeight:+ """A spawned worker re-imports this module, so its imports are per-worker cost.++ A string constant imported from the persistence package once pulled+ SQLAlchemy into every worker: +31 MB and +209 modules each.+ """++ def test_worker_module_does_not_load_persistence(self):+ script = (+ "import sys, repowise.core.pipeline.phases.ingestion; "+ "print('\\n'.join(sorted(m for m in sys.modules if m == 'sqlalchemy' "+ "or m.startswith(('sqlalchemy.', 'repowise.core.persistence')))))"+ )+ done = subprocess.run(+ [sys.executable, "-c", script], capture_output=True, text=True, check=False+ )+ assert done.returncode == 0, done.stderr+ assert done.stdout.split() == [], f"parse worker imports: {done.stdout.split()[:10]}"