Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(generation): stop holding a second copy of the repo's source through the whole run (#1403)
Raghav Chamadiyac846831c77b4
diff --git a/docs/reference/COMPUTED_GLOSSARY.md b/docs/reference/COMPUTED_GLOSSARY.mdindex 5c8eeb09e..bc448e76d 100644--- a/docs/reference/COMPUTED_GLOSSARY.md+++ b/docs/reference/COMPUTED_GLOSSARY.md@@ -179,7 +179,6 @@ workspace overlays, MCP responses, and CLI output.| Dependency summaries | Summaries of already-generated dependency pages. | `assemble_file_page()` with `page_summaries` | `{ "src/db.py": "Database access layer..." }` || RAG context | Snippets from vector search for related generated pages. | `_generate_file_page_from_ctx()` | `["[file_page:src/schema.py]\nDefines API schema..."]` || Token estimate | `len(text) // 4` heuristic. | `ContextAssembler._estimate_tokens()` | `3200` |-| Structural summary mode | Large-file outline instead of raw source snippet. | `_build_structural_summary()` | `[Large file - structural summary mode]` || Significant file | File selected for its own `file_page`. | `_is_significant_file()` | Entry point, top PageRank, bridge file, package `__init__.py`, or test with symbols || Top symbol selection | Public symbols selected by their file PageRank and percentile budget. | `PageGenerator.generate_all()` | Top 10 percent of public symbols, capped by page budget || Page budget | Hard cap `max(50, int(num_files * max_pages_pct))`. | `PageGenerator.generate_all()` | 800 files with 10 percent cap -> 80-page budget |diff --git a/packages/core/src/repowise/core/generation/context/assembler.py b/packages/core/src/repowise/core/generation/context/assembler.pyindex 4a5efe7a0..9d6fcb626 100644--- a/packages/core/src/repowise/core/generation/context/assembler.py+++ b/packages/core/src/repowise/core/generation/context/assembler.py@@ -328,16 +328,16 @@ class ContextAssembler:in_edges = _file_dependency_neighbors(graph, path, incoming=True)out_edges = _file_dependency_neighbors(graph, path, incoming=False)- # Source snippet — use structural summary for large files+ # Decoded to derive the vocabulary below, and deliberately not carried on+ # the returned context. The context used to hold a ``file_source_snippet``+ # trimmed to ``token_budget`` (48k tokens ≈ 190KB), which for most files+ # is the entire file — and no template, prompt or caller ever read it.+ # A generation run keeps one context per code file alive from level 2+ # until it ends, so that field amounted to a second copy of the whole+ # repository's source resident for the length of the run, which is what+ # exhausted memory on large repositories (issue #1394). ``source_text``+ # is local, so it is freed as soon as this function returns.source_text = source_bytes.decode("utf-8", errors="replace")- remaining = budget - used- source_tokens = self._estimate_tokens(source_text)- threshold = self._config.token_budget * self._config.large_file_source_pct- if source_tokens > remaining and source_tokens > threshold:- snippet = self._build_structural_summary(parsed, source_text, remaining)- else:- snippet = self._trim_to_budget(source_text, remaining)- used += self._estimate_tokens(snippet)# Dependency summaries from already-completed pagesdep_summaries: dict[str, str] = {}@@ -361,7 +361,6 @@ class ContextAssembler:symbols=sym_dicts,imports=import_list,exports=parsed.exports,- file_source_snippet=snippet,pagerank_score=pagerank.get(path, 0.0),betweenness_score=betweenness.get(path, 0.0),community_id=community.get(path, 0),@@ -394,12 +393,11 @@ class ContextAssembler:kg_tour_step=kg_context.tour_step if kg_context else None,kg_tags=kg_context.tags if kg_context else [],kg_node_summary=kg_context.node_summary if kg_context else "",- # Computed from the whole decoded source, not from ``snippet``.- # The snippet is budget-trimmed and on a large file is replaced by- # a structural summary, so reading it would make the vocabulary of- # the biggest files the thinnest, which is backwards. This section- # carries its own cap and is not charged against the prompt budget- # because it is page content rather than model input.+ # Computed from the whole decoded source rather than any trimmed+ # excerpt, so the biggest files do not end up with the thinnest+ # vocabulary. This section carries its own cap and is not charged+ # against the prompt budget because it is page content rather than+ # model input.file_vocabulary=file_vocabulary(source_text),)@@ -935,36 +933,6 @@ class ContextAssembler:targets=targets,)- # ------------------------------------------------------------------- # Structural summary for large files (Phase 9 C1)- # -------------------------------------------------------------------- def _build_structural_summary(self, parsed: ParsedFile, source_text: str, budget: int) -> str:- """Build a structural outline for large files instead of raw truncation.-- Includes full body for the 3 most complex symbols; signature-only for rest.- """- lines = source_text.splitlines()- parts = ["[Large file — structural summary mode]"]- top3_complex = {- s.name- for s in sorted(parsed.symbols, key=lambda s: s.complexity_estimate, reverse=True)[:3]- }-- for sym in parsed.symbols:- if sym.start_line and sym.end_line and sym.name in top3_complex:- body = "\n".join(lines[sym.start_line - 1 : sym.end_line])- parts.append(- f"\n# {sym.name} (full body, complexity={sym.complexity_estimate})\n{body}"- )- else:- parts.append(f"# {sym.signature or sym.name}")- if self._estimate_tokens("\n".join(parts)) >= budget:- parts.append("...[remaining symbols omitted]")- break-- return self._trim_to_budget("\n".join(parts), budget)-# ------------------------------------------------------------------# Generation depth selection (Phase 5.5)# ------------------------------------------------------------------diff --git a/packages/core/src/repowise/core/generation/context/contexts.py b/packages/core/src/repowise/core/generation/context/contexts.pyindex 8f3952987..3930cbddb 100644--- a/packages/core/src/repowise/core/generation/context/contexts.py+++ b/packages/core/src/repowise/core/generation/context/contexts.py@@ -21,7 +21,6 @@ class FilePageContext:symbols: list[dict[str, Any]]imports: list[str]exports: list[str]- file_source_snippet: strpagerank_score: floatbetweenness_score: floatcommunity_id: intdiff --git a/packages/core/src/repowise/core/generation/models.py b/packages/core/src/repowise/core/generation/models.pyindex e2c2b745a..959985eb8 100644--- a/packages/core/src/repowise/core/generation/models.py+++ b/packages/core/src/repowise/core/generation/models.py@@ -296,7 +296,6 @@ class GenerationConfig:# Pages reused from a prior run are never retried (validated back then).repair_warning_threshold: int = 2jobs_dir: str = ".repowise/jobs"- large_file_source_pct: float = 0.4 # use structural summary when source tokens > budget * thislanguage: str = "en"# Wiki documentation style (voice/density). Resolved to a StyleSpec by# ``generation.styles.resolve_style``. "comprehensive" (default) is inert anddiff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.pyindex a46be740c..4c41da491 100644--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py@@ -705,6 +705,7 @@ class _GenerationRun:# would rewrite a whole-repo page from a one-commit view. They stay as# the last full run left them.if getattr(self.config, "file_pages_only", False):+ self.file_page_contexts.clear()return self._finalize(all_pages)# Level 3 (scc_page).@@ -713,6 +714,14 @@ class _GenerationRun:# Level 4 (module_page).all_pages.extend(await self.run_level(_levels.build_level4_coros(self), 4))+ # Level 4 is the last reader of the per-file contexts, and there is one+ # per code file. Dropping them here rather than at the end of the run+ # keeps them out of levels 6-8 and out of _finalize, whose interlinking+ # and related-pages passes are themselves the memory-heaviest part of a+ # large run (issue #1394). Every later level works off pages, the graph+ # and the selection groups, none of which live in this map.+ self.file_page_contexts.clear()+# Level 5 was one page per KG layer. Those pages retired: the layer a# page belongs to is stamped on the page itself, so the docs tree can# group by it without a page to hang the group off.diff --git a/tests/unit/generation/test_concept_index_reaches_search.py b/tests/unit/generation/test_concept_index_reaches_search.pyindex e9ad64dd2..d4e53c30b 100644--- a/tests/unit/generation/test_concept_index_reaches_search.py+++ b/tests/unit/generation/test_concept_index_reaches_search.py@@ -77,7 +77,6 @@ def _file_context(path: str, names: list[str]) -> FilePageContext:],imports=[],exports=names,- file_source_snippet="",pagerank_score=0.5,betweenness_score=0.0,community_id=0,diff --git a/tests/unit/generation/test_context_assembler.py b/tests/unit/generation/test_context_assembler.pyindex ebafaf274..e6a5b7da1 100644--- a/tests/unit/generation/test_context_assembler.py+++ b/tests/unit/generation/test_context_assembler.py@@ -2,6 +2,8 @@from __future__ import annotations+import dataclasses+import networkx as nxfrom repowise.core.generation.context_assembler import (@@ -85,6 +87,38 @@ def test_assemble_file_page_returns_context(assert ctx.language == "python"+def test_assemble_file_page_does_not_retain_source(+ sample_config, sample_parsed_file, sample_graph, graph_metrics+):+ """No field carries the file's source text (issue #1394).++ A run keeps one context per code file alive from level 2 until it ends, so+ anything on the context is multiplied by the size of the repository. The+ context used to carry a budget-trimmed copy of the whole file that nothing+ read, which is what exhausted memory on large repositories. Asserted on the+ values rather than on a field name so re-introducing the source under any+ other name fails here too.+ """+ # A multi-line slice, because the per-file vocabulary legitimately keeps+ # single words off the source and would make a one-word sentinel ambiguous.+ body = "def sentinel_body():\n return 41 + 1\n"+ ctx = ContextAssembler(sample_config).assemble_file_page(+ sample_parsed_file,+ sample_graph,+ graph_metrics["pagerank"],+ graph_metrics["betweenness"],+ graph_metrics["community"],+ f"# header\n{body}".encode(),+ )++ retained = [+ f.name+ for f in dataclasses.fields(ctx)+ if body in str(getattr(ctx, f.name))+ ]+ assert retained == []++def test_assemble_file_page_dependents_from_graph(sample_config, sample_parsed_file, sample_graph, graph_metrics, sample_source_bytes):diff --git a/tests/unit/generation/test_deterministic_templates.py b/tests/unit/generation/test_deterministic_templates.pyindex 8b2eb5804..dc33e44c6 100644--- a/tests/unit/generation/test_deterministic_templates.py+++ b/tests/unit/generation/test_deterministic_templates.py@@ -317,7 +317,6 @@ def _module_ctx(assembler, files):pagerank_score=0.1,betweenness_score=0.0,is_entry_point=False,- file_source_snippet="",community_id=None,is_api_contract=False,is_test=False,diff --git a/tests/unit/generation/test_file_vocabulary.py b/tests/unit/generation/test_file_vocabulary.pyindex 9974bf279..d796f7b27 100644--- a/tests/unit/generation/test_file_vocabulary.py+++ b/tests/unit/generation/test_file_vocabulary.py@@ -205,7 +205,6 @@ def _file_page(**overrides) -> FilePageContext:],imports=[],exports=["NewCmdList"],- file_source_snippet="",pagerank_score=0.5,betweenness_score=0.1,community_id=0,diff --git a/tests/unit/generation/test_kg_aware_file_pages.py b/tests/unit/generation/test_kg_aware_file_pages.pyindex 57b58dead..b213e90cf 100644--- a/tests/unit/generation/test_kg_aware_file_pages.py+++ b/tests/unit/generation/test_kg_aware_file_pages.py@@ -21,7 +21,6 @@ def _make_config(**overrides: Any) -> GenerationConfig:defaults = {"token_budget": 4000,"max_concurrency": 2,- "large_file_source_pct": 0.5,}defaults.update(overrides)return GenerationConfig(**defaults)@@ -340,7 +339,6 @@ class TestFilePageTemplate:symbols=[],imports=[],exports=[],- file_source_snippet="",pagerank_score=0.0,betweenness_score=0.0,community_id=0,@@ -367,7 +365,6 @@ class TestFilePageTemplate:symbols=[],imports=[],exports=[],- file_source_snippet="",pagerank_score=0.0,betweenness_score=0.0,community_id=0,diff --git a/tests/unit/generation/test_models.py b/tests/unit/generation/test_models.pyindex a74ab863e..da4ed6bb8 100644--- a/tests/unit/generation/test_models.py+++ b/tests/unit/generation/test_models.py@@ -157,7 +157,6 @@ def test_generation_config_defaults():assert config.staleness_threshold_days == 7assert config.expiry_threshold_days == 30assert config.top_symbol_percentile == 0.10- assert config.large_file_source_pct == 0.4assert config.reasoning == "auto"diff --git a/tests/unit/generation/test_module_page_concept_index.py b/tests/unit/generation/test_module_page_concept_index.pyindex 518237489..d5116f184 100644--- a/tests/unit/generation/test_module_page_concept_index.py+++ b/tests/unit/generation/test_module_page_concept_index.py@@ -75,7 +75,6 @@ def _file_context(path: str, symbols: list[dict], *, pagerank: float = 0.1) -> Fsymbols=symbols,imports=[],exports=[],- file_source_snippet="",pagerank_score=pagerank,betweenness_score=0.0,community_id=0,diff --git a/tests/unit/generation/test_question_text_reaches_search.py b/tests/unit/generation/test_question_text_reaches_search.pyindex 4d56d84e3..f60de508c 100644--- a/tests/unit/generation/test_question_text_reaches_search.py+++ b/tests/unit/generation/test_question_text_reaches_search.py@@ -93,7 +93,6 @@ def _file_page(**overrides) -> FilePageContext:],imports=[],exports=["FileTraverser"],- file_source_snippet="class FileTraverser: pass",pagerank_score=0.5,betweenness_score=0.1,community_id=0,diff --git a/tests/unit/generation/test_templates.py b/tests/unit/generation/test_templates.pyindex 405c3ea40..9dfbf2a46 100644--- a/tests/unit/generation/test_templates.py+++ b/tests/unit/generation/test_templates.py@@ -89,7 +89,6 @@ def file_page_ctx() -> FilePageContext:],imports=["from python_pkg import models"],exports=["Calculator"],- file_source_snippet="class Calculator:\n pass",pagerank_score=0.5,betweenness_score=0.2,community_id=0,@@ -133,7 +132,6 @@ def bare_file_page_ctx() -> FilePageContext:symbols=[],imports=[],exports=[],- file_source_snippet="DEBUG = False",pagerank_score=0.0,betweenness_score=0.0,community_id=0,