Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(generation): stop holding a second copy of the repo's source through the whole run (#1403)

Raghav Chamadiyac846831c77b4
diff --git a/docs/reference/COMPUTED_GLOSSARY.md b/docs/reference/COMPUTED_GLOSSARY.md
index 5c8eeb09e..bc448e76d 100644
--- a/docs/reference/COMPUTED_GLOSSARY.md
+++ b/docs/reference/COMPUTED_GLOSSARY.md
@@ -179,7 +179,6 @@ workspace overlays, MCP responses, and CLI output.
| Dependency summaries | Summaries of already-generated dependency pages. | `assemble_file_page()` with `page_summaries` | `{ "src/db.py": "Database access layer..." }` |
| RAG context | Snippets from vector search for related generated pages. | `_generate_file_page_from_ctx()` | `["[file_page:src/schema.py]\nDefines API schema..."]` |
| Token estimate | `len(text) // 4` heuristic. | `ContextAssembler._estimate_tokens()` | `3200` |
-| Structural summary mode | Large-file outline instead of raw source snippet. | `_build_structural_summary()` | `[Large file - structural summary mode]` |
| Significant file | File selected for its own `file_page`. | `_is_significant_file()` | Entry point, top PageRank, bridge file, package `__init__.py`, or test with symbols |
| Top symbol selection | Public symbols selected by their file PageRank and percentile budget. | `PageGenerator.generate_all()` | Top 10 percent of public symbols, capped by page budget |
| Page budget | Hard cap `max(50, int(num_files * max_pages_pct))`. | `PageGenerator.generate_all()` | 800 files with 10 percent cap -> 80-page budget |
diff --git a/packages/core/src/repowise/core/generation/context/assembler.py b/packages/core/src/repowise/core/generation/context/assembler.py
index 4a5efe7a0..9d6fcb626 100644
--- a/packages/core/src/repowise/core/generation/context/assembler.py
+++ b/packages/core/src/repowise/core/generation/context/assembler.py
@@ -328,16 +328,16 @@ class ContextAssembler:
in_edges = _file_dependency_neighbors(graph, path, incoming=True)
out_edges = _file_dependency_neighbors(graph, path, incoming=False)
- # Source snippet — use structural summary for large files
+ # Decoded to derive the vocabulary below, and deliberately not carried on
+ # the returned context. The context used to hold a ``file_source_snippet``
+ # trimmed to ``token_budget`` (48k tokens ≈ 190KB), which for most files
+ # is the entire file — and no template, prompt or caller ever read it.
+ # A generation run keeps one context per code file alive from level 2
+ # until it ends, so that field amounted to a second copy of the whole
+ # repository's source resident for the length of the run, which is what
+ # exhausted memory on large repositories (issue #1394). ``source_text``
+ # is local, so it is freed as soon as this function returns.
source_text = source_bytes.decode("utf-8", errors="replace")
- remaining = budget - used
- source_tokens = self._estimate_tokens(source_text)
- threshold = self._config.token_budget * self._config.large_file_source_pct
- if source_tokens > remaining and source_tokens > threshold:
- snippet = self._build_structural_summary(parsed, source_text, remaining)
- else:
- snippet = self._trim_to_budget(source_text, remaining)
- used += self._estimate_tokens(snippet)
# Dependency summaries from already-completed pages
dep_summaries: dict[str, str] = {}
@@ -361,7 +361,6 @@ class ContextAssembler:
symbols=sym_dicts,
imports=import_list,
exports=parsed.exports,
- file_source_snippet=snippet,
pagerank_score=pagerank.get(path, 0.0),
betweenness_score=betweenness.get(path, 0.0),
community_id=community.get(path, 0),
@@ -394,12 +393,11 @@ class ContextAssembler:
kg_tour_step=kg_context.tour_step if kg_context else None,
kg_tags=kg_context.tags if kg_context else [],
kg_node_summary=kg_context.node_summary if kg_context else "",
- # Computed from the whole decoded source, not from ``snippet``.
- # The snippet is budget-trimmed and on a large file is replaced by
- # a structural summary, so reading it would make the vocabulary of
- # the biggest files the thinnest, which is backwards. This section
- # carries its own cap and is not charged against the prompt budget
- # because it is page content rather than model input.
+ # Computed from the whole decoded source rather than any trimmed
+ # excerpt, so the biggest files do not end up with the thinnest
+ # vocabulary. This section carries its own cap and is not charged
+ # against the prompt budget because it is page content rather than
+ # model input.
file_vocabulary=file_vocabulary(source_text),
)
@@ -935,36 +933,6 @@ class ContextAssembler:
targets=targets,
)
- # ------------------------------------------------------------------
- # Structural summary for large files (Phase 9 C1)
- # ------------------------------------------------------------------
-
- def _build_structural_summary(self, parsed: ParsedFile, source_text: str, budget: int) -> str:
- """Build a structural outline for large files instead of raw truncation.
-
- Includes full body for the 3 most complex symbols; signature-only for rest.
- """
- lines = source_text.splitlines()
- parts = ["[Large file — structural summary mode]"]
- top3_complex = {
- s.name
- for s in sorted(parsed.symbols, key=lambda s: s.complexity_estimate, reverse=True)[:3]
- }
-
- for sym in parsed.symbols:
- if sym.start_line and sym.end_line and sym.name in top3_complex:
- body = "\n".join(lines[sym.start_line - 1 : sym.end_line])
- parts.append(
- f"\n# {sym.name} (full body, complexity={sym.complexity_estimate})\n{body}"
- )
- else:
- parts.append(f"# {sym.signature or sym.name}")
- if self._estimate_tokens("\n".join(parts)) >= budget:
- parts.append("...[remaining symbols omitted]")
- break
-
- return self._trim_to_budget("\n".join(parts), budget)
-
# ------------------------------------------------------------------
# Generation depth selection (Phase 5.5)
# ------------------------------------------------------------------
diff --git a/packages/core/src/repowise/core/generation/context/contexts.py b/packages/core/src/repowise/core/generation/context/contexts.py
index 8f3952987..3930cbddb 100644
--- a/packages/core/src/repowise/core/generation/context/contexts.py
+++ b/packages/core/src/repowise/core/generation/context/contexts.py
@@ -21,7 +21,6 @@ class FilePageContext:
symbols: list[dict[str, Any]]
imports: list[str]
exports: list[str]
- file_source_snippet: str
pagerank_score: float
betweenness_score: float
community_id: int
diff --git a/packages/core/src/repowise/core/generation/models.py b/packages/core/src/repowise/core/generation/models.py
index e2c2b745a..959985eb8 100644
--- a/packages/core/src/repowise/core/generation/models.py
+++ b/packages/core/src/repowise/core/generation/models.py
@@ -296,7 +296,6 @@ class GenerationConfig:
# Pages reused from a prior run are never retried (validated back then).
repair_warning_threshold: int = 2
jobs_dir: str = ".repowise/jobs"
- large_file_source_pct: float = 0.4 # use structural summary when source tokens > budget * this
language: str = "en"
# Wiki documentation style (voice/density). Resolved to a StyleSpec by
# ``generation.styles.resolve_style``. "comprehensive" (default) is inert and
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
index a46be740c..4c41da491 100644
--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py
@@ -705,6 +705,7 @@ class _GenerationRun:
# would rewrite a whole-repo page from a one-commit view. They stay as
# the last full run left them.
if getattr(self.config, "file_pages_only", False):
+ self.file_page_contexts.clear()
return self._finalize(all_pages)
# Level 3 (scc_page).
@@ -713,6 +714,14 @@ class _GenerationRun:
# Level 4 (module_page).
all_pages.extend(await self.run_level(_levels.build_level4_coros(self), 4))
+ # Level 4 is the last reader of the per-file contexts, and there is one
+ # per code file. Dropping them here rather than at the end of the run
+ # keeps them out of levels 6-8 and out of _finalize, whose interlinking
+ # and related-pages passes are themselves the memory-heaviest part of a
+ # large run (issue #1394). Every later level works off pages, the graph
+ # and the selection groups, none of which live in this map.
+ self.file_page_contexts.clear()
+
# Level 5 was one page per KG layer. Those pages retired: the layer a
# page belongs to is stamped on the page itself, so the docs tree can
# group by it without a page to hang the group off.
diff --git a/tests/unit/generation/test_concept_index_reaches_search.py b/tests/unit/generation/test_concept_index_reaches_search.py
index e9ad64dd2..d4e53c30b 100644
--- a/tests/unit/generation/test_concept_index_reaches_search.py
+++ b/tests/unit/generation/test_concept_index_reaches_search.py
@@ -77,7 +77,6 @@ def _file_context(path: str, names: list[str]) -> FilePageContext:
],
imports=[],
exports=names,
- file_source_snippet="",
pagerank_score=0.5,
betweenness_score=0.0,
community_id=0,
diff --git a/tests/unit/generation/test_context_assembler.py b/tests/unit/generation/test_context_assembler.py
index ebafaf274..e6a5b7da1 100644
--- a/tests/unit/generation/test_context_assembler.py
+++ b/tests/unit/generation/test_context_assembler.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+import dataclasses
+
import networkx as nx
from repowise.core.generation.context_assembler import (
@@ -85,6 +87,38 @@ def test_assemble_file_page_returns_context(
assert ctx.language == "python"
+def test_assemble_file_page_does_not_retain_source(
+ sample_config, sample_parsed_file, sample_graph, graph_metrics
+):
+ """No field carries the file's source text (issue #1394).
+
+ A run keeps one context per code file alive from level 2 until it ends, so
+ anything on the context is multiplied by the size of the repository. The
+ context used to carry a budget-trimmed copy of the whole file that nothing
+ read, which is what exhausted memory on large repositories. Asserted on the
+ values rather than on a field name so re-introducing the source under any
+ other name fails here too.
+ """
+ # A multi-line slice, because the per-file vocabulary legitimately keeps
+ # single words off the source and would make a one-word sentinel ambiguous.
+ body = "def sentinel_body():\n return 41 + 1\n"
+ ctx = ContextAssembler(sample_config).assemble_file_page(
+ sample_parsed_file,
+ sample_graph,
+ graph_metrics["pagerank"],
+ graph_metrics["betweenness"],
+ graph_metrics["community"],
+ f"# header\n{body}".encode(),
+ )
+
+ retained = [
+ f.name
+ for f in dataclasses.fields(ctx)
+ if body in str(getattr(ctx, f.name))
+ ]
+ assert retained == []
+
+
def test_assemble_file_page_dependents_from_graph(
sample_config, sample_parsed_file, sample_graph, graph_metrics, sample_source_bytes
):
diff --git a/tests/unit/generation/test_deterministic_templates.py b/tests/unit/generation/test_deterministic_templates.py
index 8b2eb5804..dc33e44c6 100644
--- a/tests/unit/generation/test_deterministic_templates.py
+++ b/tests/unit/generation/test_deterministic_templates.py
@@ -317,7 +317,6 @@ def _module_ctx(assembler, files):
pagerank_score=0.1,
betweenness_score=0.0,
is_entry_point=False,
- file_source_snippet="",
community_id=None,
is_api_contract=False,
is_test=False,
diff --git a/tests/unit/generation/test_file_vocabulary.py b/tests/unit/generation/test_file_vocabulary.py
index 9974bf279..d796f7b27 100644
--- a/tests/unit/generation/test_file_vocabulary.py
+++ b/tests/unit/generation/test_file_vocabulary.py
@@ -205,7 +205,6 @@ def _file_page(**overrides) -> FilePageContext:
],
imports=[],
exports=["NewCmdList"],
- file_source_snippet="",
pagerank_score=0.5,
betweenness_score=0.1,
community_id=0,
diff --git a/tests/unit/generation/test_kg_aware_file_pages.py b/tests/unit/generation/test_kg_aware_file_pages.py
index 57b58dead..b213e90cf 100644
--- a/tests/unit/generation/test_kg_aware_file_pages.py
+++ b/tests/unit/generation/test_kg_aware_file_pages.py
@@ -21,7 +21,6 @@ def _make_config(**overrides: Any) -> GenerationConfig:
defaults = {
"token_budget": 4000,
"max_concurrency": 2,
- "large_file_source_pct": 0.5,
}
defaults.update(overrides)
return GenerationConfig(**defaults)
@@ -340,7 +339,6 @@ class TestFilePageTemplate:
symbols=[],
imports=[],
exports=[],
- file_source_snippet="",
pagerank_score=0.0,
betweenness_score=0.0,
community_id=0,
@@ -367,7 +365,6 @@ class TestFilePageTemplate:
symbols=[],
imports=[],
exports=[],
- file_source_snippet="",
pagerank_score=0.0,
betweenness_score=0.0,
community_id=0,
diff --git a/tests/unit/generation/test_models.py b/tests/unit/generation/test_models.py
index a74ab863e..da4ed6bb8 100644
--- a/tests/unit/generation/test_models.py
+++ b/tests/unit/generation/test_models.py
@@ -157,7 +157,6 @@ def test_generation_config_defaults():
assert config.staleness_threshold_days == 7
assert config.expiry_threshold_days == 30
assert config.top_symbol_percentile == 0.10
- assert config.large_file_source_pct == 0.4
assert config.reasoning == "auto"
diff --git a/tests/unit/generation/test_module_page_concept_index.py b/tests/unit/generation/test_module_page_concept_index.py
index 518237489..d5116f184 100644
--- a/tests/unit/generation/test_module_page_concept_index.py
+++ b/tests/unit/generation/test_module_page_concept_index.py
@@ -75,7 +75,6 @@ def _file_context(path: str, symbols: list[dict], *, pagerank: float = 0.1) -> F
symbols=symbols,
imports=[],
exports=[],
- file_source_snippet="",
pagerank_score=pagerank,
betweenness_score=0.0,
community_id=0,
diff --git a/tests/unit/generation/test_question_text_reaches_search.py b/tests/unit/generation/test_question_text_reaches_search.py
index 4d56d84e3..f60de508c 100644
--- a/tests/unit/generation/test_question_text_reaches_search.py
+++ b/tests/unit/generation/test_question_text_reaches_search.py
@@ -93,7 +93,6 @@ def _file_page(**overrides) -> FilePageContext:
],
imports=[],
exports=["FileTraverser"],
- file_source_snippet="class FileTraverser: pass",
pagerank_score=0.5,
betweenness_score=0.1,
community_id=0,
diff --git a/tests/unit/generation/test_templates.py b/tests/unit/generation/test_templates.py
index 405c3ea40..9dfbf2a46 100644
--- a/tests/unit/generation/test_templates.py
+++ b/tests/unit/generation/test_templates.py
@@ -89,7 +89,6 @@ def file_page_ctx() -> FilePageContext:
],
imports=["from python_pkg import models"],
exports=["Calculator"],
- file_source_snippet="class Calculator:\n pass",
pagerank_score=0.5,
betweenness_score=0.2,
community_id=0,
@@ -133,7 +132,6 @@ def bare_file_page_ctx() -> FilePageContext:
symbols=[],
imports=[],
exports=[],
- file_source_snippet="DEBUG = False",
pagerank_score=0.0,
betweenness_score=0.0,
community_id=0,