Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(editor-files): skip structure-only preamble in key modules and architecture (#3261)

c0mradeb95a1d32ab39
diff --git a/packages/core/src/repowise/core/generation/editor_files/fetcher.py b/packages/core/src/repowise/core/generation/editor_files/fetcher.py
index c6e95799f..310ad8683 100644
--- a/packages/core/src/repowise/core/generation/editor_files/fetcher.py
+++ b/packages/core/src/repowise/core/generation/editor_files/fetcher.py
@@ -160,7 +160,8 @@ class EditorFileDataFetcher:
if not pages:
return ""
content = pages[0].content or ""
- return _extract_sentences(content, max_sentences=4)
+ skip_preamble = _is_structure_only(pages[0])
+ return _extract_sentences(content, max_sentences=4, skip_preamble=skip_preamble)
async def _get_key_modules(self) -> list[KeyModule]:
"""Top modules by PageRank with owner from git metadata."""
@@ -192,7 +193,10 @@ class EditorFileDataFetcher:
modules: list[KeyModule] = []
for page, _pagerank, symbol_count in rows:
- purpose = _extract_sentences(page.content or "", max_sentences=1)
+ skip_preamble = _is_structure_only(page)
+ purpose = _extract_sentences(
+ page.content or "", max_sentences=1, skip_preamble=skip_preamble
+ )
# 80 chars cut every purpose mid-thought ("…is the **test-layer
# ingestion subsystem** for…"); 140 fits one real clause.
purpose = _truncate_at_word(purpose, 140).rstrip(".") if purpose else ""
@@ -615,7 +619,28 @@ def _get_head_short_sha(repo_path: Path) -> str:
return ""
-def _extract_sentences(text: str, max_sentences: int) -> str:
+def _is_all_emphasis(paragraph: str) -> bool:
+ """True when one emphasis span wraps the whole paragraph."""
+ p = paragraph.strip()
+ for marker in ("**", "*", "__", "_"):
+ if len(p) > 2 * len(marker) and p.startswith(marker) and p.endswith(marker):
+ inner = p[len(marker) : -len(marker)]
+ if marker[0] not in inner and not inner[0].isspace():
+ return True
+ return False
+
+
+def _is_structure_only(page: object) -> bool:
+ """True when no model wrote *page*: it was rendered from structure.
+
+ Structure-only pages are stored with ``provider_name='template'``. Their
+ preamble (directory line, ``**Language:** ...`` row, scope sentence) is
+ page furniture, not prose about what the module is for.
+ """
+ return getattr(page, "provider_name", "") == "template"
+
+
+def _extract_sentences(text: str, max_sentences: int, *, skip_preamble: bool = False) -> str:
"""Return up to *max_sentences* prose sentences from the start of *text*.
Strips markdown headers/code fences AND list/table lines so only prose
@@ -623,7 +648,16 @@ def _extract_sentences(text: str, max_sentences: int) -> str:
them used to jam bullets onto the tail of the last real sentence
("...rendered in a web UI. - **Languages**") and leave orphaned
fragments like "1. **Inputs**" in the rendered CLAUDE.md.
+
+ When *skip_preamble* is True (used for structure-only pages), all content
+ prior to the first ``##`` section header is treated as template furniture
+ and discarded. If the text has no ``##`` headings, it yields an empty string.
"""
+ if skip_preamble:
+ first_section = re.search(r"^##\s", text, flags=re.MULTILINE)
+ if first_section is None:
+ return ""
+ text = text[first_section.start() :]
# Remove markdown headers and code fences
text = re.sub(r"^#{1,6}\s+.*$", "", text, flags=re.MULTILINE)
text = re.sub(r"```.*?```", "", text, flags=re.DOTALL)
@@ -636,7 +670,21 @@ def _extract_sentences(text: str, max_sentences: int) -> str:
text = re.sub(r"^.*:\s*$", "", text, flags=re.MULTILINE)
text = re.sub(r"`([^`]+)`", r"\1", text) # strip backticks, keep text
text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text) # links → text
- text = re.sub(r"\n{2,}", "\n", text).strip()
+ # Strips Markdown horizontal rules / divider lines (---, ***, ___)
+ text = re.sub(r"^[ \t]*([-*_])(?:[ \t]*\1){2,}[ \t]*$", "", text, flags=re.MULTILINE)
+ # Strip bold metadata rows; requiring '|' between 2+ segments ensures model prose
+ # starting with '**Note:**' is preserved
+ text = re.sub(
+ r"^[ \t]*\*\*[^*\n]+:\*\*[^|\n]*(?:\|[ \t]*\*\*[^*\n]+:\*\*[^|\n]*)+$",
+ "",
+ text,
+ flags=re.MULTILINE,
+ )
+ # Drop paragraphs wrapped entirely in one emphasis span
+ # (the footer note on structure-only pages).
+ text = "\n\n".join(p for p in re.split(r"\n\s*\n", text) if not _is_all_emphasis(p))
+ # Collapse all remaining whitespace and newlines into a single flat line of prose
+ text = re.sub(r"\s+", " ", text).strip()
# Split on sentence boundaries
sentences = re.split(r"(?<=[.!?])\s+", text)
diff --git a/tests/unit/generation/test_editor_fetcher_helpers.py b/tests/unit/generation/test_editor_fetcher_helpers.py
index 5fe5ddff0..8464e36a2 100644
--- a/tests/unit/generation/test_editor_fetcher_helpers.py
+++ b/tests/unit/generation/test_editor_fetcher_helpers.py
@@ -74,6 +74,90 @@ class TestExtractSentences:
def test_empty_input(self) -> None:
assert _extract_sentences("", max_sentences=3) == ""
+ def test_drops_thematic_break(self) -> None:
+ text = "---\n\nThe accounts module manages authentication and sessions."
+ got = _extract_sentences(text, max_sentences=1)
+ assert got == "The accounts module manages authentication and sessions."
+
+ def test_drops_meta_badge_rows(self) -> None:
+ text = (
+ "**Language:** python | **Files:** 9 | **Public symbols:** 5 / 5\n\n"
+ "The accounts module manages authentication and sessions."
+ )
+ got = _extract_sentences(text, max_sentences=1)
+ assert got == "The accounts module manages authentication and sessions."
+
+ def test_joins_new_lines_into_spaces(self) -> None:
+ text = (
+ "The accounts module manages\n"
+ "authentication and active user\n"
+ "sessions across requests."
+ )
+ got = _extract_sentences(text, max_sentences=1)
+ assert "\n" not in got
+ assert got == (
+ "The accounts module manages authentication and active "
+ "user sessions across requests."
+ )
+
+ def test_keeps_a_single_bold_label_sentence(self) -> None:
+ text = "**Note:** Handles invoicing and payment retries."
+ assert _extract_sentences(text, max_sentences=1) == text
+
+ def test_drops_all_emphasis_footer(self) -> None:
+ text = (
+ "The accounts module manages authentication and sessions.\n\n"
+ "*Built from the code's structure. It states what is there, not why it is that\n"
+ "way. The explanatory prose is a separate, model-written layer.*\n"
+ )
+ got = _extract_sentences(text, max_sentences=4)
+ assert got == "The accounts module manages authentication and sessions."
+
+ def test_drops_thematic_break_and_all_emphasis_footer(self) -> None:
+ text = (
+ "---\n\n"
+ "*Built from the code's structure. It states what is there, not why it is that\n"
+ "way. The explanatory prose is a separate, model-written layer.*\n"
+ )
+ assert _extract_sentences(text, max_sentences=2) == ""
+
+ def test_keeps_prose_that_starts_and_ends_with_bold(self) -> None:
+ text = "**Billing** is used by **reports**"
+ assert _extract_sentences(text, max_sentences=1) == text
+
+ def test_skip_preamble_starts_at_first_section(self) -> None:
+ text = (
+ "# billing\n\n`billing`\n\n"
+ "**Language:** python | **Files:** 6\n\n"
+ "Covers the 6 source files in billing.\n\n"
+ "## Overview\n\n"
+ "billing covers 6 python files, exposing 5 public symbols.\n"
+ )
+ got = _extract_sentences(text, max_sentences=1, skip_preamble=True)
+ assert got == "billing covers 6 python files, exposing 5 public symbols."
+
+ def test_skip_preamble_without_a_section_returns_empty(self) -> None:
+ text = "# billing\n\n**Language:** python | **Files:** 6\n\nCovers the 6 files."
+ assert _extract_sentences(text, max_sentences=1, skip_preamble=True) == ""
+
+ def test_preamble_prose_is_kept_by_default(self) -> None:
+ text = "# Billing\n\nHandles invoicing and payment retries.\n\n## Details\n\nMore."
+ assert _extract_sentences(text, max_sentences=1) == (
+ "Handles invoicing and payment retries."
+ )
+
+ def test_skip_preamble_passes_over_a_list_only_section(self) -> None:
+ text = (
+ "# billing\n\nCovers the 6 files.\n\n"
+ "## Parts of this subsystem\n\n"
+ "- `billing/a.py` handles invoices\n"
+ "- `billing/b.py` handles refunds\n\n"
+ "## Overview\n\n"
+ "billing covers 6 python files, exposing 5 public symbols.\n"
+ )
+ got = _extract_sentences(text, max_sentences=1, skip_preamble=True)
+ assert got == "billing covers 6 python files, exposing 5 public symbols."
+
class TestTruncateAtWord:
def test_short_text_unchanged(self) -> None:
diff --git a/tests/unit/generation/test_editor_file_fetcher.py b/tests/unit/generation/test_editor_file_fetcher.py
index 7f01c7a35..2a517b1a2 100644
--- a/tests/unit/generation/test_editor_file_fetcher.py
+++ b/tests/unit/generation/test_editor_file_fetcher.py
@@ -79,7 +79,9 @@ async def _add_graph_node(session, repo_id, node_id, *, is_entry_point=False, pa
return node
-async def _add_page(session, repo_id, page_id, page_type, target_path, content):
+async def _add_page(
+ session, repo_id, page_id, page_type, target_path, content, *, provider_name="mock"
+):
page = Page(
id=page_id,
repository_id=repo_id,
@@ -89,7 +91,7 @@ async def _add_page(session, repo_id, page_id, page_type, target_path, content):
target_path=target_path,
source_hash="abc",
model_name="mock",
- provider_name="mock",
+ provider_name=provider_name,
generation_level=0,
confidence=0.9,
freshness_status="fresh",
@@ -698,3 +700,68 @@ async def test_the_default_health_line_skips_the_detail_queries(session, repo, t
assert data.code_health.worst_path == "src/a.py"
assert data.code_health.maintainability_average is None
assert data.code_health.fix_first == []
+
+
+async def test_structure_only_module_purpose_has_no_metadata_and_no_newline(
+ session, repo, tmp_path
+):
+ content = (
+ "# billing\n\n`billing`\n\n"
+ "**Language:** python | **Files:** 6 | **Public symbols:** 5 / 5\n\n"
+ "Covers the 6 source files in billing.\n\n"
+ "## Overview\n\n"
+ "billing covers 6 python files, exposing 5 public symbols.\n"
+ )
+ await _add_page(
+ session, repo.id, "module_page:billing", "module_page", "billing",
+ content, provider_name="template",
+ )
+ await session.commit()
+
+ data = await EditorFileDataFetcher(session, repo.id, tmp_path).fetch()
+
+ (module,) = data.key_modules
+ assert "**Language:**" not in module.purpose
+ assert "Covers the" not in module.purpose
+ assert "\n" not in module.purpose
+ assert module.purpose == "billing covers 6 python files, exposing 5 public symbols"
+
+
+async def test_structure_only_overview_skips_the_preamble(session, repo, tmp_path):
+ content = (
+ "# Repository Overview: fx2\n\n"
+ "**Files:** 19 | **Lines:** 43\n\n"
+ "Covers the whole repository.\n\n"
+ "## Project Summary\n\n"
+ "`fx2` is a python codebase of 19 files. Execution starts at `main.py`.\n"
+ )
+ await _add_page(
+ session,
+ repo.id,
+ "repo_overview:.",
+ "repo_overview",
+ ".",
+ content,
+ provider_name="template",
+ )
+ await session.commit()
+
+ data = await EditorFileDataFetcher(session, repo.id, tmp_path).fetch()
+
+ assert data.architecture_summary.startswith("fx2 is a python codebase of 19 files.")
+ assert "Covers the whole" not in data.architecture_summary
+ assert "\n" not in data.architecture_summary
+
+
+async def test_model_written_module_keeps_its_opening_sentence(session, repo, tmp_path):
+ content = (
+ "# billing\n\n"
+ "billing covers 6 python files, exposing 5 public symbols.\n\n"
+ "## Overview\n\nMore detail.\n"
+ )
+ await _add_page(session, repo.id, "module_page:billing", "module_page", "billing", content)
+ await session.commit()
+
+ data = await EditorFileDataFetcher(session, repo.id, tmp_path).fetch()
+
+ assert data.key_modules[0].purpose == "billing covers 6 python files, exposing 5 public symbols"