Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(editor-files): skip structure-only preamble in key modules and architecture (#3261)
c0mradeb95a1d32ab39
diff --git a/packages/core/src/repowise/core/generation/editor_files/fetcher.py b/packages/core/src/repowise/core/generation/editor_files/fetcher.pyindex c6e95799f..310ad8683 100644--- a/packages/core/src/repowise/core/generation/editor_files/fetcher.py+++ b/packages/core/src/repowise/core/generation/editor_files/fetcher.py@@ -160,7 +160,8 @@ class EditorFileDataFetcher:if not pages:return ""content = pages[0].content or ""- return _extract_sentences(content, max_sentences=4)+ skip_preamble = _is_structure_only(pages[0])+ return _extract_sentences(content, max_sentences=4, skip_preamble=skip_preamble)async def _get_key_modules(self) -> list[KeyModule]:"""Top modules by PageRank with owner from git metadata."""@@ -192,7 +193,10 @@ class EditorFileDataFetcher:modules: list[KeyModule] = []for page, _pagerank, symbol_count in rows:- purpose = _extract_sentences(page.content or "", max_sentences=1)+ skip_preamble = _is_structure_only(page)+ purpose = _extract_sentences(+ page.content or "", max_sentences=1, skip_preamble=skip_preamble+ )# 80 chars cut every purpose mid-thought ("…is the **test-layer# ingestion subsystem** for…"); 140 fits one real clause.purpose = _truncate_at_word(purpose, 140).rstrip(".") if purpose else ""@@ -615,7 +619,28 @@ def _get_head_short_sha(repo_path: Path) -> str:return ""-def _extract_sentences(text: str, max_sentences: int) -> str:+def _is_all_emphasis(paragraph: str) -> bool:+ """True when one emphasis span wraps the whole paragraph."""+ p = paragraph.strip()+ for marker in ("**", "*", "__", "_"):+ if len(p) > 2 * len(marker) and p.startswith(marker) and p.endswith(marker):+ inner = p[len(marker) : -len(marker)]+ if marker[0] not in inner and not inner[0].isspace():+ return True+ return False+++def _is_structure_only(page: object) -> bool:+ """True when no model wrote *page*: it was rendered from structure.++ Structure-only pages are stored with ``provider_name='template'``. Their+ preamble (directory line, ``**Language:** ...`` row, scope sentence) is+ page furniture, not prose about what the module is for.+ """+ return getattr(page, "provider_name", "") == "template"+++def _extract_sentences(text: str, max_sentences: int, *, skip_preamble: bool = False) -> str:"""Return up to *max_sentences* prose sentences from the start of *text*.Strips markdown headers/code fences AND list/table lines so only prose@@ -623,7 +648,16 @@ def _extract_sentences(text: str, max_sentences: int) -> str:them used to jam bullets onto the tail of the last real sentence("...rendered in a web UI. - **Languages**") and leave orphanedfragments like "1. **Inputs**" in the rendered CLAUDE.md.++ When *skip_preamble* is True (used for structure-only pages), all content+ prior to the first ``##`` section header is treated as template furniture+ and discarded. If the text has no ``##`` headings, it yields an empty string."""+ if skip_preamble:+ first_section = re.search(r"^##\s", text, flags=re.MULTILINE)+ if first_section is None:+ return ""+ text = text[first_section.start() :]# Remove markdown headers and code fencestext = re.sub(r"^#{1,6}\s+.*$", "", text, flags=re.MULTILINE)text = re.sub(r"```.*?```", "", text, flags=re.DOTALL)@@ -636,7 +670,21 @@ def _extract_sentences(text: str, max_sentences: int) -> str:text = re.sub(r"^.*:\s*$", "", text, flags=re.MULTILINE)text = re.sub(r"`([^`]+)`", r"\1", text) # strip backticks, keep texttext = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text) # links → text- text = re.sub(r"\n{2,}", "\n", text).strip()+ # Strips Markdown horizontal rules / divider lines (---, ***, ___)+ text = re.sub(r"^[ \t]*([-*_])(?:[ \t]*\1){2,}[ \t]*$", "", text, flags=re.MULTILINE)+ # Strip bold metadata rows; requiring '|' between 2+ segments ensures model prose+ # starting with '**Note:**' is preserved+ text = re.sub(+ r"^[ \t]*\*\*[^*\n]+:\*\*[^|\n]*(?:\|[ \t]*\*\*[^*\n]+:\*\*[^|\n]*)+$",+ "",+ text,+ flags=re.MULTILINE,+ )+ # Drop paragraphs wrapped entirely in one emphasis span+ # (the footer note on structure-only pages).+ text = "\n\n".join(p for p in re.split(r"\n\s*\n", text) if not _is_all_emphasis(p))+ # Collapse all remaining whitespace and newlines into a single flat line of prose+ text = re.sub(r"\s+", " ", text).strip()# Split on sentence boundariessentences = re.split(r"(?<=[.!?])\s+", text)diff --git a/tests/unit/generation/test_editor_fetcher_helpers.py b/tests/unit/generation/test_editor_fetcher_helpers.pyindex 5fe5ddff0..8464e36a2 100644--- a/tests/unit/generation/test_editor_fetcher_helpers.py+++ b/tests/unit/generation/test_editor_fetcher_helpers.py@@ -74,6 +74,90 @@ class TestExtractSentences:def test_empty_input(self) -> None:assert _extract_sentences("", max_sentences=3) == ""+ def test_drops_thematic_break(self) -> None:+ text = "---\n\nThe accounts module manages authentication and sessions."+ got = _extract_sentences(text, max_sentences=1)+ assert got == "The accounts module manages authentication and sessions."++ def test_drops_meta_badge_rows(self) -> None:+ text = (+ "**Language:** python | **Files:** 9 | **Public symbols:** 5 / 5\n\n"+ "The accounts module manages authentication and sessions."+ )+ got = _extract_sentences(text, max_sentences=1)+ assert got == "The accounts module manages authentication and sessions."++ def test_joins_new_lines_into_spaces(self) -> None:+ text = (+ "The accounts module manages\n"+ "authentication and active user\n"+ "sessions across requests."+ )+ got = _extract_sentences(text, max_sentences=1)+ assert "\n" not in got+ assert got == (+ "The accounts module manages authentication and active "+ "user sessions across requests."+ )++ def test_keeps_a_single_bold_label_sentence(self) -> None:+ text = "**Note:** Handles invoicing and payment retries."+ assert _extract_sentences(text, max_sentences=1) == text++ def test_drops_all_emphasis_footer(self) -> None:+ text = (+ "The accounts module manages authentication and sessions.\n\n"+ "*Built from the code's structure. It states what is there, not why it is that\n"+ "way. The explanatory prose is a separate, model-written layer.*\n"+ )+ got = _extract_sentences(text, max_sentences=4)+ assert got == "The accounts module manages authentication and sessions."++ def test_drops_thematic_break_and_all_emphasis_footer(self) -> None:+ text = (+ "---\n\n"+ "*Built from the code's structure. It states what is there, not why it is that\n"+ "way. The explanatory prose is a separate, model-written layer.*\n"+ )+ assert _extract_sentences(text, max_sentences=2) == ""++ def test_keeps_prose_that_starts_and_ends_with_bold(self) -> None:+ text = "**Billing** is used by **reports**"+ assert _extract_sentences(text, max_sentences=1) == text++ def test_skip_preamble_starts_at_first_section(self) -> None:+ text = (+ "# billing\n\n`billing`\n\n"+ "**Language:** python | **Files:** 6\n\n"+ "Covers the 6 source files in billing.\n\n"+ "## Overview\n\n"+ "billing covers 6 python files, exposing 5 public symbols.\n"+ )+ got = _extract_sentences(text, max_sentences=1, skip_preamble=True)+ assert got == "billing covers 6 python files, exposing 5 public symbols."++ def test_skip_preamble_without_a_section_returns_empty(self) -> None:+ text = "# billing\n\n**Language:** python | **Files:** 6\n\nCovers the 6 files."+ assert _extract_sentences(text, max_sentences=1, skip_preamble=True) == ""++ def test_preamble_prose_is_kept_by_default(self) -> None:+ text = "# Billing\n\nHandles invoicing and payment retries.\n\n## Details\n\nMore."+ assert _extract_sentences(text, max_sentences=1) == (+ "Handles invoicing and payment retries."+ )++ def test_skip_preamble_passes_over_a_list_only_section(self) -> None:+ text = (+ "# billing\n\nCovers the 6 files.\n\n"+ "## Parts of this subsystem\n\n"+ "- `billing/a.py` handles invoices\n"+ "- `billing/b.py` handles refunds\n\n"+ "## Overview\n\n"+ "billing covers 6 python files, exposing 5 public symbols.\n"+ )+ got = _extract_sentences(text, max_sentences=1, skip_preamble=True)+ assert got == "billing covers 6 python files, exposing 5 public symbols."+class TestTruncateAtWord:def test_short_text_unchanged(self) -> None:diff --git a/tests/unit/generation/test_editor_file_fetcher.py b/tests/unit/generation/test_editor_file_fetcher.pyindex 7f01c7a35..2a517b1a2 100644--- a/tests/unit/generation/test_editor_file_fetcher.py+++ b/tests/unit/generation/test_editor_file_fetcher.py@@ -79,7 +79,9 @@ async def _add_graph_node(session, repo_id, node_id, *, is_entry_point=False, pareturn node-async def _add_page(session, repo_id, page_id, page_type, target_path, content):+async def _add_page(+ session, repo_id, page_id, page_type, target_path, content, *, provider_name="mock"+):page = Page(id=page_id,repository_id=repo_id,@@ -89,7 +91,7 @@ async def _add_page(session, repo_id, page_id, page_type, target_path, content):target_path=target_path,source_hash="abc",model_name="mock",- provider_name="mock",+ provider_name=provider_name,generation_level=0,confidence=0.9,freshness_status="fresh",@@ -698,3 +700,68 @@ async def test_the_default_health_line_skips_the_detail_queries(session, repo, tassert data.code_health.worst_path == "src/a.py"assert data.code_health.maintainability_average is Noneassert data.code_health.fix_first == []+++async def test_structure_only_module_purpose_has_no_metadata_and_no_newline(+ session, repo, tmp_path+):+ content = (+ "# billing\n\n`billing`\n\n"+ "**Language:** python | **Files:** 6 | **Public symbols:** 5 / 5\n\n"+ "Covers the 6 source files in billing.\n\n"+ "## Overview\n\n"+ "billing covers 6 python files, exposing 5 public symbols.\n"+ )+ await _add_page(+ session, repo.id, "module_page:billing", "module_page", "billing",+ content, provider_name="template",+ )+ await session.commit()++ data = await EditorFileDataFetcher(session, repo.id, tmp_path).fetch()++ (module,) = data.key_modules+ assert "**Language:**" not in module.purpose+ assert "Covers the" not in module.purpose+ assert "\n" not in module.purpose+ assert module.purpose == "billing covers 6 python files, exposing 5 public symbols"+++async def test_structure_only_overview_skips_the_preamble(session, repo, tmp_path):+ content = (+ "# Repository Overview: fx2\n\n"+ "**Files:** 19 | **Lines:** 43\n\n"+ "Covers the whole repository.\n\n"+ "## Project Summary\n\n"+ "`fx2` is a python codebase of 19 files. Execution starts at `main.py`.\n"+ )+ await _add_page(+ session,+ repo.id,+ "repo_overview:.",+ "repo_overview",+ ".",+ content,+ provider_name="template",+ )+ await session.commit()++ data = await EditorFileDataFetcher(session, repo.id, tmp_path).fetch()++ assert data.architecture_summary.startswith("fx2 is a python codebase of 19 files.")+ assert "Covers the whole" not in data.architecture_summary+ assert "\n" not in data.architecture_summary+++async def test_model_written_module_keeps_its_opening_sentence(session, repo, tmp_path):+ content = (+ "# billing\n\n"+ "billing covers 6 python files, exposing 5 public symbols.\n\n"+ "## Overview\n\nMore detail.\n"+ )+ await _add_page(session, repo.id, "module_page:billing", "module_page", "billing", content)+ await session.commit()++ data = await EditorFileDataFetcher(session, repo.id, tmp_path).fetch()++ assert data.key_modules[0].purpose == "billing covers 6 python files, exposing 5 public symbols"