Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(generation): a definition is prose the author wrote, not the markup below it (#1248)
Raghav Chamadiya008799a0a6c6
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.pyindex 1a5c98db6..f20f52cab 100644--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py@@ -291,6 +291,17 @@ _RST_DIRECTIVE = re.compile(r"^\s*\.\.\s")#: The cost of guessing wrong is bounded: a ``.txt`` that is not reST has no#: title underlines, so the reST scan returns no sections rather than bad ones._RST_SUFFIXES = frozenset({".rst", ".txt"})+#: A line that is reStructuredText markup rather than a sentence: bullet and+#: enumerated list items, grid- and simple-table rules and rows, line blocks,+#: and field lists or bare roles.+#:+#: The markdown scan has always refused these shapes — see ``_NOT_PROSE`` — and+#: the reST scan reading them as prose is how a table of contents supplied a+#: subsystem's definition. Mined from django, whose "Models" section opens with+#: a grid of ``:doc:`` links, the sentence taken was+#: ``:doc:`Introduction to models <topics/db/models>` |`` — markup, a wrong+#: definition, and a stray pipe that would break any table it is rendered into.+_RST_MARKUP_LINE = re.compile(r"^\s*(?:[*+\-|>:=~^#]|\d+[.)]\s)")def _is_rst_underline(line: str, title: str) -> bool:@@ -355,13 +366,43 @@ def _definition_after_rst_section(lines: list[str], underline: int) -> str | Non# A line that is itself underlined is the next section's title.if offset + 1 < len(lines) and _is_rst_underline(lines[offset + 1], stripped):return None- sentence = _first_sentence(_strip_rst_roles(stripped))+ if _RST_MARKUP_LINE.match(stripped):+ # Markup, not a sentence. Skipped rather than taken, exactly as the+ # markdown scan skips a table row or a badge line above the prose.+ continue+ paragraph = _join_wrapped(lines, offset, stop)+ sentence = _first_sentence(_strip_rst_roles(paragraph))if _MIN_DEFINITION_CHARS <= len(sentence) <= _MAX_DEFINITION_CHARS:return sentencereturn Nonereturn None+def _join_wrapped(lines: list[str], start: int, stop: int) -> str:+ """The paragraph beginning at ``start``, rejoined into one string.++ reStructuredText hard-wraps prose at the column, so a line is a line and+ not a sentence. Reading one line was how django's Forms section defined+ itself as "Django provides a rich framework to facilitate the creation of+ forms and the" — the author's sentence, cut where the editor wrapped it.++ Stops at the paragraph break, at markup, and at the next section title,+ which are the same boundaries the caller respects; the caller's scan+ window bounds it.+ """+ parts = [lines[start].strip()]+ for offset in range(start + 1, stop):+ if _SENTENCE_END.search(" ".join(parts)):+ break+ nxt = lines[offset].strip()+ if not nxt or _RST_MARKUP_LINE.match(nxt) or _RST_DIRECTIVE.match(lines[offset]):+ break+ if offset + 1 < len(lines) and _is_rst_underline(lines[offset + 1], nxt):+ break+ parts.append(nxt)+ return " ".join(parts)++#: ``:doc:`quickstart``` and ``:class:`Flask``` are cross-references. The text#: inside is what a reader sees, so that is what a definition should carry._RST_ROLE = re.compile(r":[a-zA-Z:+-]+:`([^`]*)`")@@ -389,7 +430,18 @@ _DEFINITION_SCAN_LINES = 5_SENTENCE_END = re.compile(r"(?<=[.!?])(?:\s|$)")# A bolded lead-in that carries its own definition: "**Blast radius** — ...".-_BOLD_DEFINITION = re.compile(r"\*\*([A-Z][^*\n]{2,40})\*\*\s*[—–:-]\s*([^\n]{10,300})")+#+# The gap around the separator is horizontal whitespace, not ``\s``: a lead-in+# and the sentence that defines it are on one line, and that is the whole shape+# this pattern is for. Letting the gap cross a newline turned a bolded label+# standing alone on its line into a claim on whatever the next line held. On+# django's contents page, "**Models:**" followed by a row of links produced+#+# Models -> "doc:`Introduction to models <topics/db/models>` |"+#+# where the separator matched the colon of ``:doc:`` — markup, a wrong+# definition, and a stray pipe that breaks any table it is rendered into.+_BOLD_DEFINITION = re.compile(r"\*\*([A-Z][^*\n]{2,40})\*\*[ \t]*[—–:-][ \t]*([^\n]{10,300})")def _first_sentence(line: str) -> str:@@ -554,6 +606,8 @@ def _harvest(unreadable = 0skipped: list[str] = []sectionless: list[str] = []+ rst_sections_seen = 0+ rst_sections_undefined = 0patterns = ("*.md", "*.rst", "*.txt") if read_rst else ("*.md",)for path in _doc_paths(repo_root, patterns=patterns):@@ -598,6 +652,8 @@ def _harvest((title, _definition_after_rst_section(lines, underline))for title, underline in sections]+ rst_sections_seen += len(entries)+ rst_sections_undefined += sum(1 for _t, d in entries if d is None)else:entries = [(m.group(1), _definition_after_heading(text, m.end()))@@ -652,6 +708,16 @@ def _harvest(len(sectionless),", ".join(sorted(sectionless)[:10]),)+ if rst_sections_seen:+ # A term with no sentence is a supported outcome, but a document whose+ # every section is markup is usually a table of contents rather than+ # prose — worth being able to see rather than inferring from an+ # unexplained absence of definitions.+ logger.info(+ "vocabulary: %d of %d reStructuredText sections carried no defining sentence",+ rst_sections_undefined,+ rst_sections_seen,+ )return [candidates[k] for k in order]diff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.pyindex b1c0b2cde..dbb131268 100644--- a/tests/unit/generation/test_vocabulary_house_terms.py+++ b/tests/unit/generation/test_vocabulary_house_terms.py@@ -616,6 +616,130 @@ def test_an_overlined_title_is_read_once(rst_repo: Path) -> None:assert all(t.strip("=-~ ") for t in terms)+# ---------------------------------------------------------------------------+# A reStructuredText section that opens with markup has no definition+# ---------------------------------------------------------------------------+#+# Mined from django, whose ``docs/index.txt`` is a table of contents: its+# "Models" section opens with a grid of ``:doc:`` links, and the sentence taken+# from it was ``:doc:`Introduction to models <topics/db/models>` |``. Markup, a+# wrong definition, and a stray pipe that breaks any table it lands in.++_RST_TOC = """\+Models+======++:doc:`Introduction to models <topics/db/models>` |+:doc:`Field types <ref/models/fields>` |+:doc:`Indexes <ref/models/indexes>`++* **Hotspots:**+ :doc:`Churn ranking <ref/hotspots>` |+ :doc:`Bug history <ref/history>`++Co-change+~~~~~~~~~++Co-change counts how often two files land in the same commit.++Bug magnet+~~~~~~~~~~++A bug magnet is a file that has absorbed an unusual share of the+repository's bug fixes over its lifetime.++Blast radius+~~~~~~~~~~~~+++------------------+-------------------++| Column | Meaning |++==================+===================++| depth | hops from the seed|++------------------+-------------------++"""+++def toc_repo(tmp_path: Path) -> Path:+ root = tmp_path / "ledger"+ root.mkdir()+ docs = root / "docs"+ docs.mkdir()+ (docs / "index.rst").write_text(_RST_TOC, encoding="utf-8")++ src = root / "src"+ src.mkdir()+ (src / "history.py").write_text(_SRC_HISTORY, encoding="utf-8")+ hotspots = src / "hotspots"+ hotspots.mkdir()+ (hotspots / "rank.py").write_text('"""Churn ranking for hotspots."""\n', encoding="utf-8")+ (src / "magnet.py").write_text('"""Scoring for the bug magnet view."""\n', encoding="utf-8")+ # Both spell their term, so both pass the code gate; neither *opens* a+ # sentence with it, so neither supplies a definition of its own. What is+ # left is whatever the document offered — which is the point of the tests.+ (src / "radius.py").write_text(+ '"""Rendering for the blast radius overlay."""\n', encoding="utf-8"+ )+ # A directory named for the term: how a one-word term earns its place.+ models = src / "models"+ models.mkdir()+ (models / "base.py").write_text('"""Declarations that back the models."""\n', encoding="utf-8")+ return root+++def test_a_toctree_entry_is_not_a_definition(toc_repo: Path) -> None:+ """A section whose opening lines are list markup defines nothing.++ Reporting no sentence is correct. Reporting a link out of a contents grid+ as the repository's own definition is confidently wrong, which is the one+ outcome this module exists to avoid.+ """+ models = {t.term: t for t in extract_house_terms(toc_repo)}["Models"]+ assert models.definition is None+++def test_a_definition_never_carries_a_table_cell_separator(toc_repo: Path) -> None:+ """Definitions are rendered into markdown tables downstream."""+ for term in extract_house_terms(toc_repo):+ assert "|" not in (term.definition or "")+++def test_a_grid_table_row_is_not_a_definition(toc_repo: Path) -> None:+ blast = {t.term: t for t in extract_house_terms(toc_repo)}["Blast radius"]+ assert blast.definition is None+++def test_a_bolded_label_alone_on_its_line_claims_no_definition(toc_repo: Path) -> None:+ """A lead-in and the sentence defining it share a line.++ A bolded label standing alone above a list of links defines nothing, and+ reading the next line as its meaning is how ``:doc:`` markup became a+ definition of "Hotspots".+ """+ hotspots = {t.term: t for t in extract_house_terms(toc_repo)}["Hotspots"]+ assert hotspots.definition is None+++def test_a_hard_wrapped_sentence_is_read_whole(toc_repo: Path) -> None:+ """reStructuredText wraps prose at the column, so a line is not a sentence.++ Reading one line is how django's Forms section defined itself as "Django+ provides a rich framework to facilitate the creation of forms and the" —+ the author's sentence, cut where the editor wrapped it.+ """+ magnet = {t.term: t for t in extract_house_terms(toc_repo)}["Bug magnet"]+ assert magnet.definition == (+ "A bug magnet is a file that has absorbed an unusual share of the "+ "repository's bug fixes over its lifetime."+ )+++def test_prose_after_the_section_title_is_still_taken(toc_repo: Path) -> None:+ """Skipping markup must not skip the sentence that follows it."""+ co = {t.term: t for t in extract_house_terms(toc_repo)}["Co-change"]+ assert co.definition == "Co-change counts how often two files land in the same commit."++def test_extract_terms_ignores_rst_headings(rst_repo: Path) -> None:"""The planner's input does not move. README.rst is already read today andits bolded lead-ins already count; its section titles do not, and this