Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(generation): a definition is prose the author wrote, not the markup below it (#1248)

Raghav Chamadiya008799a0a6c6
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
index 1a5c98db6..f20f52cab 100644
--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
@@ -291,6 +291,17 @@ _RST_DIRECTIVE = re.compile(r"^\s*\.\.\s")
#: The cost of guessing wrong is bounded: a ``.txt`` that is not reST has no
#: title underlines, so the reST scan returns no sections rather than bad ones.
_RST_SUFFIXES = frozenset({".rst", ".txt"})
+#: A line that is reStructuredText markup rather than a sentence: bullet and
+#: enumerated list items, grid- and simple-table rules and rows, line blocks,
+#: and field lists or bare roles.
+#:
+#: The markdown scan has always refused these shapes — see ``_NOT_PROSE`` — and
+#: the reST scan reading them as prose is how a table of contents supplied a
+#: subsystem's definition. Mined from django, whose "Models" section opens with
+#: a grid of ``:doc:`` links, the sentence taken was
+#: ``:doc:`Introduction to models <topics/db/models>` |`` — markup, a wrong
+#: definition, and a stray pipe that would break any table it is rendered into.
+_RST_MARKUP_LINE = re.compile(r"^\s*(?:[*+\-|>:=~^#]|\d+[.)]\s)")
def _is_rst_underline(line: str, title: str) -> bool:
@@ -355,13 +366,43 @@ def _definition_after_rst_section(lines: list[str], underline: int) -> str | Non
# A line that is itself underlined is the next section's title.
if offset + 1 < len(lines) and _is_rst_underline(lines[offset + 1], stripped):
return None
- sentence = _first_sentence(_strip_rst_roles(stripped))
+ if _RST_MARKUP_LINE.match(stripped):
+ # Markup, not a sentence. Skipped rather than taken, exactly as the
+ # markdown scan skips a table row or a badge line above the prose.
+ continue
+ paragraph = _join_wrapped(lines, offset, stop)
+ sentence = _first_sentence(_strip_rst_roles(paragraph))
if _MIN_DEFINITION_CHARS <= len(sentence) <= _MAX_DEFINITION_CHARS:
return sentence
return None
return None
+def _join_wrapped(lines: list[str], start: int, stop: int) -> str:
+ """The paragraph beginning at ``start``, rejoined into one string.
+
+ reStructuredText hard-wraps prose at the column, so a line is a line and
+ not a sentence. Reading one line was how django's Forms section defined
+ itself as "Django provides a rich framework to facilitate the creation of
+ forms and the" — the author's sentence, cut where the editor wrapped it.
+
+ Stops at the paragraph break, at markup, and at the next section title,
+ which are the same boundaries the caller respects; the caller's scan
+ window bounds it.
+ """
+ parts = [lines[start].strip()]
+ for offset in range(start + 1, stop):
+ if _SENTENCE_END.search(" ".join(parts)):
+ break
+ nxt = lines[offset].strip()
+ if not nxt or _RST_MARKUP_LINE.match(nxt) or _RST_DIRECTIVE.match(lines[offset]):
+ break
+ if offset + 1 < len(lines) and _is_rst_underline(lines[offset + 1], nxt):
+ break
+ parts.append(nxt)
+ return " ".join(parts)
+
+
#: ``:doc:`quickstart``` and ``:class:`Flask``` are cross-references. The text
#: inside is what a reader sees, so that is what a definition should carry.
_RST_ROLE = re.compile(r":[a-zA-Z:+-]+:`([^`]*)`")
@@ -389,7 +430,18 @@ _DEFINITION_SCAN_LINES = 5
_SENTENCE_END = re.compile(r"(?<=[.!?])(?:\s|$)")
# A bolded lead-in that carries its own definition: "**Blast radius** — ...".
-_BOLD_DEFINITION = re.compile(r"\*\*([A-Z][^*\n]{2,40})\*\*\s*[—–:-]\s*([^\n]{10,300})")
+#
+# The gap around the separator is horizontal whitespace, not ``\s``: a lead-in
+# and the sentence that defines it are on one line, and that is the whole shape
+# this pattern is for. Letting the gap cross a newline turned a bolded label
+# standing alone on its line into a claim on whatever the next line held. On
+# django's contents page, "**Models:**" followed by a row of links produced
+#
+# Models -> "doc:`Introduction to models <topics/db/models>` |"
+#
+# where the separator matched the colon of ``:doc:`` — markup, a wrong
+# definition, and a stray pipe that breaks any table it is rendered into.
+_BOLD_DEFINITION = re.compile(r"\*\*([A-Z][^*\n]{2,40})\*\*[ \t]*[—–:-][ \t]*([^\n]{10,300})")
def _first_sentence(line: str) -> str:
@@ -554,6 +606,8 @@ def _harvest(
unreadable = 0
skipped: list[str] = []
sectionless: list[str] = []
+ rst_sections_seen = 0
+ rst_sections_undefined = 0
patterns = ("*.md", "*.rst", "*.txt") if read_rst else ("*.md",)
for path in _doc_paths(repo_root, patterns=patterns):
@@ -598,6 +652,8 @@ def _harvest(
(title, _definition_after_rst_section(lines, underline))
for title, underline in sections
]
+ rst_sections_seen += len(entries)
+ rst_sections_undefined += sum(1 for _t, d in entries if d is None)
else:
entries = [
(m.group(1), _definition_after_heading(text, m.end()))
@@ -652,6 +708,16 @@ def _harvest(
len(sectionless),
", ".join(sorted(sectionless)[:10]),
)
+ if rst_sections_seen:
+ # A term with no sentence is a supported outcome, but a document whose
+ # every section is markup is usually a table of contents rather than
+ # prose — worth being able to see rather than inferring from an
+ # unexplained absence of definitions.
+ logger.info(
+ "vocabulary: %d of %d reStructuredText sections carried no defining sentence",
+ rst_sections_undefined,
+ rst_sections_seen,
+ )
return [candidates[k] for k in order]
diff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.py
index b1c0b2cde..dbb131268 100644
--- a/tests/unit/generation/test_vocabulary_house_terms.py
+++ b/tests/unit/generation/test_vocabulary_house_terms.py
@@ -616,6 +616,130 @@ def test_an_overlined_title_is_read_once(rst_repo: Path) -> None:
assert all(t.strip("=-~ ") for t in terms)
+# ---------------------------------------------------------------------------
+# A reStructuredText section that opens with markup has no definition
+# ---------------------------------------------------------------------------
+#
+# Mined from django, whose ``docs/index.txt`` is a table of contents: its
+# "Models" section opens with a grid of ``:doc:`` links, and the sentence taken
+# from it was ``:doc:`Introduction to models <topics/db/models>` |``. Markup, a
+# wrong definition, and a stray pipe that breaks any table it lands in.
+
+_RST_TOC = """\
+Models
+======
+
+:doc:`Introduction to models <topics/db/models>` |
+:doc:`Field types <ref/models/fields>` |
+:doc:`Indexes <ref/models/indexes>`
+
+* **Hotspots:**
+ :doc:`Churn ranking <ref/hotspots>` |
+ :doc:`Bug history <ref/history>`
+
+Co-change
+~~~~~~~~~
+
+Co-change counts how often two files land in the same commit.
+
+Bug magnet
+~~~~~~~~~~
+
+A bug magnet is a file that has absorbed an unusual share of the
+repository's bug fixes over its lifetime.
+
+Blast radius
+~~~~~~~~~~~~
+
++------------------+-------------------+
+| Column | Meaning |
++==================+===================+
+| depth | hops from the seed|
++------------------+-------------------+
+"""
+
+
+def toc_repo(tmp_path: Path) -> Path:
+ root = tmp_path / "ledger"
+ root.mkdir()
+ docs = root / "docs"
+ docs.mkdir()
+ (docs / "index.rst").write_text(_RST_TOC, encoding="utf-8")
+
+ src = root / "src"
+ src.mkdir()
+ (src / "history.py").write_text(_SRC_HISTORY, encoding="utf-8")
+ hotspots = src / "hotspots"
+ hotspots.mkdir()
+ (hotspots / "rank.py").write_text('"""Churn ranking for hotspots."""\n', encoding="utf-8")
+ (src / "magnet.py").write_text('"""Scoring for the bug magnet view."""\n', encoding="utf-8")
+ # Both spell their term, so both pass the code gate; neither *opens* a
+ # sentence with it, so neither supplies a definition of its own. What is
+ # left is whatever the document offered — which is the point of the tests.
+ (src / "radius.py").write_text(
+ '"""Rendering for the blast radius overlay."""\n', encoding="utf-8"
+ )
+ # A directory named for the term: how a one-word term earns its place.
+ models = src / "models"
+ models.mkdir()
+ (models / "base.py").write_text('"""Declarations that back the models."""\n', encoding="utf-8")
+ return root
+
+
+def test_a_toctree_entry_is_not_a_definition(toc_repo: Path) -> None:
+ """A section whose opening lines are list markup defines nothing.
+
+ Reporting no sentence is correct. Reporting a link out of a contents grid
+ as the repository's own definition is confidently wrong, which is the one
+ outcome this module exists to avoid.
+ """
+ models = {t.term: t for t in extract_house_terms(toc_repo)}["Models"]
+ assert models.definition is None
+
+
+def test_a_definition_never_carries_a_table_cell_separator(toc_repo: Path) -> None:
+ """Definitions are rendered into markdown tables downstream."""
+ for term in extract_house_terms(toc_repo):
+ assert "|" not in (term.definition or "")
+
+
+def test_a_grid_table_row_is_not_a_definition(toc_repo: Path) -> None:
+ blast = {t.term: t for t in extract_house_terms(toc_repo)}["Blast radius"]
+ assert blast.definition is None
+
+
+def test_a_bolded_label_alone_on_its_line_claims_no_definition(toc_repo: Path) -> None:
+ """A lead-in and the sentence defining it share a line.
+
+ A bolded label standing alone above a list of links defines nothing, and
+ reading the next line as its meaning is how ``:doc:`` markup became a
+ definition of "Hotspots".
+ """
+ hotspots = {t.term: t for t in extract_house_terms(toc_repo)}["Hotspots"]
+ assert hotspots.definition is None
+
+
+def test_a_hard_wrapped_sentence_is_read_whole(toc_repo: Path) -> None:
+ """reStructuredText wraps prose at the column, so a line is not a sentence.
+
+ Reading one line is how django's Forms section defined itself as "Django
+ provides a rich framework to facilitate the creation of forms and the" —
+ the author's sentence, cut where the editor wrapped it.
+ """
+ magnet = {t.term: t for t in extract_house_terms(toc_repo)}["Bug magnet"]
+ assert magnet.definition == (
+ "A bug magnet is a file that has absorbed an unusual share of the "
+ "repository's bug fixes over its lifetime."
+ )
+
+
+def test_prose_after_the_section_title_is_still_taken(toc_repo: Path) -> None:
+ """Skipping markup must not skip the sentence that follows it."""
+ co = {t.term: t for t in extract_house_terms(toc_repo)}["Co-change"]
+ assert co.definition == "Co-change counts how often two files land in the same commit."
+
+
def test_extract_terms_ignores_rst_headings(rst_repo: Path) -> None:
"""The planner's input does not move. README.rst is already read today and
its bolded lead-ins already count; its section titles do not, and this