Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(generation): read reStructuredText documents as reStructuredText (#1238)
Swati Ahujaf4de0afb1b2b
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.pyindex 50c4ffe98..541e1b830 100644--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py@@ -235,7 +235,7 @@ class HouseTerm:is_indexed_symbol: bool-def _doc_paths(repo_root: Path) -> list[Path]:+def _doc_paths(repo_root: Path, *, patterns: tuple[str, ...] = ("*.md",)) -> list[Path]:"""The documents worth mining, in a fixed order. Never raises."""files: list[Path] = []try:@@ -247,7 +247,10 @@ def _doc_paths(repo_root: Path) -> list[Path]:directory = repo_root / rel_dirif not directory.is_dir():continue- for candidate in sorted(directory.glob("*.md")):+ found: list[Path] = []+ for pattern in patterns:+ found.extend(directory.glob(pattern))+ for candidate in sorted(found):if candidate.is_file() and candidate not in files:files.append(candidate)if len(files) >= _MAX_DOCS:@@ -263,6 +266,103 @@ def _doc_paths(repo_root: Path) -> list[Path]:return files+# ---------------------------------------------------------------------------+# reStructuredText+# ---------------------------------------------------------------------------+#+# A large share of Python projects document in reStructuredText, and every+# pattern above is markdown-only. Read as markdown, a reST document yields no+# headings at all — not an error, just silence, which is the worst shape a+# failure can take here. Of seven repositories checked, five were in this+# position: flask, requests, django, sphinx and reflex.++#: The punctuation reST uses to underline a title. Any of these, repeated.+_RST_UNDERLINE = re.compile(r"^([=\-`:.'\"~^_*+#<>])\1{2,}\s*$")+#: ``.. note::``, ``.. _label:``, ``.. image::`` — markup, never a title.+_RST_DIRECTIVE = re.compile(r"^\s*\.\.\s")+++def _is_rst_underline(line: str, title: str) -> bool:+ """Whether ``line`` underlines ``title``.++ reStructuredText requires the underline to be at least as long as the+ title it carries. Checking that is what separates a real section title+ from a line of prose that happens to sit above a horizontal rule or a+ row of dashes in a table.+ """+ if not _RST_UNDERLINE.match(line):+ return False+ return len(line.rstrip()) >= len(title.rstrip())+++def _rst_sections(lines: list[str]) -> list[tuple[str, int]]:+ """``(title, index of its underline)`` for each section title.++ The overline form — punctuation above *and* below the title — falls out+ for free: the overline is not a title (the line under it is not an+ underline of it), and the title/underline pair below matches normally.+ """+ out: list[tuple[str, int]] = []+ for i in range(len(lines) - 1):+ title = lines[i].strip()+ if not title or _RST_DIRECTIVE.match(lines[i]) or _RST_UNDERLINE.match(title):+ continue+ if _is_rst_underline(lines[i + 1], title):+ out.append((title, i + 1))+ return out+++def _definition_after_rst_section(lines: list[str], underline: int) -> str | None:+ """The first prose sentence of a reST section.++ Stops at the next title for the same reason the markdown scan does: a+ sentence lifted from the following section is a confident wrong+ definition, which is worse than none.+ """+ directive_indent: int | None = None+ stop = min(underline + 1 + _DEFINITION_SCAN_LINES, len(lines))+ for offset in range(underline + 1, stop):+ line = lines[offset]+ stripped = line.strip()+ if not stripped:+ continue+ indent = len(line) - len(line.lstrip())+ if directive_indent is not None:+ # Still inside the directive: its body is indented under it.+ if indent > directive_indent:+ continue+ directive_indent = None+ if _RST_DIRECTIVE.match(line):+ # A directive and everything it contains is markup, not prose.+ # ``.. note::`` happens to hold a sentence; ``.. code-block::``+ # holds code and ``.. toctree::`` holds filenames, and a+ # definition scan cannot tell them apart from the outside.+ directive_indent = indent+ continue+ if _RST_UNDERLINE.match(stripped):+ continue+ # A line that is itself underlined is the next section's title.+ if offset + 1 < len(lines) and _is_rst_underline(lines[offset + 1], stripped):+ return None+ sentence = _first_sentence(_strip_rst_roles(stripped))+ if _MIN_DEFINITION_CHARS <= len(sentence) <= _MAX_DEFINITION_CHARS:+ return sentence+ return None+ return None+++#: ``:doc:`quickstart``` and ``:class:`Flask``` are cross-references. The text+#: inside is what a reader sees, so that is what a definition should carry.+_RST_ROLE = re.compile(r":[a-zA-Z:+-]+:`([^`]*)`")+_RST_LINK = re.compile(r"`([^`<]*?)\s*(?:<[^>]*>)?`_+")+++def _strip_rst_roles(text: str) -> str:+ text = _RST_ROLE.sub(r"\1", text)+ text = _RST_LINK.sub(r"\1", text)+ return re.sub(r"``([^`]*)``", r"\1", text)++# A line that opens a new section. The definition scan stops here: running on# into the next section hands back a sentence about a different subject, which# reads as authoritative and is wrong.@@ -421,6 +521,7 @@ def _harvest(expand: Callable[[str], list[str]] | None = None,accept: Callable[[str], bool] | None = None,skip_release_notes: bool = False,+ read_rst: bool = False,) -> list[_Candidate]:"""Every candidate term in document order, deduplicated, first spelling wins.@@ -434,13 +535,16 @@ def _harvest(``expand`` turns one raw heading into several candidate spellings.``accept`` is an extra test applied after :func:`_is_useful`.``skip_release_notes`` drops documents whose headings are version numbers.+ ``read_rst`` reads reStructuredText documents as reStructuredText rather+ than looking for markdown headings in them and finding none."""candidates: dict[str, _Candidate] = {}order: list[str] = []unreadable = 0skipped: list[str] = []- for path in _doc_paths(repo_root):+ patterns = ("*.md", "*.rst") if read_rst else ("*.md",)+ for path in _doc_paths(repo_root, patterns=patterns):try:text = path.read_text(encoding="utf-8", errors="replace")[:_MAX_DOC_BYTES]except OSError:@@ -463,17 +567,30 @@ def _harvest(}# Headings first, then bolded lead-ins — the order the planner's- # binding was measured against. Which of the two a term came from- # decides where its definition may be looked for: the sentence under a- # heading belongs to that heading, but the lines under a bolded term+ # binding was measured against. Each entry carries the definition+ # already resolved, because where a definition may be looked for+ # depends on where the term came from: the sentence under a heading+ # belongs to that heading, but the lines under a bolded term# mid-paragraph belong to the paragraph, not to the term.- matches = [(True, m) for m in _HEADING.finditer(text)]- matches += [(False, m) for m in _BOLD_TERM.finditer(text)]- for is_heading, match in matches:+ entries: list[tuple[str, str | None]] = []+ if read_rst and path.suffix.lower() == ".rst":+ lines = text.split("\n")+ entries = [+ (title, _definition_after_rst_section(lines, underline))+ for title, underline in _rst_sections(lines)+ ]+ else:+ entries = [+ (m.group(1), _definition_after_heading(text, m.end()))+ for m in _HEADING.finditer(text)+ ]+ # Bolded lead-ins read the same in both markup languages.+ entries += [(m.group(1), None) for m in _BOLD_TERM.finditer(text)]++ for raw, heading_definition in entries:# Expansion runs on the raw heading, before normalisation:# normalising strips the punctuation — the colon, the bullet —# that says where the name ends and the gloss begins.- raw = match.group(1)spellings = expand(raw) if expand is not None else [raw]for spelling in spellings:term = _normalise(spelling)@@ -493,9 +610,7 @@ def _harvest(if rel not in candidate.source_paths:candidate.source_paths.append(rel)if candidate.definition is None:- definition = definitions.get(key)- if not definition and is_heading:- definition = _definition_after_heading(text, match.end())+ definition = definitions.get(key) or heading_definitionif definition:candidate.definition = definitioncandidate.definition_source = rel@@ -737,6 +852,7 @@ def extract_house_terms(expand=_expand_heading,accept=lambda t: _is_name_like(t, excluded=excluded),skip_release_notes=True,+ read_rst=True,)if not candidates:_warn_empty(repo_root)diff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.pyindex 176b4660e..30c288e2b 100644--- a/tests/unit/generation/test_vocabulary_house_terms.py+++ b/tests/unit/generation/test_vocabulary_house_terms.py@@ -480,3 +480,145 @@ def test_documents_read_but_nothing_survived_is_a_different_report(with caplog.at_level(logging.WARNING, logger=_LOGGER):assert extract_house_terms(tmp_path) == []assert "none survived" in caplog.text+++# ---------------------------------------------------------------------------+# reStructuredText+# ---------------------------------------------------------------------------+#+# Read as markdown, a reStructuredText document yields no headings at all —+# not an error, just silence. Of the repositories checked by hand, flask,+# requests and django were all in this position.++_RST_README = """\+=======+Ledger+=======++A ledger for small shops.++Blast radius+============++Blast radius is the set of files a change can reach through the import graph.++Change risk+-----------++.. note::++ Change risk scores a diff against the history of the files it touches.++Empty section+-------------++Bug magnet+----------++See :doc:`hotspots` and the `guide <https://example.com>`_ for ``details``.++**Dead code** -- code no import path reaches.++Not a heading+some prose that happens to sit above a short rule+---+"""++_RST_GUIDE = """\+Co-change+~~~~~~~~~++Co-change counts how often two files land in the same commit.+"""+++def rst_repo(tmp_path: Path) -> Path:+ root = tmp_path / "ledger"+ root.mkdir()+ (root / "README.rst").write_text(_RST_README, encoding="utf-8")+ docs = root / "docs"+ docs.mkdir()+ (docs / "guide.rst").write_text(_RST_GUIDE, encoding="utf-8")++ src = root / "src"+ src.mkdir()+ (src / "analysis.py").write_text(_SRC_ANALYSIS, encoding="utf-8")+ (src / "history.py").write_text(_SRC_HISTORY, encoding="utf-8")+ (src / "magnet.py").write_text('"""Bug magnet scoring."""\n', encoding="utf-8")+ (src / "dead.py").write_text('"""Dead code sweep."""\n', encoding="utf-8")+ return root+++def test_restructuredtext_headings_are_read(rst_repo: Path) -> None:+ found = {t.term for t in extract_house_terms(rst_repo)}+ assert {"Blast radius", "Change risk", "Bug magnet", "Co-change"} <= found+++def test_a_docs_directory_of_rst_is_mined(rst_repo: Path) -> None:+ """docs/guide.rst is the only place "Co-change" is named."""+ co = {t.term: t for t in extract_house_terms(rst_repo)}["Co-change"]+ assert co.source_paths[0] == "docs/guide.rst"+++def test_an_rst_section_yields_its_following_sentence(rst_repo: Path) -> None:+ blast = {t.term: t for t in extract_house_terms(rst_repo)}["Blast radius"]+ assert blast.definition == (+ "Blast radius is the set of files a change can reach through the import graph."+ )+ assert blast.definition_source == "README.rst"+++def test_a_directive_is_markup_not_prose(rst_repo: Path) -> None:+ """A directive and everything indented under it is markup.++ ``.. note::`` happens to hold a sentence, so taking it would look right+ here. ``.. code-block::`` holds code and ``.. toctree::`` holds+ filenames, and the scan cannot tell them apart from the outside — so it+ takes none of them.+ """+ by_name = {t.term: t for t in extract_house_terms(rst_repo)}+ assert "Note" not in by_name+ # The only sentence about "Change risk" in the document sits inside the+ # note, and is not taken. The definition it ends up with comes from the+ # code, which is the documented fallback.+ risk = by_name["Change risk"]+ assert risk.definition_source == "src/analysis.py"+++def test_the_rst_definition_scan_stops_at_the_next_section(rst_repo: Path) -> None:+ """ "Empty section" has no prose of its own; the sentence below it belongs+ to "Bug magnet"."""+ magnet = {t.term: t for t in extract_house_terms(rst_repo)}["Bug magnet"]+ assert magnet.definition is not None+ assert "Bug magnet" not in (magnet.definition or "")+++def test_rst_roles_and_links_are_reduced_to_their_text(rst_repo: Path) -> None:+ """A definition is prose for a reader, not markup."""+ magnet = {t.term: t for t in extract_house_terms(rst_repo)}["Bug magnet"]+ assert ":doc:" not in (magnet.definition or "")+ assert "``" not in (magnet.definition or "")+ assert "<https://example.com>" not in (magnet.definition or "")+++def test_a_short_rule_does_not_make_the_line_above_a_title(rst_repo: Path) -> None:+ """reStructuredText requires the underline to be at least as long as its+ title. Without that check every table rule and thematic break in the+ document becomes a heading."""+ assert "Not a heading" not in {t.term for t in extract_house_terms(rst_repo)}+++def test_an_overlined_title_is_read_once(rst_repo: Path) -> None:+ """The overline form is punctuation above *and* below. It must not yield+ a term made of punctuation, and must not double-count the title."""+ terms = [t.term for t in extract_house_terms(rst_repo)]+ assert all(t.strip("=-~ ") for t in terms)+++def test_extract_terms_ignores_rst_headings(rst_repo: Path) -> None:+ """The planner's input does not move. README.rst is already read today and+ its bolded lead-ins already count; its section titles do not, and this+ change must not alter that.+ """+ assert extract_terms(rst_repo) == ["Dead code"]