Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(generation): read reStructuredText documents written under .txt (#1247)

Raghav Chamadiya8bf84957ed18
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
index fe0786530..1a5c98db6 100644
--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
@@ -281,6 +281,17 @@ _RST_UNDERLINE = re.compile(r"^([=\-`:.'\"~^_*+#<>])\1{2,}\s*$")
#: ``.. note::``, ``.. _label:``, ``.. image::`` — markup, never a title.
_RST_DIRECTIVE = re.compile(r"^\s*\.\.\s")
+#: Extensions a reStructuredText document is written under. ``.txt`` is here
+#: because Sphinx's ``source_suffix`` is a project setting and a large project
+#: is as likely to have set it to ``.txt`` as to have left it at ``.rst`` —
+#: django writes its entire ``docs/`` tree that way. Read as markdown a
+#: ``.txt`` document yields nothing at all, which is indistinguishable from a
+#: repository that documents nothing.
+#:
+#: The cost of guessing wrong is bounded: a ``.txt`` that is not reST has no
+#: title underlines, so the reST scan returns no sections rather than bad ones.
+_RST_SUFFIXES = frozenset({".rst", ".txt"})
+
def _is_rst_underline(line: str, title: str) -> bool:
"""Whether ``line`` underlines ``title``.
@@ -417,7 +428,7 @@ def _definition_after_heading(text: str, offset: int) -> str | None:
#: than subsystem names. A changelog is usually the largest document in a
#: repository and the least useful one to mine: on this repository it consumed
#: 142 of the first 200 term slots before the tool guide was ever opened.
-_RELEASE_NOTE_NAMES = re.compile(r"(change ?log|releases?|news|history)\.(md|rst)$", re.I)
+_RELEASE_NOTE_NAMES = re.compile(r"(change ?log|releases?|news|history)\.(md|rst|txt)$", re.I)
_VERSION_HEADING = re.compile(r"^v?\d+\.\d+")
#: A document has to be overwhelmingly version headings before it is dropped. A
#: guide that cites a few release numbers is not release notes, and dropping a
@@ -542,8 +553,9 @@ def _harvest(
order: list[str] = []
unreadable = 0
skipped: list[str] = []
+ sectionless: list[str] = []
- patterns = ("*.md", "*.rst") if read_rst else ("*.md",)
+ patterns = ("*.md", "*.rst", "*.txt") if read_rst else ("*.md",)
for path in _doc_paths(repo_root, patterns=patterns):
try:
text = path.read_text(encoding="utf-8", errors="replace")[:_MAX_DOC_BYTES]
@@ -573,11 +585,18 @@ def _harvest(
# belongs to that heading, but the lines under a bolded term
# mid-paragraph belong to the paragraph, not to the term.
entries: list[tuple[str, str | None]] = []
- if read_rst and path.suffix.lower() == ".rst":
+ if read_rst and path.suffix.lower() in _RST_SUFFIXES:
lines = text.split("\n")
+ sections = _rst_sections(lines)
+ if not sections:
+ # A document read under a markup guess that turned out wrong
+ # is indistinguishable downstream from a document with nothing
+ # in it. Counted, so "we found no vocabulary" can be told
+ # apart from "we read the wrong thing".
+ sectionless.append(rel)
entries = [
(title, _definition_after_rst_section(lines, underline))
- for title, underline in _rst_sections(lines)
+ for title, underline in sections
]
else:
entries = [
@@ -626,6 +645,13 @@ def _harvest(
len(skipped),
", ".join(sorted(skipped)),
)
+ if sectionless:
+ logger.info(
+ "vocabulary: %d document(s) read as reStructuredText carried no section "
+ "titles and contributed no terms: %s",
+ len(sectionless),
+ ", ".join(sorted(sectionless)[:10]),
+ )
return [candidates[k] for k in order]
diff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.py
index 30c288e2b..b1c0b2cde 100644
--- a/tests/unit/generation/test_vocabulary_house_terms.py
+++ b/tests/unit/generation/test_vocabulary_house_terms.py
@@ -622,3 +622,82 @@ def test_extract_terms_ignores_rst_headings(rst_repo: Path) -> None:
change must not alter that.
"""
assert extract_terms(rst_repo) == ["Dead code"]
+
+
+# ---------------------------------------------------------------------------
+# reStructuredText written under .txt
+# ---------------------------------------------------------------------------
+#
+# Sphinx's ``source_suffix`` is a project setting, and a project is as free to
+# set it to ``.txt`` as to leave it at ``.rst``. django writes its whole
+# ``docs/`` tree that way. A glob that reads only ``.rst`` skips every one of
+# those documents and reports the repository as having nothing to say.
+
+
+def txt_docs_repo(tmp_path: Path) -> Path:
+ """A repository whose only document is reStructuredText named ``.txt``."""
+ root = tmp_path / "ledger"
+ root.mkdir()
+ docs = root / "docs"
+ docs.mkdir()
+ (docs / "guide.txt").write_text(_RST_GUIDE, encoding="utf-8")
+
+ src = root / "src"
+ src.mkdir()
+ (src / "history.py").write_text(_SRC_HISTORY, encoding="utf-8")
+ return root
+
+
+def test_a_docs_directory_of_txt_is_read_as_restructuredtext(
+ txt_docs_repo: Path,
+) -> None:
+ """The extension is the only thing that differs from ``guide.rst``."""
+ co = {t.term: t for t in extract_house_terms(txt_docs_repo)}["Co-change"]
+ assert co.source_paths[0] == "docs/guide.txt"
+ assert co.definition == "Co-change counts how often two files land in the same commit."
+
+
+def test_a_txt_document_that_is_not_restructuredtext_yields_no_sections(
+ tmp_path: Path,
+) -> None:
+ """Guessing wrong is bounded.
+
+ A ``.txt`` with no title underlines has no sections, so the reST scan
+ returns nothing rather than reading its prose lines as names.
+ """
+ root = tmp_path / "ledger"
+ root.mkdir()
+ docs = root / "docs"
+ docs.mkdir()
+ (docs / "requirements.txt").write_text(
+ "Sphinx==7.2.6\nfuro==2024.1.29\nCo-change==1.0\n", encoding="utf-8"
+ )
+ src = root / "src"
+ src.mkdir()
+ (src / "history.py").write_text(_SRC_HISTORY, encoding="utf-8")
+
+ assert extract_house_terms(root) == []
+
+
+def test_release_notes_named_txt_are_skipped(tmp_path: Path) -> None:
+ """A changelog is the largest and least useful document in a repository
+ whatever extension it carries."""
+ root = tmp_path / "ledger"
+ root.mkdir()
+ docs = root / "docs"
+ docs.mkdir()
+ (docs / "changelog.txt").write_text(
+ "Co-change\n~~~~~~~~~\n\nCo-change counts commits.\n", encoding="utf-8"
+ )
+ src = root / "src"
+ src.mkdir()
+ (src / "history.py").write_text(_SRC_HISTORY, encoding="utf-8")
+
+ assert extract_house_terms(root) == []
+
+
+def test_extract_terms_does_not_read_txt_documents(txt_docs_repo: Path) -> None:
+ """The planner's input does not move. It mines markdown only, and the
+ binding it performs is measured against exactly that corpus."""
+ assert extract_terms(txt_docs_repo) == []