Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(generation): read reStructuredText documents written under .txt (#1247)
Raghav Chamadiya8bf84957ed18
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.pyindex fe0786530..1a5c98db6 100644--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py@@ -281,6 +281,17 @@ _RST_UNDERLINE = re.compile(r"^([=\-`:.'\"~^_*+#<>])\1{2,}\s*$")#: ``.. note::``, ``.. _label:``, ``.. image::`` — markup, never a title._RST_DIRECTIVE = re.compile(r"^\s*\.\.\s")+#: Extensions a reStructuredText document is written under. ``.txt`` is here+#: because Sphinx's ``source_suffix`` is a project setting and a large project+#: is as likely to have set it to ``.txt`` as to have left it at ``.rst`` —+#: django writes its entire ``docs/`` tree that way. Read as markdown a+#: ``.txt`` document yields nothing at all, which is indistinguishable from a+#: repository that documents nothing.+#:+#: The cost of guessing wrong is bounded: a ``.txt`` that is not reST has no+#: title underlines, so the reST scan returns no sections rather than bad ones.+_RST_SUFFIXES = frozenset({".rst", ".txt"})+def _is_rst_underline(line: str, title: str) -> bool:"""Whether ``line`` underlines ``title``.@@ -417,7 +428,7 @@ def _definition_after_heading(text: str, offset: int) -> str | None:#: than subsystem names. A changelog is usually the largest document in a#: repository and the least useful one to mine: on this repository it consumed#: 142 of the first 200 term slots before the tool guide was ever opened.-_RELEASE_NOTE_NAMES = re.compile(r"(change ?log|releases?|news|history)\.(md|rst)$", re.I)+_RELEASE_NOTE_NAMES = re.compile(r"(change ?log|releases?|news|history)\.(md|rst|txt)$", re.I)_VERSION_HEADING = re.compile(r"^v?\d+\.\d+")#: A document has to be overwhelmingly version headings before it is dropped. A#: guide that cites a few release numbers is not release notes, and dropping a@@ -542,8 +553,9 @@ def _harvest(order: list[str] = []unreadable = 0skipped: list[str] = []+ sectionless: list[str] = []- patterns = ("*.md", "*.rst") if read_rst else ("*.md",)+ patterns = ("*.md", "*.rst", "*.txt") if read_rst else ("*.md",)for path in _doc_paths(repo_root, patterns=patterns):try:text = path.read_text(encoding="utf-8", errors="replace")[:_MAX_DOC_BYTES]@@ -573,11 +585,18 @@ def _harvest(# belongs to that heading, but the lines under a bolded term# mid-paragraph belong to the paragraph, not to the term.entries: list[tuple[str, str | None]] = []- if read_rst and path.suffix.lower() == ".rst":+ if read_rst and path.suffix.lower() in _RST_SUFFIXES:lines = text.split("\n")+ sections = _rst_sections(lines)+ if not sections:+ # A document read under a markup guess that turned out wrong+ # is indistinguishable downstream from a document with nothing+ # in it. Counted, so "we found no vocabulary" can be told+ # apart from "we read the wrong thing".+ sectionless.append(rel)entries = [(title, _definition_after_rst_section(lines, underline))- for title, underline in _rst_sections(lines)+ for title, underline in sections]else:entries = [@@ -626,6 +645,13 @@ def _harvest(len(skipped),", ".join(sorted(skipped)),)+ if sectionless:+ logger.info(+ "vocabulary: %d document(s) read as reStructuredText carried no section "+ "titles and contributed no terms: %s",+ len(sectionless),+ ", ".join(sorted(sectionless)[:10]),+ )return [candidates[k] for k in order]diff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.pyindex 30c288e2b..b1c0b2cde 100644--- a/tests/unit/generation/test_vocabulary_house_terms.py+++ b/tests/unit/generation/test_vocabulary_house_terms.py@@ -622,3 +622,82 @@ def test_extract_terms_ignores_rst_headings(rst_repo: Path) -> None:change must not alter that."""assert extract_terms(rst_repo) == ["Dead code"]+++# ---------------------------------------------------------------------------+# reStructuredText written under .txt+# ---------------------------------------------------------------------------+#+# Sphinx's ``source_suffix`` is a project setting, and a project is as free to+# set it to ``.txt`` as to leave it at ``.rst``. django writes its whole+# ``docs/`` tree that way. A glob that reads only ``.rst`` skips every one of+# those documents and reports the repository as having nothing to say.+++def txt_docs_repo(tmp_path: Path) -> Path:+ """A repository whose only document is reStructuredText named ``.txt``."""+ root = tmp_path / "ledger"+ root.mkdir()+ docs = root / "docs"+ docs.mkdir()+ (docs / "guide.txt").write_text(_RST_GUIDE, encoding="utf-8")++ src = root / "src"+ src.mkdir()+ (src / "history.py").write_text(_SRC_HISTORY, encoding="utf-8")+ return root+++def test_a_docs_directory_of_txt_is_read_as_restructuredtext(+ txt_docs_repo: Path,+) -> None:+ """The extension is the only thing that differs from ``guide.rst``."""+ co = {t.term: t for t in extract_house_terms(txt_docs_repo)}["Co-change"]+ assert co.source_paths[0] == "docs/guide.txt"+ assert co.definition == "Co-change counts how often two files land in the same commit."+++def test_a_txt_document_that_is_not_restructuredtext_yields_no_sections(+ tmp_path: Path,+) -> None:+ """Guessing wrong is bounded.++ A ``.txt`` with no title underlines has no sections, so the reST scan+ returns nothing rather than reading its prose lines as names.+ """+ root = tmp_path / "ledger"+ root.mkdir()+ docs = root / "docs"+ docs.mkdir()+ (docs / "requirements.txt").write_text(+ "Sphinx==7.2.6\nfuro==2024.1.29\nCo-change==1.0\n", encoding="utf-8"+ )+ src = root / "src"+ src.mkdir()+ (src / "history.py").write_text(_SRC_HISTORY, encoding="utf-8")++ assert extract_house_terms(root) == []+++def test_release_notes_named_txt_are_skipped(tmp_path: Path) -> None:+ """A changelog is the largest and least useful document in a repository+ whatever extension it carries."""+ root = tmp_path / "ledger"+ root.mkdir()+ docs = root / "docs"+ docs.mkdir()+ (docs / "changelog.txt").write_text(+ "Co-change\n~~~~~~~~~\n\nCo-change counts commits.\n", encoding="utf-8"+ )+ src = root / "src"+ src.mkdir()+ (src / "history.py").write_text(_SRC_HISTORY, encoding="utf-8")++ assert extract_house_terms(root) == []+++def test_extract_terms_does_not_read_txt_documents(txt_docs_repo: Path) -> None:+ """The planner's input does not move. It mines markdown only, and the+ binding it performs is measured against exactly that corpus."""+ assert extract_terms(txt_docs_repo) == []