Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(vocabulary): skip fenced code blocks when harvesting headings (#2150)
Akshat malikc9df1947eddf
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.pyindex 095028429..9fbff872f 100644--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py@@ -63,6 +63,51 @@ _HEADING = re.compile(r"^\s{0,3}#{1,4}\s+(.+?)\s*#*\s*$", re.MULTILINE)# Bolded lead-ins are how glossaries are usually written: "**Blast radius** — "._BOLD_TERM = re.compile(r"\*\*([A-Z][^*\n]{2,40})\*\*")+# Opening fence: 0-3 leading spaces, then 3+ backticks or tildes, optional info+_FENCE_OPEN = re.compile(r"^[ ]{0,3}(`{3,}|~{3,})", re.MULTILINE)+++def _strip_fenced_blocks(text: str) -> str:+ """Return *text* with lines inside fenced code blocks replaced with empty strings.++ Lines inside a fenced block are replaced with empty strings so that code+ and shell comments are stripped while keeping line breaks intact.++ Both backtick fences (``` … ```) and tilde fences (~~~ … ~~~) are+ supported. The fence marker on the *opening* line determines which marker+ closes the block (CommonMark §4.5).+ """+ lines = text.split("\n")+ result: list[str] = []+ inside = False+ fence_char: str = ""+ fence_len: int = 0++ for line in lines:+ if not inside:+ m = _FENCE_OPEN.match(line)+ if m:+ inside = True+ fence_char = m.group(1)[0] # '`' or '~'+ fence_len = len(m.group(1))+ result.append(line) # keep the opening fence line itself+ else:+ result.append(line)+ else:+ stripped = line.lstrip(" ")+ # A closing fence: same character, at least as many, no info string+ if (+ stripped.startswith(fence_char * fence_len)+ and stripped.rstrip(fence_char).rstrip() == ""+ ):+ inside = False+ result.append(line) # keep the closing fence line itself+ else:+ # Replace content line with an empty string to strip code/comments.+ result.append("")++ return "\n".join(result)+_STOPWORDS = frozenset({"the",@@ -577,7 +622,10 @@ _MIN_HEADINGS_TO_JUDGE = 8def _is_release_notes(path: Path, text: str) -> bool:if _RELEASE_NOTE_NAMES.search(path.name):return True- headings = _HEADING.findall(text)+ # Strip fenced blocks so that shell/script comments (# …) inside code+ # examples are not counted as headings. Fixes #2142.+ prose = _strip_fenced_blocks(text)+ headings = _HEADING.findall(prose)if len(headings) < _MIN_HEADINGS_TO_JUDGE:return Falseversiony = sum(1 for h in headings if _VERSION_HEADING.match(h.strip()))@@ -737,9 +785,13 @@ def _harvest(rst_sections_seen += len(entries)rst_sections_undefined += sum(1 for _t, d in entries if d is None)else:+ # Strip fenced blocks before scanning so that code block comments+ # (e.g. shell script or config comments) are not harvested as headings.+ # Pass stripped prose to _definition_after_heading so search offsets align.+ prose = _strip_fenced_blocks(text)entries = [- (m.group(1), _definition_after_heading(text, m.end()))- for m in _HEADING.finditer(text)+ (m.group(1), _definition_after_heading(prose, m.end()))+ for m in _HEADING.finditer(prose)]# Bolded lead-ins read the same in both markup languages.entries += [(m.group(1), None) for m in _BOLD_TERM.finditer(text)]diff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.pyindex 4899802b0..731a27548 100644--- a/tests/unit/generation/test_vocabulary_house_terms.py+++ b/tests/unit/generation/test_vocabulary_house_terms.py@@ -1031,3 +1031,220 @@ def test_normal_heading_with_words_still_produces_a_term(assert "Cache Layer" in terms, ("A normal two-word heading must survive _is_useful and reach the output")+++# ---------------------------------------------------------------------------+# Shell comments inside fenced blocks must not be mined as headings (#2142)+# ---------------------------------------------------------------------------+#+# _HEADING.finditer runs on the raw document over the whole file with+# re.MULTILINE, so every line that starts with '#' — including shell comments+# inside ```bash … ``` blocks — looks identical to a real Markdown heading.+#+# The fix is _strip_fenced_blocks(), called before _HEADING.finditer in both+# _harvest() and _is_release_notes(). Lines inside the fence are replaced with+# empty strings (not deleted) so that m.end() byte offsets remain valid for+# _definition_after_heading.+++def test_shell_comment_in_backtick_fence_is_not_mined(tmp_path: Path) -> None:+ """A # comment inside a ```bash block must not become a house term.++ Regression for #2142: the miner saw '# Install dependencies' and treated+ it as a '## Install dependencies' heading because _HEADING matched every+ line starting with '#' without checking whether that line was inside a+ fenced code block.+ """+ root = tmp_path / "repo"+ root.mkdir()+ (root / "README.md").write_text(+ "# Ledger\n\n"+ "Here is how to install:\n\n"+ "```bash\n"+ "# Install dependencies\n"+ "npm install\n"+ "# Start the server\n"+ "npm start\n"+ "```\n",+ encoding="utf-8",+ )+ result = extract_house_terms(root)+ terms = [t.term for t in result]+ assert "Install dependencies" not in terms, (+ "Shell comment inside a backtick fence must not be mined as a heading"+ )+ assert "Start the server" not in terms, (+ "Shell comment inside a backtick fence must not be mined as a heading"+ )+++def test_shell_comment_in_tilde_fence_is_not_mined(tmp_path: Path) -> None:+ """A # comment inside a ~~~sh block must not become a house term.++ Tilde fences are valid CommonMark (§4.5) and are used by some doc tools.+ The fix must handle both backtick and tilde fence markers.+ """+ root = tmp_path / "repo"+ root.mkdir()+ (root / "README.md").write_text(+ "# Ledger\n\n"+ "~~~sh\n"+ "# Configure secrets\n"+ "export SECRET_KEY=...\n"+ "~~~\n",+ encoding="utf-8",+ )+ result = extract_house_terms(root)+ terms = [t.term for t in result]+ assert "Configure secrets" not in terms, (+ "Shell comment inside a tilde fence must not be mined as a heading"+ )+++def test_real_heading_after_fenced_block_is_still_mined(tmp_path: Path) -> None:+ """A genuine heading that follows a fenced block must still be harvested.++ Stripping fenced blocks must not accidentally erase real headings that+ appear *after* the closing fence.+ """+ root = tmp_path / "repo"+ root.mkdir()+ (root / "README.md").write_text(+ "# Ledger\n\n"+ "```bash\n"+ "# just a comment\n"+ "```\n\n"+ "## Blast radius\n\n"+ "Blast radius is the set of files a change can reach.\n",+ encoding="utf-8",+ )+ src = root / "src"+ src.mkdir()+ (src / "core.py").write_text(+ '"""Blast radius computation."""\n', encoding="utf-8"+ )+ result = extract_house_terms(root)+ terms = [t.term for t in result]+ assert "Blast radius" in terms, (+ "A real heading after a fenced block must still be harvested"+ )+++def test_heading_with_inline_backtick_span_is_not_suppressed(tmp_path: Path) -> None:+ """A heading containing an inline code span must still be harvested.++ Inline backtick spans (single backtick) are not fences. A heading like+ '## Cache Layer' that contains no backtick fences must be unaffected by the+ fence-stripping logic.+ """+ root = tmp_path / "repo"+ root.mkdir()+ (root / "README.md").write_text(+ "## Cache Layer\n\n"+ "Cache Layer stores pre-computed results.\n",+ encoding="utf-8",+ )+ src = root / "src"+ src.mkdir()+ (src / "cache.py").write_text(+ '"""Cache Layer implementation."""\n', encoding="utf-8"+ )+ result = extract_house_terms(root)+ terms = [t.term for t in result]+ assert "Cache Layer" in terms, (+ "A heading with no fenced block must still be harvested after the fix"+ )+++def test_version_comments_in_code_block_do_not_trigger_release_notes(+ tmp_path: Path,+) -> None:+ """Version-like # comments inside fenced blocks must not trigger release-note filtering.++ _is_release_notes() counts how many headings look like version numbers. If+ a changelog-style code example contains many '# v1.0.0' comments, the doc+ could be misclassified as release notes and silently skipped. The fix must+ apply to _is_release_notes() as well as _harvest().+ """+ root = tmp_path / "repo"+ root.mkdir()+ # Build a real doc that has a code block full of version comments.+ # The ratio of version-like "headings" must stay below the threshold+ # once fenced blocks are stripped.+ doc_lines = [+ "# Deployment Guide",+ "",+ "## Overview",+ "",+ "This guide explains how to deploy.",+ "",+ "## Prerequisites",+ "",+ "```bash",+ ]+ # Add 20 version-like shell comments — enough to exceed the ratio if counted.+ for i in range(20):+ doc_lines.append(f"# v1.{i}.0")+ doc_lines += [+ "```",+ "",+ "## Configuration",+ "",+ "Set your environment variables.",+ "",+ "## Installation",+ "",+ "Run the installer.",+ ]+ (root / "README.md").write_text("\n".join(doc_lines), encoding="utf-8")+ src = root / "src"+ src.mkdir()+ # Spell all three real terms in source so they pass the code-frequency gate.+ (src / "deploy.py").write_text(+ '"""Deployment Guide utilities.\n\n'+ "Configuration is read from environment variables.\n"+ "Installation runs the package installer.\n"+ '"""\n',+ encoding="utf-8",+ )+ result = extract_house_terms(root)+ terms = [t.term for t in result]+ # The document must NOT have been skipped as release notes.+ # At least one of its real headings must survive.+ # ('Overview' may be filtered as a stopword; 'Prerequisites' is also common.)+ assert "Installation" in terms or "Configuration" in terms or "Deployment Guide" in terms, (+ "A doc with version comments only in a code block must not be classified "+ "as release notes and must still yield its real headings as terms"+ )+++def test_heading_definition_after_fenced_block_is_extracted_correctly(+ tmp_path: Path,+) -> None:+ """Verify that headings occurring after fenced code blocks extract their correct definitions.++ Ensures offset indices for definition extraction align with the stripped prose text,+ preventing offsets from slicing into preceding code blocks.+ """+ root = tmp_path / "repo"+ root.mkdir()+ (root / "README.md").write_text(+ "# Ledger\n\n"+ "```bash\n"+ "# install the dependencies for the ledger service\n"+ "npm install\n"+ "```\n\n"+ "## Blast radius\n\n"+ "Blast radius is the set of files a change can reach through the import graph.\n",+ encoding="utf-8",+ )+ src = root / "src"+ src.mkdir()+ (src / "analysis.py").write_text('"""Blast radius calculation."""\n', encoding="utf-8")++ result = {t.term: t for t in extract_house_terms(root)}+ assert "Blast radius" in result+ assert result["Blast radius"].definition == (+ "Blast radius is the set of files a change can reach through the import graph."+ )+