Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(vocabulary): skip fenced code blocks when harvesting headings (#2150)

Akshat malikc9df1947eddf
diff --git a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
index 095028429..9fbff872f 100644
--- a/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
+++ b/packages/core/src/repowise/core/generation/concept_tree/vocabulary.py
@@ -63,6 +63,51 @@ _HEADING = re.compile(r"^\s{0,3}#{1,4}\s+(.+?)\s*#*\s*$", re.MULTILINE)
# Bolded lead-ins are how glossaries are usually written: "**Blast radius** — ".
_BOLD_TERM = re.compile(r"\*\*([A-Z][^*\n]{2,40})\*\*")
+# Opening fence: 0-3 leading spaces, then 3+ backticks or tildes, optional info
+_FENCE_OPEN = re.compile(r"^[ ]{0,3}(`{3,}|~{3,})", re.MULTILINE)
+
+
+def _strip_fenced_blocks(text: str) -> str:
+ """Return *text* with lines inside fenced code blocks replaced with empty strings.
+
+ Lines inside a fenced block are replaced with empty strings so that code
+ and shell comments are stripped while keeping line breaks intact.
+
+ Both backtick fences (``` … ```) and tilde fences (~~~ … ~~~) are
+ supported. The fence marker on the *opening* line determines which marker
+ closes the block (CommonMark §4.5).
+ """
+ lines = text.split("\n")
+ result: list[str] = []
+ inside = False
+ fence_char: str = ""
+ fence_len: int = 0
+
+ for line in lines:
+ if not inside:
+ m = _FENCE_OPEN.match(line)
+ if m:
+ inside = True
+ fence_char = m.group(1)[0] # '`' or '~'
+ fence_len = len(m.group(1))
+ result.append(line) # keep the opening fence line itself
+ else:
+ result.append(line)
+ else:
+ stripped = line.lstrip(" ")
+ # A closing fence: same character, at least as many, no info string
+ if (
+ stripped.startswith(fence_char * fence_len)
+ and stripped.rstrip(fence_char).rstrip() == ""
+ ):
+ inside = False
+ result.append(line) # keep the closing fence line itself
+ else:
+ # Replace content line with an empty string to strip code/comments.
+ result.append("")
+
+ return "\n".join(result)
+
_STOPWORDS = frozenset(
{
"the",
@@ -577,7 +622,10 @@ _MIN_HEADINGS_TO_JUDGE = 8
def _is_release_notes(path: Path, text: str) -> bool:
if _RELEASE_NOTE_NAMES.search(path.name):
return True
- headings = _HEADING.findall(text)
+ # Strip fenced blocks so that shell/script comments (# …) inside code
+ # examples are not counted as headings. Fixes #2142.
+ prose = _strip_fenced_blocks(text)
+ headings = _HEADING.findall(prose)
if len(headings) < _MIN_HEADINGS_TO_JUDGE:
return False
versiony = sum(1 for h in headings if _VERSION_HEADING.match(h.strip()))
@@ -737,9 +785,13 @@ def _harvest(
rst_sections_seen += len(entries)
rst_sections_undefined += sum(1 for _t, d in entries if d is None)
else:
+ # Strip fenced blocks before scanning so that code block comments
+ # (e.g. shell script or config comments) are not harvested as headings.
+ # Pass stripped prose to _definition_after_heading so search offsets align.
+ prose = _strip_fenced_blocks(text)
entries = [
- (m.group(1), _definition_after_heading(text, m.end()))
- for m in _HEADING.finditer(text)
+ (m.group(1), _definition_after_heading(prose, m.end()))
+ for m in _HEADING.finditer(prose)
]
# Bolded lead-ins read the same in both markup languages.
entries += [(m.group(1), None) for m in _BOLD_TERM.finditer(text)]
diff --git a/tests/unit/generation/test_vocabulary_house_terms.py b/tests/unit/generation/test_vocabulary_house_terms.py
index 4899802b0..731a27548 100644
--- a/tests/unit/generation/test_vocabulary_house_terms.py
+++ b/tests/unit/generation/test_vocabulary_house_terms.py
@@ -1031,3 +1031,220 @@ def test_normal_heading_with_words_still_produces_a_term(
assert "Cache Layer" in terms, (
"A normal two-word heading must survive _is_useful and reach the output"
)
+
+
+# ---------------------------------------------------------------------------
+# Shell comments inside fenced blocks must not be mined as headings (#2142)
+# ---------------------------------------------------------------------------
+#
+# _HEADING.finditer runs on the raw document over the whole file with
+# re.MULTILINE, so every line that starts with '#' — including shell comments
+# inside ```bash … ``` blocks — looks identical to a real Markdown heading.
+#
+# The fix is _strip_fenced_blocks(), called before _HEADING.finditer in both
+# _harvest() and _is_release_notes(). Lines inside the fence are replaced with
+# empty strings (not deleted) so that m.end() byte offsets remain valid for
+# _definition_after_heading.
+
+
+def test_shell_comment_in_backtick_fence_is_not_mined(tmp_path: Path) -> None:
+ """A # comment inside a ```bash block must not become a house term.
+
+ Regression for #2142: the miner saw '# Install dependencies' and treated
+ it as a '## Install dependencies' heading because _HEADING matched every
+ line starting with '#' without checking whether that line was inside a
+ fenced code block.
+ """
+ root = tmp_path / "repo"
+ root.mkdir()
+ (root / "README.md").write_text(
+ "# Ledger\n\n"
+ "Here is how to install:\n\n"
+ "```bash\n"
+ "# Install dependencies\n"
+ "npm install\n"
+ "# Start the server\n"
+ "npm start\n"
+ "```\n",
+ encoding="utf-8",
+ )
+ result = extract_house_terms(root)
+ terms = [t.term for t in result]
+ assert "Install dependencies" not in terms, (
+ "Shell comment inside a backtick fence must not be mined as a heading"
+ )
+ assert "Start the server" not in terms, (
+ "Shell comment inside a backtick fence must not be mined as a heading"
+ )
+
+
+def test_shell_comment_in_tilde_fence_is_not_mined(tmp_path: Path) -> None:
+ """A # comment inside a ~~~sh block must not become a house term.
+
+ Tilde fences are valid CommonMark (§4.5) and are used by some doc tools.
+ The fix must handle both backtick and tilde fence markers.
+ """
+ root = tmp_path / "repo"
+ root.mkdir()
+ (root / "README.md").write_text(
+ "# Ledger\n\n"
+ "~~~sh\n"
+ "# Configure secrets\n"
+ "export SECRET_KEY=...\n"
+ "~~~\n",
+ encoding="utf-8",
+ )
+ result = extract_house_terms(root)
+ terms = [t.term for t in result]
+ assert "Configure secrets" not in terms, (
+ "Shell comment inside a tilde fence must not be mined as a heading"
+ )
+
+
+def test_real_heading_after_fenced_block_is_still_mined(tmp_path: Path) -> None:
+ """A genuine heading that follows a fenced block must still be harvested.
+
+ Stripping fenced blocks must not accidentally erase real headings that
+ appear *after* the closing fence.
+ """
+ root = tmp_path / "repo"
+ root.mkdir()
+ (root / "README.md").write_text(
+ "# Ledger\n\n"
+ "```bash\n"
+ "# just a comment\n"
+ "```\n\n"
+ "## Blast radius\n\n"
+ "Blast radius is the set of files a change can reach.\n",
+ encoding="utf-8",
+ )
+ src = root / "src"
+ src.mkdir()
+ (src / "core.py").write_text(
+ '"""Blast radius computation."""\n', encoding="utf-8"
+ )
+ result = extract_house_terms(root)
+ terms = [t.term for t in result]
+ assert "Blast radius" in terms, (
+ "A real heading after a fenced block must still be harvested"
+ )
+
+
+def test_heading_with_inline_backtick_span_is_not_suppressed(tmp_path: Path) -> None:
+ """A heading containing an inline code span must still be harvested.
+
+ Inline backtick spans (single backtick) are not fences. A heading like
+ '## Cache Layer' that contains no backtick fences must be unaffected by the
+ fence-stripping logic.
+ """
+ root = tmp_path / "repo"
+ root.mkdir()
+ (root / "README.md").write_text(
+ "## Cache Layer\n\n"
+ "Cache Layer stores pre-computed results.\n",
+ encoding="utf-8",
+ )
+ src = root / "src"
+ src.mkdir()
+ (src / "cache.py").write_text(
+ '"""Cache Layer implementation."""\n', encoding="utf-8"
+ )
+ result = extract_house_terms(root)
+ terms = [t.term for t in result]
+ assert "Cache Layer" in terms, (
+ "A heading with no fenced block must still be harvested after the fix"
+ )
+
+
+def test_version_comments_in_code_block_do_not_trigger_release_notes(
+ tmp_path: Path,
+) -> None:
+ """Version-like # comments inside fenced blocks must not trigger release-note filtering.
+
+ _is_release_notes() counts how many headings look like version numbers. If
+ a changelog-style code example contains many '# v1.0.0' comments, the doc
+ could be misclassified as release notes and silently skipped. The fix must
+ apply to _is_release_notes() as well as _harvest().
+ """
+ root = tmp_path / "repo"
+ root.mkdir()
+ # Build a real doc that has a code block full of version comments.
+ # The ratio of version-like "headings" must stay below the threshold
+ # once fenced blocks are stripped.
+ doc_lines = [
+ "# Deployment Guide",
+ "",
+ "## Overview",
+ "",
+ "This guide explains how to deploy.",
+ "",
+ "## Prerequisites",
+ "",
+ "```bash",
+ ]
+ # Add 20 version-like shell comments — enough to exceed the ratio if counted.
+ for i in range(20):
+ doc_lines.append(f"# v1.{i}.0")
+ doc_lines += [
+ "```",
+ "",
+ "## Configuration",
+ "",
+ "Set your environment variables.",
+ "",
+ "## Installation",
+ "",
+ "Run the installer.",
+ ]
+ (root / "README.md").write_text("\n".join(doc_lines), encoding="utf-8")
+ src = root / "src"
+ src.mkdir()
+ # Spell all three real terms in source so they pass the code-frequency gate.
+ (src / "deploy.py").write_text(
+ '"""Deployment Guide utilities.\n\n'
+ "Configuration is read from environment variables.\n"
+ "Installation runs the package installer.\n"
+ '"""\n',
+ encoding="utf-8",
+ )
+ result = extract_house_terms(root)
+ terms = [t.term for t in result]
+ # The document must NOT have been skipped as release notes.
+ # At least one of its real headings must survive.
+ # ('Overview' may be filtered as a stopword; 'Prerequisites' is also common.)
+ assert "Installation" in terms or "Configuration" in terms or "Deployment Guide" in terms, (
+ "A doc with version comments only in a code block must not be classified "
+ "as release notes and must still yield its real headings as terms"
+ )
+
+
+def test_heading_definition_after_fenced_block_is_extracted_correctly(
+ tmp_path: Path,
+) -> None:
+ """Verify that headings occurring after fenced code blocks extract their correct definitions.
+
+ Ensures offset indices for definition extraction align with the stripped prose text,
+ preventing offsets from slicing into preceding code blocks.
+ """
+ root = tmp_path / "repo"
+ root.mkdir()
+ (root / "README.md").write_text(
+ "# Ledger\n\n"
+ "```bash\n"
+ "# install the dependencies for the ledger service\n"
+ "npm install\n"
+ "```\n\n"
+ "## Blast radius\n\n"
+ "Blast radius is the set of files a change can reach through the import graph.\n",
+ encoding="utf-8",
+ )
+ src = root / "src"
+ src.mkdir()
+ (src / "analysis.py").write_text('"""Blast radius calculation."""\n', encoding="utf-8")
+
+ result = {t.term: t for t in extract_house_terms(root)}
+ assert "Blast radius" in result
+ assert result["Blast radius"].definition == (
+ "Blast radius is the set of files a change can reach through the import graph."
+ )
+