Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(generation): name the per-file vocabulary for what it is (#1265)

Raghav Chamadiyac710ea047011
diff --git a/packages/core/src/repowise/core/generation/context/assembler.py b/packages/core/src/repowise/core/generation/context/assembler.py
index bbb2ef561..4a5efe7a0 100644
--- a/packages/core/src/repowise/core/generation/context/assembler.py
+++ b/packages/core/src/repowise/core/generation/context/assembler.py
@@ -14,7 +14,6 @@ from repowise.core.ingestion.models import ParsedFile, RepoStructure, Symbol
from ..categories import file_category
from ..models import GenerationConfig
-from .code_vocabulary import code_vocabulary
from .contexts import (
ApiContractContext,
ArchitectureDiagramContext,
@@ -26,6 +25,7 @@ from .contexts import (
SymbolSpotlightContext,
_TopFile,
)
+from .file_vocabulary import file_vocabulary
from .graph_intelligence import (
extract_call_graph,
extract_community_meta,
@@ -400,7 +400,7 @@ class ContextAssembler:
# the biggest files the thinnest, which is backwards. This section
# carries its own cap and is not charged against the prompt budget
# because it is page content rather than model input.
- code_vocabulary=code_vocabulary(source_text),
+ file_vocabulary=file_vocabulary(source_text),
)
# ------------------------------------------------------------------
diff --git a/packages/core/src/repowise/core/generation/context/contexts.py b/packages/core/src/repowise/core/generation/context/contexts.py
index 93e01ef1c..8f3952987 100644
--- a/packages/core/src/repowise/core/generation/context/contexts.py
+++ b/packages/core/src/repowise/core/generation/context/contexts.py
@@ -63,10 +63,12 @@ class FilePageContext:
kg_tags: list[str] = field(default_factory=list)
kg_node_summary: str = ""
# The words the file's own source uses, bounded and ordered
- # most-distinguishing first. See ``context/code_vocabulary.py`` for what
- # goes in it and why. Empty when the file yields nothing, in which case the
- # template drops the section rather than rendering an empty heading.
- code_vocabulary: str = ""
+ # most-distinguishing first. See ``context/file_vocabulary.py`` for what
+ # goes in it and why, including why it is not the repo-level vocabulary in
+ # ``concept_tree/vocabulary.py``. Empty when the file yields nothing, in
+ # which case the template drops the section rather than rendering an empty
+ # heading.
+ file_vocabulary: str = ""
@dataclass
diff --git a/packages/core/src/repowise/core/generation/context/code_vocabulary.py b/packages/core/src/repowise/core/generation/context/file_vocabulary.py
similarity index 92%
rename from packages/core/src/repowise/core/generation/context/code_vocabulary.py
rename to packages/core/src/repowise/core/generation/context/file_vocabulary.py
index a386b49ef..c8a7eb8c5 100644
--- a/packages/core/src/repowise/core/generation/context/code_vocabulary.py
+++ b/packages/core/src/repowise/core/generation/context/file_vocabulary.py
@@ -1,5 +1,11 @@
"""The file's own vocabulary: the words a question about this file would use.
+Not to be confused with ``concept_tree/vocabulary.py``, which mines doc headings
+and binds repo-level terms to concept groups. Those terms are the same on every
+file page in the repository and so carry no discriminative power between files,
+which is the reason this per-file module exists alongside it. The two are not
+duplicates and must not be merged.
+
A file page renders an overview, a table of symbols and a list of dependency
paths. None of those carry the words a reader actually searches with: a flag
name, an error message, a struct field, the phrasing of a doc comment. So a
@@ -67,7 +73,7 @@ def _words(token: str) -> list[str]:
return [w.lower() for w in out if len(w) > 2]
-def code_vocabulary(source: str, *, max_chars: int = _MAX_CHARS) -> str:
+def file_vocabulary(source: str, *, max_chars: int = _MAX_CHARS) -> str:
"""The bounded vocabulary of one source file, most distinguishing first.
Returns a plain space-joined string. Empty when *source* yields nothing,
diff --git a/packages/core/src/repowise/core/generation/templates/file_page.j2 b/packages/core/src/repowise/core/generation/templates/file_page.j2
index d0c3d0bf9..a1d245d77 100644
--- a/packages/core/src/repowise/core/generation/templates/file_page.j2
+++ b/packages/core/src/repowise/core/generation/templates/file_page.j2
@@ -123,11 +123,11 @@ Part of the **{{ ctx.community_label }}** module cluster.
words is not a summary. Conditional, like every section below the
Overview, so a file that yields no vocabulary renders no heading.
-#}
-{%- if ctx.code_vocabulary %}
+{%- if ctx.file_vocabulary %}
## In the code
-{{ ctx.code_vocabulary }}
+{{ ctx.file_vocabulary }}
{%- endif %}
{% include "_footer.j2" %}
diff --git a/tests/unit/generation/test_code_vocabulary.py b/tests/unit/generation/test_file_vocabulary.py
similarity index 91%
rename from tests/unit/generation/test_code_vocabulary.py
rename to tests/unit/generation/test_file_vocabulary.py
index fb9afea4f..9974bf279 100644
--- a/tests/unit/generation/test_code_vocabulary.py
+++ b/tests/unit/generation/test_file_vocabulary.py
@@ -29,7 +29,7 @@ import pytest
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.pool import StaticPool
-from repowise.core.generation.context.code_vocabulary import code_vocabulary
+from repowise.core.generation.context.file_vocabulary import file_vocabulary
from repowise.core.generation.context_assembler import FilePageContext
from repowise.core.generation.page_generator.structural import (
as_markdown,
@@ -70,13 +70,13 @@ func listRun(opts *ListOptions) error {
class TestWhatItLifts:
def test_declared_field_names_survive(self):
- out = code_vocabulary(GO_SOURCE)
+ out = file_vocabulary(GO_SOURCE)
assert "Order" in out
assert "ExcludePreReleases" in out
def test_string_literals_survive(self):
"""Flag names and help text: the wording a bug report repeats back."""
- out = code_vocabulary(GO_SOURCE)
+ out = file_vocabulary(GO_SOURCE)
assert "Order of releases returned" in out
assert "desc" in out
@@ -92,7 +92,7 @@ class TestWhatItLifts:
copy of the word, which is correct, because the phrase is what a bug
report quotes back.
"""
- out = code_vocabulary(' Order string\nvar f = "order"\n')
+ out = file_vocabulary(' Order string\nvar f = "order"\n')
assert out.lower().split().count("order") == 1
def test_camel_case_is_also_offered_as_words(self):
@@ -101,16 +101,16 @@ class TestWhatItLifts:
``ExcludePreReleases`` is kept whole *and* split, because a reader asks
"how do I exclude pre-releases" and never types the identifier.
"""
- out = code_vocabulary(GO_SOURCE).lower()
+ out = file_vocabulary(GO_SOURCE).lower()
for word in ("exclude", "pre", "releases"):
assert word in out.split() or word in out
def test_comment_prose_survives(self):
- assert "ListOptions carries the flags" in code_vocabulary(GO_SOURCE)
+ assert "ListOptions carries the flags" in file_vocabulary(GO_SOURCE)
def test_python_attributes_are_lifted(self):
source = "class Config:\n def __init__(self):\n self.retry_budget = 3\n"
- out = code_vocabulary(source)
+ out = file_vocabulary(source)
assert "retry_budget" in out
def test_the_tokens_the_gold_instance_needed_all_arrive(self):
@@ -122,7 +122,7 @@ class TestWhatItLifts:
arrives as ``created``, which answers "ordered by creation date"
exactly as well.
"""
- out = code_vocabulary(GO_SOURCE).lower()
+ out = file_vocabulary(GO_SOURCE).lower()
for token in ("order", "desc", "created", "releases"):
assert token in out, f"{token!r} missing"
@@ -130,7 +130,7 @@ class TestWhatItLifts:
class TestBounds:
def test_capped(self):
source = "\n".join(f' Field{i} string // comment number {i}' for i in range(4000))
- assert len(code_vocabulary(source)) <= 1200
+ assert len(file_vocabulary(source)) <= 1200
def test_the_cap_cuts_the_least_specific_material(self):
"""Ordering is the reason the cap is affordable.
@@ -139,23 +139,23 @@ class TestBounds:
truncated bag keeps the distinguishing half.
"""
source = GO_SOURCE + "\n" + "\n".join(f"var filler{i} int" for i in range(3000))
- out = code_vocabulary(source)
+ out = file_vocabulary(source)
assert "Order of releases returned" in out
assert "filler2999" not in out
def test_empty_source_yields_nothing_rather_than_a_heading(self):
- assert code_vocabulary("") == ""
+ assert file_vocabulary("") == ""
def test_single_and_double_character_names_are_not_collected(self):
"""Receivers and loop variables are never what a question is about."""
- out = code_vocabulary("func (c *Client) do(i int) { x := 1; _ = x }")
+ out = file_vocabulary("func (c *Client) do(i int) { x := 1; _ = x }")
assert " i " not in f" {out} "
assert " x " not in f" {out} "
def test_long_blobs_do_not_eat_the_cap(self):
"""Minified data and embedded SQL are long and unsearchable."""
blob = "A" * 900
- out = code_vocabulary(f' Name string\nvar data = "{blob}"')
+ out = file_vocabulary(f' Name string\nvar data = "{blob}"')
assert blob not in out
assert "Name" in out
@@ -223,7 +223,7 @@ def _file_page(**overrides) -> FilePageContext:
class TestRendering:
def test_the_section_renders_when_there_is_vocabulary(self, jinja_env):
- ctx = _file_page(code_vocabulary=code_vocabulary(GO_SOURCE))
+ ctx = _file_page(file_vocabulary=file_vocabulary(GO_SOURCE))
page = jinja_env.get_template("file_page.j2").render(ctx=ctx)
assert VOCAB_HEADING in page
assert "Order of releases returned" in page
@@ -234,13 +234,13 @@ class TestRendering:
An empty heading puts the same stock line into the index on thousands
of pages, where it matches every query and distinguishes none.
"""
- page = jinja_env.get_template("file_page.j2").render(ctx=_file_page(code_vocabulary=""))
+ page = jinja_env.get_template("file_page.j2").render(ctx=_file_page(file_vocabulary=""))
assert VOCAB_HEADING not in page
def test_it_sits_below_the_questions_block(self, jinja_env):
"""``_extract_summary`` reads back from the top. A bag of words is not
a summary, so it must not be the first thing the page says."""
- ctx = _file_page(code_vocabulary=code_vocabulary(GO_SOURCE))
+ ctx = _file_page(file_vocabulary=file_vocabulary(GO_SOURCE))
page = jinja_env.get_template("file_page.j2").render(ctx=ctx)
assert page.index("## Overview") < page.index(VOCAB_HEADING)
assert page.index("## Questions this page answers") < page.index(VOCAB_HEADING)
@@ -253,11 +253,11 @@ class TestRendering:
opposite granularity, deliberately.
"""
tmpl = jinja_env.get_template("file_page.j2")
- a = tmpl.render(ctx=_file_page(code_vocabulary=code_vocabulary(GO_SOURCE)))
+ a = tmpl.render(ctx=_file_page(file_vocabulary=file_vocabulary(GO_SOURCE)))
b = tmpl.render(
ctx=_file_page(
file_path="api/client.go",
- code_vocabulary=code_vocabulary('type Client struct {\n\tHTTP string\n}'),
+ file_vocabulary=file_vocabulary('type Client struct {\n\tHTTP string\n}'),
)
)
assert "Order of releases returned" in a
@@ -311,7 +311,7 @@ async def test_the_words_reach_the_full_text_row(fts, jinja_env):
claim.
"""
engine, search = fts
- ctx = _file_page(code_vocabulary=code_vocabulary(GO_SOURCE))
+ ctx = _file_page(file_vocabulary=file_vocabulary(GO_SOURCE))
content = jinja_env.get_template("file_page.j2").render(ctx=ctx)
await _index(
engine,