Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(generation): name the per-file vocabulary for what it is (#1265)
Raghav Chamadiyac710ea047011
diff --git a/packages/core/src/repowise/core/generation/context/assembler.py b/packages/core/src/repowise/core/generation/context/assembler.pyindex bbb2ef561..4a5efe7a0 100644--- a/packages/core/src/repowise/core/generation/context/assembler.py+++ b/packages/core/src/repowise/core/generation/context/assembler.py@@ -14,7 +14,6 @@ from repowise.core.ingestion.models import ParsedFile, RepoStructure, Symbolfrom ..categories import file_categoryfrom ..models import GenerationConfig-from .code_vocabulary import code_vocabularyfrom .contexts import (ApiContractContext,ArchitectureDiagramContext,@@ -26,6 +25,7 @@ from .contexts import (SymbolSpotlightContext,_TopFile,)+from .file_vocabulary import file_vocabularyfrom .graph_intelligence import (extract_call_graph,extract_community_meta,@@ -400,7 +400,7 @@ class ContextAssembler:# the biggest files the thinnest, which is backwards. This section# carries its own cap and is not charged against the prompt budget# because it is page content rather than model input.- code_vocabulary=code_vocabulary(source_text),+ file_vocabulary=file_vocabulary(source_text),)# ------------------------------------------------------------------diff --git a/packages/core/src/repowise/core/generation/context/contexts.py b/packages/core/src/repowise/core/generation/context/contexts.pyindex 93e01ef1c..8f3952987 100644--- a/packages/core/src/repowise/core/generation/context/contexts.py+++ b/packages/core/src/repowise/core/generation/context/contexts.py@@ -63,10 +63,12 @@ class FilePageContext:kg_tags: list[str] = field(default_factory=list)kg_node_summary: str = ""# The words the file's own source uses, bounded and ordered- # most-distinguishing first. See ``context/code_vocabulary.py`` for what- # goes in it and why. Empty when the file yields nothing, in which case the- # template drops the section rather than rendering an empty heading.- code_vocabulary: str = ""+ # most-distinguishing first. See ``context/file_vocabulary.py`` for what+ # goes in it and why, including why it is not the repo-level vocabulary in+ # ``concept_tree/vocabulary.py``. Empty when the file yields nothing, in+ # which case the template drops the section rather than rendering an empty+ # heading.+ file_vocabulary: str = ""@dataclassdiff --git a/packages/core/src/repowise/core/generation/context/code_vocabulary.py b/packages/core/src/repowise/core/generation/context/file_vocabulary.pysimilarity index 92%rename from packages/core/src/repowise/core/generation/context/code_vocabulary.pyrename to packages/core/src/repowise/core/generation/context/file_vocabulary.pyindex a386b49ef..c8a7eb8c5 100644--- a/packages/core/src/repowise/core/generation/context/code_vocabulary.py+++ b/packages/core/src/repowise/core/generation/context/file_vocabulary.py@@ -1,5 +1,11 @@"""The file's own vocabulary: the words a question about this file would use.+Not to be confused with ``concept_tree/vocabulary.py``, which mines doc headings+and binds repo-level terms to concept groups. Those terms are the same on every+file page in the repository and so carry no discriminative power between files,+which is the reason this per-file module exists alongside it. The two are not+duplicates and must not be merged.+A file page renders an overview, a table of symbols and a list of dependencypaths. None of those carry the words a reader actually searches with: a flagname, an error message, a struct field, the phrasing of a doc comment. So a@@ -67,7 +73,7 @@ def _words(token: str) -> list[str]:return [w.lower() for w in out if len(w) > 2]-def code_vocabulary(source: str, *, max_chars: int = _MAX_CHARS) -> str:+def file_vocabulary(source: str, *, max_chars: int = _MAX_CHARS) -> str:"""The bounded vocabulary of one source file, most distinguishing first.Returns a plain space-joined string. Empty when *source* yields nothing,diff --git a/packages/core/src/repowise/core/generation/templates/file_page.j2 b/packages/core/src/repowise/core/generation/templates/file_page.j2index d0c3d0bf9..a1d245d77 100644--- a/packages/core/src/repowise/core/generation/templates/file_page.j2+++ b/packages/core/src/repowise/core/generation/templates/file_page.j2@@ -123,11 +123,11 @@ Part of the **{{ ctx.community_label }}** module cluster.words is not a summary. Conditional, like every section below theOverview, so a file that yields no vocabulary renders no heading.-#}-{%- if ctx.code_vocabulary %}+{%- if ctx.file_vocabulary %}## In the code-{{ ctx.code_vocabulary }}+{{ ctx.file_vocabulary }}{%- endif %}{% include "_footer.j2" %}diff --git a/tests/unit/generation/test_code_vocabulary.py b/tests/unit/generation/test_file_vocabulary.pysimilarity index 91%rename from tests/unit/generation/test_code_vocabulary.pyrename to tests/unit/generation/test_file_vocabulary.pyindex fb9afea4f..9974bf279 100644--- a/tests/unit/generation/test_code_vocabulary.py+++ b/tests/unit/generation/test_file_vocabulary.py@@ -29,7 +29,7 @@ import pytestfrom sqlalchemy.ext.asyncio import async_sessionmaker, create_async_enginefrom sqlalchemy.pool import StaticPool-from repowise.core.generation.context.code_vocabulary import code_vocabulary+from repowise.core.generation.context.file_vocabulary import file_vocabularyfrom repowise.core.generation.context_assembler import FilePageContextfrom repowise.core.generation.page_generator.structural import (as_markdown,@@ -70,13 +70,13 @@ func listRun(opts *ListOptions) error {class TestWhatItLifts:def test_declared_field_names_survive(self):- out = code_vocabulary(GO_SOURCE)+ out = file_vocabulary(GO_SOURCE)assert "Order" in outassert "ExcludePreReleases" in outdef test_string_literals_survive(self):"""Flag names and help text: the wording a bug report repeats back."""- out = code_vocabulary(GO_SOURCE)+ out = file_vocabulary(GO_SOURCE)assert "Order of releases returned" in outassert "desc" in out@@ -92,7 +92,7 @@ class TestWhatItLifts:copy of the word, which is correct, because the phrase is what a bugreport quotes back."""- out = code_vocabulary(' Order string\nvar f = "order"\n')+ out = file_vocabulary(' Order string\nvar f = "order"\n')assert out.lower().split().count("order") == 1def test_camel_case_is_also_offered_as_words(self):@@ -101,16 +101,16 @@ class TestWhatItLifts:``ExcludePreReleases`` is kept whole *and* split, because a reader asks"how do I exclude pre-releases" and never types the identifier."""- out = code_vocabulary(GO_SOURCE).lower()+ out = file_vocabulary(GO_SOURCE).lower()for word in ("exclude", "pre", "releases"):assert word in out.split() or word in outdef test_comment_prose_survives(self):- assert "ListOptions carries the flags" in code_vocabulary(GO_SOURCE)+ assert "ListOptions carries the flags" in file_vocabulary(GO_SOURCE)def test_python_attributes_are_lifted(self):source = "class Config:\n def __init__(self):\n self.retry_budget = 3\n"- out = code_vocabulary(source)+ out = file_vocabulary(source)assert "retry_budget" in outdef test_the_tokens_the_gold_instance_needed_all_arrive(self):@@ -122,7 +122,7 @@ class TestWhatItLifts:arrives as ``created``, which answers "ordered by creation date"exactly as well."""- out = code_vocabulary(GO_SOURCE).lower()+ out = file_vocabulary(GO_SOURCE).lower()for token in ("order", "desc", "created", "releases"):assert token in out, f"{token!r} missing"@@ -130,7 +130,7 @@ class TestWhatItLifts:class TestBounds:def test_capped(self):source = "\n".join(f' Field{i} string // comment number {i}' for i in range(4000))- assert len(code_vocabulary(source)) <= 1200+ assert len(file_vocabulary(source)) <= 1200def test_the_cap_cuts_the_least_specific_material(self):"""Ordering is the reason the cap is affordable.@@ -139,23 +139,23 @@ class TestBounds:truncated bag keeps the distinguishing half."""source = GO_SOURCE + "\n" + "\n".join(f"var filler{i} int" for i in range(3000))- out = code_vocabulary(source)+ out = file_vocabulary(source)assert "Order of releases returned" in outassert "filler2999" not in outdef test_empty_source_yields_nothing_rather_than_a_heading(self):- assert code_vocabulary("") == ""+ assert file_vocabulary("") == ""def test_single_and_double_character_names_are_not_collected(self):"""Receivers and loop variables are never what a question is about."""- out = code_vocabulary("func (c *Client) do(i int) { x := 1; _ = x }")+ out = file_vocabulary("func (c *Client) do(i int) { x := 1; _ = x }")assert " i " not in f" {out} "assert " x " not in f" {out} "def test_long_blobs_do_not_eat_the_cap(self):"""Minified data and embedded SQL are long and unsearchable."""blob = "A" * 900- out = code_vocabulary(f' Name string\nvar data = "{blob}"')+ out = file_vocabulary(f' Name string\nvar data = "{blob}"')assert blob not in outassert "Name" in out@@ -223,7 +223,7 @@ def _file_page(**overrides) -> FilePageContext:class TestRendering:def test_the_section_renders_when_there_is_vocabulary(self, jinja_env):- ctx = _file_page(code_vocabulary=code_vocabulary(GO_SOURCE))+ ctx = _file_page(file_vocabulary=file_vocabulary(GO_SOURCE))page = jinja_env.get_template("file_page.j2").render(ctx=ctx)assert VOCAB_HEADING in pageassert "Order of releases returned" in page@@ -234,13 +234,13 @@ class TestRendering:An empty heading puts the same stock line into the index on thousandsof pages, where it matches every query and distinguishes none."""- page = jinja_env.get_template("file_page.j2").render(ctx=_file_page(code_vocabulary=""))+ page = jinja_env.get_template("file_page.j2").render(ctx=_file_page(file_vocabulary=""))assert VOCAB_HEADING not in pagedef test_it_sits_below_the_questions_block(self, jinja_env):"""``_extract_summary`` reads back from the top. A bag of words is nota summary, so it must not be the first thing the page says."""- ctx = _file_page(code_vocabulary=code_vocabulary(GO_SOURCE))+ ctx = _file_page(file_vocabulary=file_vocabulary(GO_SOURCE))page = jinja_env.get_template("file_page.j2").render(ctx=ctx)assert page.index("## Overview") < page.index(VOCAB_HEADING)assert page.index("## Questions this page answers") < page.index(VOCAB_HEADING)@@ -253,11 +253,11 @@ class TestRendering:opposite granularity, deliberately."""tmpl = jinja_env.get_template("file_page.j2")- a = tmpl.render(ctx=_file_page(code_vocabulary=code_vocabulary(GO_SOURCE)))+ a = tmpl.render(ctx=_file_page(file_vocabulary=file_vocabulary(GO_SOURCE)))b = tmpl.render(ctx=_file_page(file_path="api/client.go",- code_vocabulary=code_vocabulary('type Client struct {\n\tHTTP string\n}'),+ file_vocabulary=file_vocabulary('type Client struct {\n\tHTTP string\n}'),))assert "Order of releases returned" in a@@ -311,7 +311,7 @@ async def test_the_words_reach_the_full_text_row(fts, jinja_env):claim."""engine, search = fts- ctx = _file_page(code_vocabulary=code_vocabulary(GO_SOURCE))+ ctx = _file_page(file_vocabulary=file_vocabulary(GO_SOURCE))content = jinja_env.get_template("file_page.j2").render(ctx=ctx)await _index(engine,