Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(answer): stop a stray delimiter hiding the rest of a file (#1580)
Raghav Chamadiya9ba3bdeff4a2
diff --git a/packages/server/src/repowise/server/mcp_server/_verify.py b/packages/server/src/repowise/server/mcp_server/_verify.pyindex b17f43c74..893d1306d 100644--- a/packages/server/src/repowise/server/mcp_server/_verify.py+++ b/packages/server/src/repowise/server/mcp_server/_verify.py@@ -180,9 +180,15 @@ def check_symbol_bounds(row: WikiSymbol, source_text: str) -> BoundsCheck:start_line=located[0], end_line=located[1], verified=True, corrected=corrected)+ # Unverified, but the end is still clamped to the live file: nothing can be+ # served past EOF, and the cheap path above already clamps. Leaving this one+ # return raw is what let a stored end that overshoots flag a body served+ # WHOLE as truncated, with a continuation pointing past the last line (D8).+ # Every consumer of an unverified bound inherited it -- get_answer's+ # hydrator, get_symbol and get_context all read this same return.return BoundsCheck(start_line=row.start_line,- end_line=row.end_line,+ end_line=min(row.end_line, len(lines)) if row.end_line else row.end_line,verified=False,approximate=True,)diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.pyindex bbef71f65..502a86af3 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py@@ -1085,13 +1085,13 @@ def _data_shape_prose(grounded: dict, citations: list[str]) -> tuple[str, str]:def _is_question_named_body_cut_by_us(entry: dict, question_ids: set[str]) -> bool:"""Whether this body is the question's own symbol, cut because WE ran out of lines.- ``truncated`` alone is not trustworthy enough to demote on. It is set by- comparing the INDEXED end line against what was read, while the read clamps- to the end of the live file, so a symbol whose stored end overshoots (an- unsupported language or a syntax error leaves the bounds unverified, and a- file that shrank since indexing does it too) is served WHOLE and still- flagged. Requiring the served span to have reached the line cap says the cut- was ours, which is the only case where something was really withheld.+ ``truncated`` alone is not trustworthy enough to demote on. The stale-bound+ case this was written for is now fixed at its source — ``check_symbol_bounds``+ clamps every bound to the live file (D8) — but the guard still earns its keep+ on the ``source_excerpt`` fallback, where the served bytes come from the+ index rather than from disk and the live length says nothing. Requiring the+ served span to have reached the line cap says the cut was ours, which is the+ only case where something was really withheld."""if not (entry.get("truncated") and entry.get("continuation")):return Falsediff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py b/packages/server/src/repowise/server/mcp_server/tool_answer/config.pyindex 51099d1b9..db38fc743 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/config.py@@ -402,7 +402,19 @@ _HIGH_CONFIDENCE_SCORE_FLOOR = 1.5# degraded returns, so a degraded payload has never been written to it. The# synthesised shape is untouched. A bump here would invalidate every keyed# install's cache, and re-synthesis is real provider spend, for nothing.-_ANSWER_SCHEMA_VERSION = 15+# v16: the same scanner again, in three places, and unlike the degraded rework+# above every one of them changes the SYNTHESISED payload, which is the shape+# that gets cached. (1) A run left open at end of file no longer masks to EOF,+# so definitions below a phantom `"""` or `/*` are listed again — 26 real Rust+# definitions across two corpus files, one of which had 3,002 lines masked.+# (2) A cut inside a multi-line string keeps the symbol enclosing it, which+# changes which entry is the HEADLINE `body_continues`, and therefore the note+# and the get_symbol pointer. (3) An unverified symbol bound is clamped to the+# live file, so a body served whole is no longer flagged `truncated` — and+# v13/v14 cap confidence on that flag, so a cached row can carry a `medium`+# this version would not produce. Cached pre-v16 rows carry all three, so they+# must bypass.+_ANSWER_SCHEMA_VERSION = 16# Hard TTL on answer-cache rows. Commit-based invalidation (the payload's# stamped ``_indexed_commit`` vs the repo's current head) is the primarydiff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py b/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.pyindex 9d6b0cc9a..af134eafb 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py@@ -10,7 +10,7 @@ from __future__ import annotationsimport refrom functools import lru_cachefrom pathlib import Path-from typing import Any+from typing import Any, NamedTuplefrom sqlalchemy import select@@ -473,6 +473,12 @@ def attach_truncation_contract(end at all, which is never a cut: it is tested explicitly rather than leftto ``indexed_end > end_served``, which would only agree with it while``end_served`` stays non-negative.++ ``indexed_end`` is trusted to lie within the live file, which is+ ``check_symbol_bounds``'s job rather than this one's: it now clamps to+ ``len(lines)`` on every return, so a stored end that overshoots cannot reach+ here and flag a body served WHOLE as truncated (D8). Clamping again here+ would be a second owner and a second disk read."""if indexed_end and indexed_end > end_served:entry["truncated"] = True@@ -1181,8 +1187,12 @@ def _skip_regex(raw: str, i: int) -> int:def _walk_string_state(lines: tuple[str, ...], *, backticks: bool-) -> tuple[set[int], bool]:- """(lines starting inside a string/comment, template literal left open at EOF).+) -> tuple[set[int], set[int], bool]:+ """(lines starting inside a string, inside a block comment, literal left open).++ The two sets are kept apart because callers need to tell them apart and this+ is a hot path: a string body proves the enclosing expression is still open,+ a comment between two declarations proves nothing.``stack`` models template-literal nesting: a ``None`` frame is the stringpart of a backtick literal, an ``int`` frame is the unclosed-brace depth@@ -1208,13 +1218,22 @@ def _walk_string_state(precision, measured as 1,926 fabrications against 60 on the 16 corpus fileswhere a flat walk and this one disagree."""- masked: set[int] = set()+ strings: set[int] = set()+ comments: set[int] = set()delim: str | None = Nonein_block = False+ # Line the currently-open ``delim`` run or ``/* */`` block started on. The+ # two are mutually exclusive (a frame is only ever opened at code level), so+ # one variable serves both.+ open_line = 0stack: list[int | None] = []for n, raw in enumerate(lines, 1):- if delim is not None or in_block or (stack and stack[-1] is None):- masked.add(n)+ # The three states are mutually exclusive: a frame is only ever pushed+ # at code level, so an if/elif chain is faithful to the walk below.+ if delim is not None or (stack and stack[-1] is None):+ strings.add(n)+ elif in_block:+ comments.add(n)elif not stack and not _QUOTEISH_RE.search(raw):# Nothing on this line can open a string or comment, so the# character walk below cannot change state. Most lines are this@@ -1270,25 +1289,25 @@ def _walk_string_state(continueif delim is not None:if raw.startswith(delim, i):- delim, i = None, i + len(delim)+ delim, open_line, i = None, 0, i + len(delim)else:i += 1continueif in_block:if raw.startswith("*/", i):- in_block, i = False, i + 2+ in_block, open_line, i = False, 0, i + 2else:i += 1continueif raw.startswith('"""', i) or raw.startswith("'''", i):- delim, i = raw[i : i + 3], i + 3+ delim, open_line, i = raw[i : i + 3], n, i + 3continueif backticks and raw[i] == "`":stack.append(None)i += 1continueif raw.startswith("/*", i):- in_block, i = True, i + 2+ in_block, open_line, i = True, n, i + 2continueif raw[i] == "#" or raw.startswith("//", i):break@@ -1304,7 +1323,20 @@ def _walk_string_state(if not raw[i].isspace():prev = raw[i]i += 1- return masked, bool(stack)+ # A run still open at EOF is a walk that lost track, not a file with an+ # unterminated construct, and masking to EOF hides every definition below+ # it. The template-literal stack has had this containment since the backtick+ # work; ``delim`` and ``/* */`` never did, and both fire on the same shape --+ # a delimiter belonging to another language, sitting inside a string this+ # walk cannot see. Measured on Rust: ``${0%/*}`` in a raw string masked 103+ # lines and cost 6 real ``fn``; ``description = """#`` masked 3,002 and cost+ # 10. Discarding the trailing run under-masks instead, which costs a+ # spurious name in a list rather than an absent real one.+ if delim is not None:+ strings = {n for n in strings if n < open_line}+ elif in_block:+ comments = {n for n in comments if n < open_line}+ return strings, comments, bool(stack)def _has_backtick_strings(file_path: str) -> bool:@@ -1313,10 +1345,20 @@ def _has_backtick_strings(file_path: str) -> bool:return dot != -1 and file_path[dot:].lower() in _BACKTICK_STRING_SUFFIXES+class _Masked(NamedTuple):+ """1-based line numbers, split by what is hiding them.++ ``all`` is precomputed rather than unioned per call: every caller wants it,+ and this is a cached whole-file walk.+ """++ strings: frozenset[int]+ comments: frozenset[int]+ all: frozenset[int]++@lru_cache(maxsize=8)-def _string_masked_lines(- lines: tuple[str, ...], backticks: bool = True-) -> frozenset[int]:+def _string_masked_lines(lines: tuple[str, ...], backticks: bool = True) -> _Masked:"""1-based line numbers that START inside a multi-line string or comment.Repowise's own docstrings are full of indented ``def``/``class`` examples,@@ -1347,7 +1389,9 @@ def _string_masked_lines(synthetic 1.2 MB file with one stray backtick on line 1. Bounded at twowalks, and the cache means it is paid once per file."""- masked, template_left_open = _walk_string_state(lines, backticks=backticks)+ strings, comments, template_left_open = _walk_string_state(+ lines, backticks=backticks+ )if template_left_open:# The lexer-lite lost track: a template literal opened and never closed,# so every line below it is masked to EOF and every definition there is@@ -1355,8 +1399,10 @@ def _string_masked_lines(# missing symbols -- so it must not be the one we ship. Fall back to the# pre-backtick walk for this file, which under-masks instead: the cost is# a spurious name in a list, not an absent real one.- masked, _ = _walk_string_state(lines, backticks=False)- return frozenset(masked)+ strings, comments, _ = _walk_string_state(lines, backticks=False)+ return _Masked(+ frozenset(strings), frozenset(comments), frozenset(strings | comments)+ )def _indent_width(raw: str) -> int:@@ -1409,7 +1455,8 @@ def withheld_definitions(lines = text.splitlines()if lo < 1 or lo > len(lines):return []- masked = _string_masked_lines(tuple(lines), _has_backtick_strings(path))+ mask = _string_masked_lines(tuple(lines), _has_backtick_strings(path))+ masked = mask.alldef _entry(line_no: int, m: re.Match[str], *, cut: bool = False) -> dict:name = m.group("name")@@ -1460,7 +1507,16 @@ def withheld_definitions(and n not in maskedand lines[n - 1].strip()[0] not in ")]}{"]- if _usable:+ if lo in mask.strings:+ # The cut is INSIDE a multi-line string, so the expression holding that+ # string -- and everything enclosing it -- is still open at ``lo``.+ # Without this the anchor reads from the first line BELOW the string,+ # usually a top-level declaration at column 0, and the walk dies at once+ # (D9: 8 real definitions lost across cli/cli and mui). A block COMMENT+ # cannot stand in for this: one sitting between two methods would report+ # the preceding method as continuing when it has already ended.+ anchor = _UNBOUNDED_INDENT+ elif _usable:anchor = _indent_width(lines[_usable[0] - 1])elif any(lines[n - 1].strip() for n in range(lo, _end + 1)):# Every withheld line is a string body or a bracket tail, so whateverdiff --git a/tests/unit/server/mcp/test_answer_backtick_masking.py b/tests/unit/server/mcp/test_answer_backtick_masking.pyindex a74fbf9f3..18031b8c3 100644--- a/tests/unit/server/mcp/test_answer_backtick_masking.py+++ b/tests/unit/server/mcp/test_answer_backtick_masking.py@@ -184,7 +184,9 @@ def test_the_nested_guard_is_not_vacuous(tmp_path) -> None:target = 11 # the `export function afterTheTemplate` lineflat_masked, _flat_open = _flat(lines)assert target in flat_masked, sorted(flat_masked)- assert target not in _string_masked_lines(lines), sorted(_string_masked_lines(lines))+ assert target not in _string_masked_lines(lines).all, sorted(+ _string_masked_lines(lines).all+ )def test_a_backtick_inside_a_regex_literal_does_not_open_a_template(tmp_path) -> None:diff --git a/tests/unit/server/mcp/test_answer_mask_containment.py b/tests/unit/server/mcp/test_answer_mask_containment.pynew file mode 100644index 000000000..316e38df3--- /dev/null+++ b/tests/unit/server/mcp/test_answer_mask_containment.py@@ -0,0 +1,182 @@+"""Masking containment, and the defect it left open.++Both are found by walking every file in a multi-language corpus rather than+sampling cut points (`local-stash/get-answer-dogfood/overmask_gate.py`), which+is the instrument the random-cut sweep never was:++* **D9** -- a cut inside a multi-line string must keep the symbol enclosing it,+ and a cut inside a block COMMENT must not, because a comment can sit between+ two declarations while a string body cannot.+* **the EOF containment** -- a triple-quote or ``/*`` run still open at end of+ file is a walk that lost track, so it must under-mask rather than hide every+ definition below it. Both fire on a delimiter belonging to another language+ sitting inside a string this walk cannot see.++D8 is not here. Its owner turned out to be `check_symbol_bounds`, which already+clamps to the live file on its other two returns, so it is covered in+`test_verify_bounds.py`.+"""++from pathlib import Path++from repowise.server.mcp_server.tool_answer.symbols import (+ _string_masked_lines,+ withheld_definitions,+)++# A Go test whose body is cut inside a raw string. The line below the string is+# a top-level `func` at column 0, which is what the anchor used to read.+_GO_CUT_IN_A_RAW_STRING = """\+package main++func TestThing(t *testing.T) {+\trun(`+\tquery {+\t\trepository(owner: $owner) {+\t\t\tname+\t\t}+\t}+\t`)+}++func AfterIt() {+}+"""++# A block comment BETWEEN two top-level functions, indented deeper than the code+# around it. `first` has ended by the time the comment starts, so nothing may be+# reported as continuing.+#+# The indent is what makes this discriminating rather than decorative. The+# comment's opening line is not masked and pins the backward walk's running+# minimum indent, so a comment at column 0 stops the walk on its own whatever+# the anchor says. Here the `}` above is skipped as a bracket tail without+# updating that minimum, so the walk runs on and reaches `first`.+_JS_CUT_IN_A_BLOCK_COMMENT = """\+export function first() {+ return 1;+}+ /* prose about what comes next+ more prose+ and more+ */+export function second() {+ return 2;+}+"""++# `${0%/*}` inside a Rust raw string. `/*` is not a comment here, and nothing+# closes it, so the parent masked to EOF and lost every `fn` below.+_RUST_SLASH_STAR_IN_A_RAW_STRING = '''\+fn shim() {+ let script = r#"#!/bin/sh+record_dir=${0%/*}+printf '%s' "$@"+"#;+ write(script);+}++fn after_the_raw_string() {+ ok()+}+'''++# A triple quote is a Python/Kotlin/Java delimiter, not a Rust one. Same shape,+# other delimiter: nothing closes it, so the parent masked to EOF. Verbatim from+# `gleam/compiler-core/src/error.rs`, where it cost 3,002 lines.+_RUST_TRIPLE_QUOTE_IN_A_STRING = '''\+fn describe() {+ let text = r#"+description = """#+"#;+ emit(text);+}++fn after_the_triple_quote() {+ ok()+}+'''+++def _write(tmp_path: Path, name: str, body: str) -> Path:+ (tmp_path / name).write_text(body, encoding="utf-8")+ return tmp_path+++def test_a_cut_inside_a_multi_line_string_keeps_the_enclosing_symbol(tmp_path):+ """FAILS on c3fdccbd with `[]` -- D9, 8 measured instances on cli/cli + mui.++ The cut is inside the raw string, so every line of it is masked and the+ first usable withheld line is `func AfterIt` at column 0. The anchor read 0+ and the backward walk exited on its first iteration.+ """+ root = _write(tmp_path, "thing_test.go", _GO_CUT_IN_A_RAW_STRING)++ got = withheld_definitions(root, "thing_test.go:5-14")++ head = next((d for d in got if d.get("body_continues")), None)+ assert head is not None, got+ assert head["name"] == "TestThing", got+++def test_a_cut_inside_a_block_comment_does_not_resurrect_the_symbol_above_it(tmp_path):+ """The control for the fix above, and the reason it is not one line.++ Verified to be discriminating rather than assumed: swap the anchor test to+ `lo in mask.all` and this returns `first` with `body_continues`, four lines+ after `first` ended. That is the whole argument for splitting the mask by+ what is hiding the line.+ """+ root = _write(tmp_path, "pair.js", _JS_CUT_IN_A_BLOCK_COMMENT)++ got = withheld_definitions(root, "pair.js:5-11")++ assert not any(d.get("body_continues") for d in got), got+++def test_an_unterminated_block_comment_does_not_mask_to_end_of_file():+ """26 real Rust definitions lost across the corpus before this.++ Measured on `goose`: `${0%/*}` in a raw string masked 103 lines and took+ six real `fn` with it. This does not "fail on the parent" in a useful way --+ the parent's masker returns a bare frozenset, so it raises AttributeError+ rather than the defect. The non-vacuity proof is `ablate_masking.py`'s+ `eof-containment-block` row, which disables the branch on THIS build.+ """+ lines = tuple(_RUST_SLASH_STAR_IN_A_RAW_STRING.splitlines())++ mask = _string_masked_lines(lines, False)++ target = next(i for i, ln in enumerate(lines, 1) if "after_the_raw_string" in ln)+ assert target not in mask.all, sorted(mask.all)+++def test_an_unterminated_triple_quote_run_does_not_mask_to_end_of_file():+ """The same defect one delimiter over.++ Measured on `gleam`: a triple quote inside a Rust string masked 3,002 lines+ and took ten real definitions with it. Non-vacuity is `ablate_masking.py`'s+ `eof-containment-delim` row, for the reason above.+ """+ lines = tuple(_RUST_TRIPLE_QUOTE_IN_A_STRING.splitlines())++ mask = _string_masked_lines(lines, False)++ target = next(i for i, ln in enumerate(lines, 1) if "after_the_triple_quote" in ln)+ assert target not in mask.all, sorted(mask.all)+++def test_the_masker_reports_strings_and_comments_separately():+ """The split D9's fix needs, from ONE walk on a documented hot path.++ Both directions, because a split that files everything under one bucket+ still satisfies a one-sided assertion.+ """+ comment_mask = _string_masked_lines(tuple(_JS_CUT_IN_A_BLOCK_COMMENT.splitlines()))+ assert comment_mask.comments and not comment_mask.strings, comment_mask++ string_mask = _string_masked_lines(tuple(_GO_CUT_IN_A_RAW_STRING.splitlines()))+ assert string_mask.strings and not string_mask.comments, string_mask++ for mask in (comment_mask, string_mask):+ assert mask.all == mask.strings | mask.commentsdiff --git a/tests/unit/server/mcp/test_verify_bounds.py b/tests/unit/server/mcp/test_verify_bounds.pyindex d62229e6a..770ea210d 100644--- a/tests/unit/server/mcp/test_verify_bounds.py+++ b/tests/unit/server/mcp/test_verify_bounds.py@@ -206,3 +206,33 @@ async def test_heal_symbol_row_persists_corrected_bounds(setup_mcp, factory, sesasync with get_session(factory) as s:healed = (await s.execute(select(WikiSymbol).where(WikiSymbol.id == row.id))).scalar_one()assert (healed.start_line, healed.end_line) == (42, 57)+++class TestUnverifiedBoundsAreStillClampedToTheLiveFile:+ """D8: an unverified end that overshoots EOF made a whole body look cut.++ The cheap gate clamps (``min(..., len(lines))``) and a re-parse returns real+ bounds, so this was the one return handing back a raw stored end. Everything+ downstream inherited it: get_answer's hydrator computes ``truncated`` from+ it, and get_symbol / get_context slice with it.+ """++ def test_an_unrelocatable_symbol_does_not_report_an_end_past_eof(self) -> None:+ # A name that is nowhere in the source, so both the cheap gate and the+ # re-parse relocation fail and the approximate branch is taken.+ row = _row(name="vanished", symbol_id="pkg/mod.py::vanished", end_line=900)++ check = check_symbol_bounds(row, FRESH_SOURCE)++ assert check.verified is False+ assert check.approximate is True+ assert check.end_line == len(FRESH_SOURCE.splitlines())++ def test_an_unverified_end_inside_the_file_is_left_alone(self) -> None:+ """The control: clamping must not shorten a bound that already fits."""+ row = _row(name="vanished", symbol_id="pkg/mod.py::vanished", end_line=6)++ check = check_symbol_bounds(row, FRESH_SOURCE)++ assert check.verified is False+ assert check.end_line == 6