Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(answer): stop a stray delimiter hiding the rest of a file (#1580)

Raghav Chamadiya9ba3bdeff4a2
diff --git a/packages/server/src/repowise/server/mcp_server/_verify.py b/packages/server/src/repowise/server/mcp_server/_verify.py
index b17f43c74..893d1306d 100644
--- a/packages/server/src/repowise/server/mcp_server/_verify.py
+++ b/packages/server/src/repowise/server/mcp_server/_verify.py
@@ -180,9 +180,15 @@ def check_symbol_bounds(row: WikiSymbol, source_text: str) -> BoundsCheck:
start_line=located[0], end_line=located[1], verified=True, corrected=corrected
)
+ # Unverified, but the end is still clamped to the live file: nothing can be
+ # served past EOF, and the cheap path above already clamps. Leaving this one
+ # return raw is what let a stored end that overshoots flag a body served
+ # WHOLE as truncated, with a continuation pointing past the last line (D8).
+ # Every consumer of an unverified bound inherited it -- get_answer's
+ # hydrator, get_symbol and get_context all read this same return.
return BoundsCheck(
start_line=row.start_line,
- end_line=row.end_line,
+ end_line=min(row.end_line, len(lines)) if row.end_line else row.end_line,
verified=False,
approximate=True,
)
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py
index bbef71f65..502a86af3 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py
@@ -1085,13 +1085,13 @@ def _data_shape_prose(grounded: dict, citations: list[str]) -> tuple[str, str]:
def _is_question_named_body_cut_by_us(entry: dict, question_ids: set[str]) -> bool:
"""Whether this body is the question's own symbol, cut because WE ran out of lines.
- ``truncated`` alone is not trustworthy enough to demote on. It is set by
- comparing the INDEXED end line against what was read, while the read clamps
- to the end of the live file, so a symbol whose stored end overshoots (an
- unsupported language or a syntax error leaves the bounds unverified, and a
- file that shrank since indexing does it too) is served WHOLE and still
- flagged. Requiring the served span to have reached the line cap says the cut
- was ours, which is the only case where something was really withheld.
+ ``truncated`` alone is not trustworthy enough to demote on. The stale-bound
+ case this was written for is now fixed at its source — ``check_symbol_bounds``
+ clamps every bound to the live file (D8) — but the guard still earns its keep
+ on the ``source_excerpt`` fallback, where the served bytes come from the
+ index rather than from disk and the live length says nothing. Requiring the
+ served span to have reached the line cap says the cut was ours, which is the
+ only case where something was really withheld.
"""
if not (entry.get("truncated") and entry.get("continuation")):
return False
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py b/packages/server/src/repowise/server/mcp_server/tool_answer/config.py
index 51099d1b9..db38fc743 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/config.py
@@ -402,7 +402,19 @@ _HIGH_CONFIDENCE_SCORE_FLOOR = 1.5
# degraded returns, so a degraded payload has never been written to it. The
# synthesised shape is untouched. A bump here would invalidate every keyed
# install's cache, and re-synthesis is real provider spend, for nothing.
-_ANSWER_SCHEMA_VERSION = 15
+# v16: the same scanner again, in three places, and unlike the degraded rework
+# above every one of them changes the SYNTHESISED payload, which is the shape
+# that gets cached. (1) A run left open at end of file no longer masks to EOF,
+# so definitions below a phantom `"""` or `/*` are listed again — 26 real Rust
+# definitions across two corpus files, one of which had 3,002 lines masked.
+# (2) A cut inside a multi-line string keeps the symbol enclosing it, which
+# changes which entry is the HEADLINE `body_continues`, and therefore the note
+# and the get_symbol pointer. (3) An unverified symbol bound is clamped to the
+# live file, so a body served whole is no longer flagged `truncated` — and
+# v13/v14 cap confidence on that flag, so a cached row can carry a `medium`
+# this version would not produce. Cached pre-v16 rows carry all three, so they
+# must bypass.
+_ANSWER_SCHEMA_VERSION = 16
# Hard TTL on answer-cache rows. Commit-based invalidation (the payload's
# stamped ``_indexed_commit`` vs the repo's current head) is the primary
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py b/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py
index 9d6b0cc9a..af134eafb 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py
@@ -10,7 +10,7 @@ from __future__ import annotations
import re
from functools import lru_cache
from pathlib import Path
-from typing import Any
+from typing import Any, NamedTuple
from sqlalchemy import select
@@ -473,6 +473,12 @@ def attach_truncation_contract(
end at all, which is never a cut: it is tested explicitly rather than left
to ``indexed_end > end_served``, which would only agree with it while
``end_served`` stays non-negative.
+
+ ``indexed_end`` is trusted to lie within the live file, which is
+ ``check_symbol_bounds``'s job rather than this one's: it now clamps to
+ ``len(lines)`` on every return, so a stored end that overshoots cannot reach
+ here and flag a body served WHOLE as truncated (D8). Clamping again here
+ would be a second owner and a second disk read.
"""
if indexed_end and indexed_end > end_served:
entry["truncated"] = True
@@ -1181,8 +1187,12 @@ def _skip_regex(raw: str, i: int) -> int:
def _walk_string_state(
lines: tuple[str, ...], *, backticks: bool
-) -> tuple[set[int], bool]:
- """(lines starting inside a string/comment, template literal left open at EOF).
+) -> tuple[set[int], set[int], bool]:
+ """(lines starting inside a string, inside a block comment, literal left open).
+
+ The two sets are kept apart because callers need to tell them apart and this
+ is a hot path: a string body proves the enclosing expression is still open,
+ a comment between two declarations proves nothing.
``stack`` models template-literal nesting: a ``None`` frame is the string
part of a backtick literal, an ``int`` frame is the unclosed-brace depth
@@ -1208,13 +1218,22 @@ def _walk_string_state(
precision, measured as 1,926 fabrications against 60 on the 16 corpus files
where a flat walk and this one disagree.
"""
- masked: set[int] = set()
+ strings: set[int] = set()
+ comments: set[int] = set()
delim: str | None = None
in_block = False
+ # Line the currently-open ``delim`` run or ``/* */`` block started on. The
+ # two are mutually exclusive (a frame is only ever opened at code level), so
+ # one variable serves both.
+ open_line = 0
stack: list[int | None] = []
for n, raw in enumerate(lines, 1):
- if delim is not None or in_block or (stack and stack[-1] is None):
- masked.add(n)
+ # The three states are mutually exclusive: a frame is only ever pushed
+ # at code level, so an if/elif chain is faithful to the walk below.
+ if delim is not None or (stack and stack[-1] is None):
+ strings.add(n)
+ elif in_block:
+ comments.add(n)
elif not stack and not _QUOTEISH_RE.search(raw):
# Nothing on this line can open a string or comment, so the
# character walk below cannot change state. Most lines are this
@@ -1270,25 +1289,25 @@ def _walk_string_state(
continue
if delim is not None:
if raw.startswith(delim, i):
- delim, i = None, i + len(delim)
+ delim, open_line, i = None, 0, i + len(delim)
else:
i += 1
continue
if in_block:
if raw.startswith("*/", i):
- in_block, i = False, i + 2
+ in_block, open_line, i = False, 0, i + 2
else:
i += 1
continue
if raw.startswith('"""', i) or raw.startswith("'''", i):
- delim, i = raw[i : i + 3], i + 3
+ delim, open_line, i = raw[i : i + 3], n, i + 3
continue
if backticks and raw[i] == "`":
stack.append(None)
i += 1
continue
if raw.startswith("/*", i):
- in_block, i = True, i + 2
+ in_block, open_line, i = True, n, i + 2
continue
if raw[i] == "#" or raw.startswith("//", i):
break
@@ -1304,7 +1323,20 @@ def _walk_string_state(
if not raw[i].isspace():
prev = raw[i]
i += 1
- return masked, bool(stack)
+ # A run still open at EOF is a walk that lost track, not a file with an
+ # unterminated construct, and masking to EOF hides every definition below
+ # it. The template-literal stack has had this containment since the backtick
+ # work; ``delim`` and ``/* */`` never did, and both fire on the same shape --
+ # a delimiter belonging to another language, sitting inside a string this
+ # walk cannot see. Measured on Rust: ``${0%/*}`` in a raw string masked 103
+ # lines and cost 6 real ``fn``; ``description = """#`` masked 3,002 and cost
+ # 10. Discarding the trailing run under-masks instead, which costs a
+ # spurious name in a list rather than an absent real one.
+ if delim is not None:
+ strings = {n for n in strings if n < open_line}
+ elif in_block:
+ comments = {n for n in comments if n < open_line}
+ return strings, comments, bool(stack)
def _has_backtick_strings(file_path: str) -> bool:
@@ -1313,10 +1345,20 @@ def _has_backtick_strings(file_path: str) -> bool:
return dot != -1 and file_path[dot:].lower() in _BACKTICK_STRING_SUFFIXES
+class _Masked(NamedTuple):
+ """1-based line numbers, split by what is hiding them.
+
+ ``all`` is precomputed rather than unioned per call: every caller wants it,
+ and this is a cached whole-file walk.
+ """
+
+ strings: frozenset[int]
+ comments: frozenset[int]
+ all: frozenset[int]
+
+
@lru_cache(maxsize=8)
-def _string_masked_lines(
- lines: tuple[str, ...], backticks: bool = True
-) -> frozenset[int]:
+def _string_masked_lines(lines: tuple[str, ...], backticks: bool = True) -> _Masked:
"""1-based line numbers that START inside a multi-line string or comment.
Repowise's own docstrings are full of indented ``def``/``class`` examples,
@@ -1347,7 +1389,9 @@ def _string_masked_lines(
synthetic 1.2 MB file with one stray backtick on line 1. Bounded at two
walks, and the cache means it is paid once per file.
"""
- masked, template_left_open = _walk_string_state(lines, backticks=backticks)
+ strings, comments, template_left_open = _walk_string_state(
+ lines, backticks=backticks
+ )
if template_left_open:
# The lexer-lite lost track: a template literal opened and never closed,
# so every line below it is masked to EOF and every definition there is
@@ -1355,8 +1399,10 @@ def _string_masked_lines(
# missing symbols -- so it must not be the one we ship. Fall back to the
# pre-backtick walk for this file, which under-masks instead: the cost is
# a spurious name in a list, not an absent real one.
- masked, _ = _walk_string_state(lines, backticks=False)
- return frozenset(masked)
+ strings, comments, _ = _walk_string_state(lines, backticks=False)
+ return _Masked(
+ frozenset(strings), frozenset(comments), frozenset(strings | comments)
+ )
def _indent_width(raw: str) -> int:
@@ -1409,7 +1455,8 @@ def withheld_definitions(
lines = text.splitlines()
if lo < 1 or lo > len(lines):
return []
- masked = _string_masked_lines(tuple(lines), _has_backtick_strings(path))
+ mask = _string_masked_lines(tuple(lines), _has_backtick_strings(path))
+ masked = mask.all
def _entry(line_no: int, m: re.Match[str], *, cut: bool = False) -> dict:
name = m.group("name")
@@ -1460,7 +1507,16 @@ def withheld_definitions(
and n not in masked
and lines[n - 1].strip()[0] not in ")]}{"
]
- if _usable:
+ if lo in mask.strings:
+ # The cut is INSIDE a multi-line string, so the expression holding that
+ # string -- and everything enclosing it -- is still open at ``lo``.
+ # Without this the anchor reads from the first line BELOW the string,
+ # usually a top-level declaration at column 0, and the walk dies at once
+ # (D9: 8 real definitions lost across cli/cli and mui). A block COMMENT
+ # cannot stand in for this: one sitting between two methods would report
+ # the preceding method as continuing when it has already ended.
+ anchor = _UNBOUNDED_INDENT
+ elif _usable:
anchor = _indent_width(lines[_usable[0] - 1])
elif any(lines[n - 1].strip() for n in range(lo, _end + 1)):
# Every withheld line is a string body or a bracket tail, so whatever
diff --git a/tests/unit/server/mcp/test_answer_backtick_masking.py b/tests/unit/server/mcp/test_answer_backtick_masking.py
index a74fbf9f3..18031b8c3 100644
--- a/tests/unit/server/mcp/test_answer_backtick_masking.py
+++ b/tests/unit/server/mcp/test_answer_backtick_masking.py
@@ -184,7 +184,9 @@ def test_the_nested_guard_is_not_vacuous(tmp_path) -> None:
target = 11 # the `export function afterTheTemplate` line
flat_masked, _flat_open = _flat(lines)
assert target in flat_masked, sorted(flat_masked)
- assert target not in _string_masked_lines(lines), sorted(_string_masked_lines(lines))
+ assert target not in _string_masked_lines(lines).all, sorted(
+ _string_masked_lines(lines).all
+ )
def test_a_backtick_inside_a_regex_literal_does_not_open_a_template(tmp_path) -> None:
diff --git a/tests/unit/server/mcp/test_answer_mask_containment.py b/tests/unit/server/mcp/test_answer_mask_containment.py
new file mode 100644
index 000000000..316e38df3
--- /dev/null
+++ b/tests/unit/server/mcp/test_answer_mask_containment.py
@@ -0,0 +1,182 @@
+"""Masking containment, and the defect it left open.
+
+Both are found by walking every file in a multi-language corpus rather than
+sampling cut points (`local-stash/get-answer-dogfood/overmask_gate.py`), which
+is the instrument the random-cut sweep never was:
+
+* **D9** -- a cut inside a multi-line string must keep the symbol enclosing it,
+ and a cut inside a block COMMENT must not, because a comment can sit between
+ two declarations while a string body cannot.
+* **the EOF containment** -- a triple-quote or ``/*`` run still open at end of
+ file is a walk that lost track, so it must under-mask rather than hide every
+ definition below it. Both fire on a delimiter belonging to another language
+ sitting inside a string this walk cannot see.
+
+D8 is not here. Its owner turned out to be `check_symbol_bounds`, which already
+clamps to the live file on its other two returns, so it is covered in
+`test_verify_bounds.py`.
+"""
+
+from pathlib import Path
+
+from repowise.server.mcp_server.tool_answer.symbols import (
+ _string_masked_lines,
+ withheld_definitions,
+)
+
+# A Go test whose body is cut inside a raw string. The line below the string is
+# a top-level `func` at column 0, which is what the anchor used to read.
+_GO_CUT_IN_A_RAW_STRING = """\
+package main
+
+func TestThing(t *testing.T) {
+\trun(`
+\tquery {
+\t\trepository(owner: $owner) {
+\t\t\tname
+\t\t}
+\t}
+\t`)
+}
+
+func AfterIt() {
+}
+"""
+
+# A block comment BETWEEN two top-level functions, indented deeper than the code
+# around it. `first` has ended by the time the comment starts, so nothing may be
+# reported as continuing.
+#
+# The indent is what makes this discriminating rather than decorative. The
+# comment's opening line is not masked and pins the backward walk's running
+# minimum indent, so a comment at column 0 stops the walk on its own whatever
+# the anchor says. Here the `}` above is skipped as a bracket tail without
+# updating that minimum, so the walk runs on and reaches `first`.
+_JS_CUT_IN_A_BLOCK_COMMENT = """\
+export function first() {
+ return 1;
+}
+ /* prose about what comes next
+ more prose
+ and more
+ */
+export function second() {
+ return 2;
+}
+"""
+
+# `${0%/*}` inside a Rust raw string. `/*` is not a comment here, and nothing
+# closes it, so the parent masked to EOF and lost every `fn` below.
+_RUST_SLASH_STAR_IN_A_RAW_STRING = '''\
+fn shim() {
+ let script = r#"#!/bin/sh
+record_dir=${0%/*}
+printf '%s' "$@"
+"#;
+ write(script);
+}
+
+fn after_the_raw_string() {
+ ok()
+}
+'''
+
+# A triple quote is a Python/Kotlin/Java delimiter, not a Rust one. Same shape,
+# other delimiter: nothing closes it, so the parent masked to EOF. Verbatim from
+# `gleam/compiler-core/src/error.rs`, where it cost 3,002 lines.
+_RUST_TRIPLE_QUOTE_IN_A_STRING = '''\
+fn describe() {
+ let text = r#"
+description = """#
+"#;
+ emit(text);
+}
+
+fn after_the_triple_quote() {
+ ok()
+}
+'''
+
+
+def _write(tmp_path: Path, name: str, body: str) -> Path:
+ (tmp_path / name).write_text(body, encoding="utf-8")
+ return tmp_path
+
+
+def test_a_cut_inside_a_multi_line_string_keeps_the_enclosing_symbol(tmp_path):
+ """FAILS on c3fdccbd with `[]` -- D9, 8 measured instances on cli/cli + mui.
+
+ The cut is inside the raw string, so every line of it is masked and the
+ first usable withheld line is `func AfterIt` at column 0. The anchor read 0
+ and the backward walk exited on its first iteration.
+ """
+ root = _write(tmp_path, "thing_test.go", _GO_CUT_IN_A_RAW_STRING)
+
+ got = withheld_definitions(root, "thing_test.go:5-14")
+
+ head = next((d for d in got if d.get("body_continues")), None)
+ assert head is not None, got
+ assert head["name"] == "TestThing", got
+
+
+def test_a_cut_inside_a_block_comment_does_not_resurrect_the_symbol_above_it(tmp_path):
+ """The control for the fix above, and the reason it is not one line.
+
+ Verified to be discriminating rather than assumed: swap the anchor test to
+ `lo in mask.all` and this returns `first` with `body_continues`, four lines
+ after `first` ended. That is the whole argument for splitting the mask by
+ what is hiding the line.
+ """
+ root = _write(tmp_path, "pair.js", _JS_CUT_IN_A_BLOCK_COMMENT)
+
+ got = withheld_definitions(root, "pair.js:5-11")
+
+ assert not any(d.get("body_continues") for d in got), got
+
+
+def test_an_unterminated_block_comment_does_not_mask_to_end_of_file():
+ """26 real Rust definitions lost across the corpus before this.
+
+ Measured on `goose`: `${0%/*}` in a raw string masked 103 lines and took
+ six real `fn` with it. This does not "fail on the parent" in a useful way --
+ the parent's masker returns a bare frozenset, so it raises AttributeError
+ rather than the defect. The non-vacuity proof is `ablate_masking.py`'s
+ `eof-containment-block` row, which disables the branch on THIS build.
+ """
+ lines = tuple(_RUST_SLASH_STAR_IN_A_RAW_STRING.splitlines())
+
+ mask = _string_masked_lines(lines, False)
+
+ target = next(i for i, ln in enumerate(lines, 1) if "after_the_raw_string" in ln)
+ assert target not in mask.all, sorted(mask.all)
+
+
+def test_an_unterminated_triple_quote_run_does_not_mask_to_end_of_file():
+ """The same defect one delimiter over.
+
+ Measured on `gleam`: a triple quote inside a Rust string masked 3,002 lines
+ and took ten real definitions with it. Non-vacuity is `ablate_masking.py`'s
+ `eof-containment-delim` row, for the reason above.
+ """
+ lines = tuple(_RUST_TRIPLE_QUOTE_IN_A_STRING.splitlines())
+
+ mask = _string_masked_lines(lines, False)
+
+ target = next(i for i, ln in enumerate(lines, 1) if "after_the_triple_quote" in ln)
+ assert target not in mask.all, sorted(mask.all)
+
+
+def test_the_masker_reports_strings_and_comments_separately():
+ """The split D9's fix needs, from ONE walk on a documented hot path.
+
+ Both directions, because a split that files everything under one bucket
+ still satisfies a one-sided assertion.
+ """
+ comment_mask = _string_masked_lines(tuple(_JS_CUT_IN_A_BLOCK_COMMENT.splitlines()))
+ assert comment_mask.comments and not comment_mask.strings, comment_mask
+
+ string_mask = _string_masked_lines(tuple(_GO_CUT_IN_A_RAW_STRING.splitlines()))
+ assert string_mask.strings and not string_mask.comments, string_mask
+
+ for mask in (comment_mask, string_mask):
+ assert mask.all == mask.strings | mask.comments
diff --git a/tests/unit/server/mcp/test_verify_bounds.py b/tests/unit/server/mcp/test_verify_bounds.py
index d62229e6a..770ea210d 100644
--- a/tests/unit/server/mcp/test_verify_bounds.py
+++ b/tests/unit/server/mcp/test_verify_bounds.py
@@ -206,3 +206,33 @@ async def test_heal_symbol_row_persists_corrected_bounds(setup_mcp, factory, ses
async with get_session(factory) as s:
healed = (await s.execute(select(WikiSymbol).where(WikiSymbol.id == row.id))).scalar_one()
assert (healed.start_line, healed.end_line) == (42, 57)
+
+
+class TestUnverifiedBoundsAreStillClampedToTheLiveFile:
+ """D8: an unverified end that overshoots EOF made a whole body look cut.
+
+ The cheap gate clamps (``min(..., len(lines))``) and a re-parse returns real
+ bounds, so this was the one return handing back a raw stored end. Everything
+ downstream inherited it: get_answer's hydrator computes ``truncated`` from
+ it, and get_symbol / get_context slice with it.
+ """
+
+ def test_an_unrelocatable_symbol_does_not_report_an_end_past_eof(self) -> None:
+ # A name that is nowhere in the source, so both the cheap gate and the
+ # re-parse relocation fail and the approximate branch is taken.
+ row = _row(name="vanished", symbol_id="pkg/mod.py::vanished", end_line=900)
+
+ check = check_symbol_bounds(row, FRESH_SOURCE)
+
+ assert check.verified is False
+ assert check.approximate is True
+ assert check.end_line == len(FRESH_SOURCE.splitlines())
+
+ def test_an_unverified_end_inside_the_file_is_left_alone(self) -> None:
+ """The control: clamping must not shorten a bound that already fits."""
+ row = _row(name="vanished", symbol_id="pkg/mod.py::vanished", end_line=6)
+
+ check = check_symbol_bounds(row, FRESH_SOURCE)
+
+ assert check.verified is False
+ assert check.end_line == 6