Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(server): split tool_answer/symbols.py and give the string scanner one handler per state (#2548)

Raghav Chamadiya1ce7a8aff598
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/hydration.py b/packages/server/src/repowise/server/mcp_server/tool_answer/hydration.py
new file mode 100644
index 000000000..ea24c3d4d
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/hydration.py
@@ -0,0 +1,417 @@
+"""WikiSymbol hydration for retrieval hits.
+
+Attach to each ranked file the symbols it defines: a cheap name list for the
+candidate pool, and for the top hits the question-ranked symbols with live
+signatures and source excerpts.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import select
+
+from repowise.core.persistence.models import WikiSymbol
+from repowise.server.mcp_server._page_paths import hit_file_path
+from repowise.server.mcp_server._query_terms import content_terms, split_humps
+from repowise.server.mcp_server._verify import verify_and_heal
+from repowise.server.mcp_server.tool_answer.config import (
+ _DEFINES_MAX_FILES,
+ _DEFINES_PER_CANDIDATE,
+ _ENRICH_TOP_N_HITS,
+ _MAX_SYMBOLS_PER_HIT,
+ _MAX_SYMBOLS_TOP_HIT,
+ _RELEVANCE_DOC_CHARS,
+ _RELEVANCE_DOC_WEIGHT,
+ _RELEVANCE_NAME_WEIGHT,
+ _RELEVANCE_SIG_WEIGHT,
+ _RELEVANT_EXCERPT_MAX_SYMBOLS,
+ _STOPWORDS,
+ _SYNTH_FULL_BODY_MAX_SYMBOLS,
+ _SYNTH_FULL_SOURCE_LINES,
+)
+from repowise.server.mcp_server.tool_answer.source import (
+ _read_repo_text,
+ _read_signature_from_source,
+ _read_symbol_source,
+)
+
+# Suffixes stripped so a question's word reaches the identifier that answers it
+# ("routing" -> the `route` symbol). Longest first; never stems below 4 chars.
+_STEM_SUFFIXES = ("tion", "ing", "ion", "es", "ed", "er", "s")
+
+
+def _stem(token: str) -> str:
+ for suffix in _STEM_SUFFIXES:
+ if token.endswith(suffix) and len(token) - len(suffix) >= 4:
+ return token[: -len(suffix)]
+ return token
+
+
+def _text_stems(text: str) -> set[str]:
+ """Stemmed content tokens of *text*, hump- and separator-split."""
+ return {
+ _stem(tok.lower())
+ for tok in re.split(r"[^A-Za-z0-9]+", split_humps(text))
+ if len(tok) >= 3 and tok.lower() not in _STOPWORDS
+ }
+
+
+def _stem_hit(term: str, tokens: set[str]) -> bool:
+ """Whether *term* names one of *tokens*, allowing a shared 4-char root."""
+ if term in tokens:
+ return True
+ return any(
+ min(len(term), len(tok)) >= 4 and (term.startswith(tok) or tok.startswith(term))
+ for tok in tokens
+ )
+
+
+def _question_names_symbol(row, qids_lower: set[str]) -> bool:
+ """Whether an identifier from the question names this symbol.
+
+ Matches the symbol's own name, its full qualified name, or its parent (so a
+ class reaches its methods), never a substring of the qualified name, which
+ would match every symbol in a package sharing a word with the question.
+ """
+ if not qids_lower:
+ return False
+ name_lower = (row.name or "").lower()
+ parent_lower = (row.parent_name or "").lower()
+ return (
+ name_lower in qids_lower
+ or (row.qualified_name or "").lower() in qids_lower
+ or (bool(parent_lower) and parent_lower in qids_lower)
+ or any(
+ q in name_lower
+ for q in qids_lower
+ if len(q) >= 5 # avoid spurious substring matches on short tokens
+ )
+ )
+
+
+def _symbol_relevance(entry: dict, terms: set[str]) -> int:
+ """How strongly a symbol's own text answers the question's content terms.
+
+ Reads only what hydration already loaded, so it adds no I/O to the call.
+ """
+ if not terms:
+ return 0
+ name_tokens = _text_stems(entry.get("name") or "")
+ sig_tokens = _text_stems(entry.get("signature") or "")
+ # Docstrings are the bulk of the text to tokenize and the weakest signal, so
+ # they are only read once a term has missed the name and the signature.
+ doc_tokens: set[str] | None = None
+ score = 0
+ for term in terms:
+ if _stem_hit(term, name_tokens):
+ score += _RELEVANCE_NAME_WEIGHT
+ elif _stem_hit(term, sig_tokens):
+ score += _RELEVANCE_SIG_WEIGHT
+ else:
+ if doc_tokens is None:
+ doc_tokens = _text_stems(
+ (entry.get("docstring") or "")[:_RELEVANCE_DOC_CHARS]
+ )
+ if _stem_hit(term, doc_tokens):
+ score += _RELEVANCE_DOC_WEIGHT
+ return score
+
+
+# Definition kinds worth naming in `defines`, best first. Absent kinds (imports,
+# re-exports) are not emitted: they would fill the budget with names that
+# answer nothing.
+_DEFINE_KIND_RANK = {
+ "class": 0,
+ "interface": 1,
+ "struct": 1,
+ "enum": 1,
+ "type": 2,
+ "function": 3,
+ "method": 4,
+ "constant": 5,
+}
+
+
+async def _hydrate_candidate_defines(
+ session,
+ repo_id: str,
+ hits: list[dict],
+ question_ids: set[str] | None = None,
+) -> None:
+ """Mutate *hits* in place: attach ``_defines`` to the candidate-pool files.
+
+ Naming a candidate file's definitions turns "search this file" into "read
+ this line", and often answers a where-is-it question outright.
+
+ Deliberately cheap and shallow: one batched, index-covered query over at
+ most ``_DEFINES_MAX_FILES`` paths; names and start lines only (bodies live
+ elsewhere); line numbers are index-recorded, not verified, a navigation hint.
+
+ Order within a file: question-named symbols, then declaration kind, then
+ position. Private names are dropped unless the question named them.
+ """
+ qids = {q.lower() for q in (question_ids or set())}
+
+ paths = _candidate_paths(hits)
+ if not paths:
+ return
+
+ res = await session.execute(
+ select(
+ WikiSymbol.file_path,
+ WikiSymbol.name,
+ WikiSymbol.kind,
+ WikiSymbol.start_line,
+ ).where(
+ WikiSymbol.repository_id == repo_id,
+ WikiSymbol.file_path.in_(paths),
+ )
+ )
+
+ by_file: dict[str, list[tuple[int, int, str, int]]] = {}
+ for file_path, name, kind, start_line in res.all():
+ rank = _DEFINE_KIND_RANK.get((kind or "").lower())
+ if rank is None or not name:
+ continue
+ matched = name.lower() in qids
+ if not matched and name.startswith("_"):
+ continue
+ by_file.setdefault(file_path, []).append(
+ (0 if matched else 1, rank, name, start_line or 0)
+ )
+ defines = {path: _pick_defines(rows) for path, rows in by_file.items()}
+
+ for h in hits:
+ p = hit_file_path(h)
+ if p and defines.get(p):
+ h["_defines"] = defines[p]
+
+
+def _candidate_paths(hits: list[dict]) -> list[str]:
+ """Distinct hit file paths in rank order, capped at ``_DEFINES_MAX_FILES``."""
+ paths: list[str] = []
+ seen: set[str] = set()
+ for h in hits:
+ p = hit_file_path(h)
+ if not p or p in seen:
+ continue
+ seen.add(p)
+ paths.append(p)
+ if len(paths) >= _DEFINES_MAX_FILES:
+ break
+ return paths
+
+
+def _pick_defines(rows: list[tuple[int, int, str, int]]) -> list[tuple[str, int]]:
+ """The first ``_DEFINES_PER_CANDIDATE`` distinct names, best-ranked first."""
+ rows.sort(key=lambda r: (r[0], r[1], r[3]))
+ picked: list[tuple[str, int]] = []
+ taken: set[str] = set()
+ for _m, _r, name, start in rows:
+ if name in taken:
+ continue
+ taken.add(name)
+ picked.append((name, start))
+ if len(picked) >= _DEFINES_PER_CANDIDATE:
+ break
+ return picked
+
+
+async def _hydrate_symbols_for_hits(
+ session,
+ repo_id: str,
+ hits: list[dict],
+ ctx: Any = None,
+ question_ids: set[str] | None = None,
+ question: str = "",
+) -> None:
+ """Mutate `hits` in place: attach `symbols` list to top-N file_page hits.
+
+ Symbols the question names by identifier move to the top of their file's
+ list and get a ``source_excerpt``, so synthesis sees the body it is asked
+ about rather than only the enclosing class.
+
+ The top hit gets ``_MAX_SYMBOLS_TOP_HIT`` slots, the rest
+ ``_MAX_SYMBOLS_PER_HIT``. ``question`` ranks which symbols fill them and
+ earns the leading few a body, since a prose question names no identifier.
+ """
+ question_ids = question_ids or set()
+ qids_lower = {q.lower() for q in question_ids}
+ # Once per call: the question's terms, stemmed to match identifier roots.
+ term_stems = {_stem(t) for t in content_terms(question)}
+
+ enrich_paths = _enrich_paths(hits)
+ if not enrich_paths:
+ return
+
+ res = await session.execute(
+ select(WikiSymbol)
+ .where(
+ WikiSymbol.repository_id == repo_id,
+ WikiSymbol.file_path.in_(enrich_paths),
+ )
+ .order_by(WikiSymbol.file_path, WikiSymbol.start_line)
+ )
+ by_file: dict[str, list[dict]] = {}
+ repo_root = Path(str(ctx.path)) if ctx and ctx.path else None
+ session_factory = getattr(ctx, "session_factory", None)
+ # One live read per hydrated file, shared by the bounds gate and the
+ # signature/body slices. None when unreadable (missing/outside root).
+ text_cache: dict[str, str | None] = {}
+ for row in res.scalars().all():
+ if row.file_path not in text_cache:
+ text_cache[row.file_path] = _read_repo_text(repo_root, row.file_path)
+ entry = await _symbol_entry(
+ row,
+ text_cache[row.file_path],
+ repo_root,
+ session_factory,
+ qids_lower,
+ term_stems,
+ )
+ by_file.setdefault(row.file_path, []).append(entry)
+
+ for i, h in enumerate(hits):
+ path = h.get("target_path")
+ if path not in by_file:
+ continue
+ cap = _MAX_SYMBOLS_TOP_HIT if i == 0 else _MAX_SYMBOLS_PER_HIT
+ kept = _keep_symbols(by_file[path], h, cap)
+ text = text_cache.get(path)
+ _attach_relevant_excerpts(kept, repo_root, path, text)
+ _deepen_matched_excerpts(kept, repo_root, path, text)
+ # Sort final slice by start_line for natural reading order.
+ kept.sort(key=lambda s: s["start_line"])
+ h["symbols"] = kept
+
+
+def _enrich_paths(hits: list[dict]) -> list[str]:
+ """The top ``_ENRICH_TOP_N_HITS`` file_page paths, in retrieval-rank order."""
+ # `hits` is already sorted by descending score upstream.
+ enrich_paths: list[str] = []
+ for h in hits:
+ if len(enrich_paths) >= _ENRICH_TOP_N_HITS:
+ break
+ if h.get("target_path") and h.get("page_type") == "file_page":
+ enrich_paths.append(h["target_path"])
+ return enrich_paths
+
+
+async def _symbol_entry(
+ row,
+ text: str | None,
+ repo_root: Path | None,
+ session_factory: Any,
+ qids_lower: set[str],
+ term_stems: set[str],
+) -> dict[str, Any]:
+ """One hydrated symbol: verified bounds, live signature, match and relevance."""
+ # Trust contract (shared with get_symbol): verify stored bounds before
+ # slicing live text. Unverified, fall back to the stored signature and no
+ # body: consistent beats a live slice at the wrong lines.
+ if text is not None:
+ check = await verify_and_heal(session_factory, row, text)
+ start_line, end_line, verified = check.start_line, check.end_line, check.verified
+ else:
+ start_line, end_line, verified = row.start_line, row.end_line, False
+ # For constants/variables the stored signature is the verbatim assignment;
+ # the def-line reader would join unrelated following lines.
+ if row.kind in ("constant", "variable") or not verified:
+ rich_sig = None
+ else:
+ rich_sig = _read_signature_from_source(
+ repo_root, row.file_path, start_line, text=text
+ )
+ matched = _question_names_symbol(row, qids_lower)
+ entry: dict[str, Any] = {
+ "name": row.name,
+ "kind": row.kind,
+ "signature": rich_sig or row.signature,
+ "docstring": row.docstring or "",
+ "start_line": start_line,
+ "end_line": end_line,
+ "_matched": matched,
+ "_verified": verified,
+ }
+ # Scored once here, not per sort comparison.
+ entry["_relevance"] = _symbol_relevance(entry, term_stems)
+ if matched and verified:
+ src = _read_symbol_source(
+ repo_root, row.file_path, start_line, end_line, text=text
+ )
+ if src:
+ entry["source_excerpt"] = src
+ return entry
+
+
+def _keep_symbols(syms: list[dict], hit: dict, cap: int) -> list[dict]:
+ """The ``cap`` symbols of one file worth the synthesis context, in priority order."""
+ # Decides which symbols are kept; the caller restores reading order.
+ syms.sort(key=lambda s: (not s["_matched"], -s["_relevance"], s["start_line"]))
+ # Anchored symbols first, so siblings matching through their parent's name
+ # cannot evict the one the question asked about.
+ anchor_names = {a.get("name") for a in (hit.get("_anchor_symbols") or [])}
+ kept: list[dict] = [s for s in syms if s["name"] in anchor_names][:cap]
+ # Then the rest of the matched symbols, then unmatched, up to the cap.
+ kept.extend(s for s in syms if s["_matched"] and s not in kept)
+ kept = kept[:cap]
+ for s in syms:
+ if s in kept:
+ continue
+ if len(kept) >= cap:
+ break
+ kept.append(s)
+ return kept
+
+
+def _attach_relevant_excerpts(
+ kept: list[dict], repo_root: Path | None, path: str, text: str | None
+) -> None:
+ """Give the leading question-relevant symbols without an excerpt a body."""
+ # A prose question matches no identifier, so relevance earns the body.
+ # `kept` is still in priority order here.
+ bodied = 0
+ for s in kept:
+ if bodied >= _RELEVANT_EXCERPT_MAX_SYMBOLS:
+ break
+ if s.get("source_excerpt") or not s["_relevance"] or not s["_verified"]:
+ continue
+ src = _read_symbol_source(
+ repo_root,
+ path,
+ s["start_line"],
+ s.get("end_line") or 0,
+ text=text,
+ )
+ if src:
+ s["source_excerpt"] = src
+ bodied += 1
+
+
+def _deepen_matched_excerpts(
+ kept: list[dict], repo_root: Path | None, path: str, text: str | None
+) -> None:
+ """Re-read the leading matched excerpts at the inline-body depth."""
+ # Runs while `kept` is in priority order. Reading the leading few at the
+ # depth ``symbol_bodies`` serves keeps synthesis consistent with it; the
+ # short excerpt can stop before the logic. Bounded to protect the prompt.
+ upgraded = 0
+ for s in kept:
+ if upgraded >= _SYNTH_FULL_BODY_MAX_SYMBOLS:
+ break
+ if not s.get("_matched") or not s.get("source_excerpt"):
+ continue
+ fuller = _read_symbol_source(
+ repo_root,
+ path,
+ s["start_line"],
+ s.get("end_line") or 0,
+ max_lines=_SYNTH_FULL_SOURCE_LINES,
+ text=text,
+ )
+ if fuller:
+ s["source_excerpt"] = fuller
+ upgraded += 1
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/source.py b/packages/server/src/repowise/server/mcp_server/tool_answer/source.py
new file mode 100644
index 000000000..a41498890
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/source.py
@@ -0,0 +1,106 @@
+"""Live source reads behind every symbol slice get_answer serves.
+
+One bounded disk read per file, and the two views taken of it: a symbol's body
+at its indexed bounds, and its full declaration line as written.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from repowise.server.mcp_server.tool_answer.config import (
+ _MATCHED_SYMBOL_SOURCE_LINES,
+ _MAX_RICH_SIG_LINES,
+)
+
+
+def _read_repo_text(repo_root: Path | None, file_path: str) -> str | None:
+ """Read a repo file's live text, refusing paths outside the root.
+
+ The single disk read shared by the bounds gate and the signature/body
+ slices below, so a hydrated file is read once rather than once per helper.
+ """
+ if repo_root is None:
+ return None
+ try:
+ abs_path = (repo_root / file_path).resolve()
+ abs_path.relative_to(repo_root.resolve())
+ return abs_path.read_text(encoding="utf-8", errors="replace")
+ except (OSError, ValueError):
+ return None
+
+
+def _read_symbol_source(
+ repo_root: Path | None,
+ file_path: str,
+ start_line: int,
+ end_line: int,
+ max_lines: int = _MATCHED_SYMBOL_SOURCE_LINES,
+ *,
+ text: str | None = None,
+) -> str | None:
+ """Return the literal source body for a symbol, bounded to max_lines.
+
+ The code itself, so synthesis can answer "how does X work" without hedging.
+ ``text`` passes in source the caller already read, so a file is read once.
+ """
+ if start_line < 1:
+ return None
+ if text is None:
+ text = _read_repo_text(repo_root, file_path)
+ if text is None:
+ return None
+ lines = text.splitlines()
+ if start_line > len(lines):
+ return None
+ hi = end_line if end_line and end_line >= start_line else start_line + max_lines
+ hi = min(hi, start_line + max_lines, len(lines))
+ body = "\n".join(lines[start_line - 1 : hi])
+ return body
+
+
+# Where a signature stops when it does not end in a Python colon: after the
+# closing paren of the parameter list (with an optional return annotation), at a
+# trailing brace, or at a semicolon (an abstract/interface member).
+_SIG_TERMINATOR_RE = re.compile(r"\)\s*(?:->[^:{]*)?\s*[:{]|\{\s*$|;\s*$")
+
+
+def _read_signature_from_source(
+ repo_root: Path | None, file_path: str, start_line: int, *, text: str | None = None
+) -> str | None:
+ """Read the symbol's actual signature line from disk.
+
+ Returns the def/class line (or its multi-line continuation) verbatim from
+ the source file. Captures everything WikiSymbol.signature strips:
+ * base classes for `class Foo(Bar, Baz):`
+ * decorators (one line above the def)
+ * full type annotations across line continuations
+
+ ``text`` reuses the caller's already-read source (see _read_symbol_source).
+ None on any failure — caller falls back to the stored signature.
+ """
+ if text is None:
+ text = _read_repo_text(repo_root, file_path)
+ if text is None:
+ return None
+ lines = text.splitlines()
+ if not lines or start_line < 1 or start_line > len(lines):
+ return None
+ # Walk forward up to _MAX_RICH_SIG_LINES until we close the parenthesis
+ # group (Python signatures often span multiple lines for type hints).
+ sig_lines: list[str] = []
+ paren_depth = 0
+ for i in range(start_line - 1, min(start_line - 1 + _MAX_RICH_SIG_LINES, len(lines))):
+ line = lines[i]
+ sig_lines.append(line.strip())
+ paren_depth += line.count("(") - line.count(")")
+ stripped = line.rstrip()
+ # A trailing colon alone misses one-line bodies and brace languages.
+ if paren_depth <= 0 and (
+ stripped.endswith(":") or _SIG_TERMINATOR_RE.search(stripped)
+ ):
+ break
+ if not sig_lines:
+ return None
+ return " ".join(sig_lines)
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/string_mask.py b/packages/server/src/repowise/server/mcp_server/tool_answer/string_mask.py
new file mode 100644
index 000000000..1aa99c467
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/string_mask.py
@@ -0,0 +1,317 @@
+"""Which source lines start inside a string or a comment.
+
+A lexer-lite over whole files, used to keep definition-shaped text inside
+docstrings, template literals and block comments out of ``withheld_symbols``.
+"""
+
+from __future__ import annotations
+
+import re
+from functools import lru_cache
+from typing import NamedTuple
+
+# Anything that could open a string or a comment. A line with none of these
+# cannot change the scanner's state, so it skips the character walk.
+_QUOTEISH_RE = re.compile(r"""["'`#]|/[*/]""")
+
+
+def _skip_quoted(raw: str, i: int) -> int:
+ """Index just past the single- or double-quoted run starting at ``i``."""
+ quote, i = raw[i], i + 1
+ while i < len(raw):
+ if raw[i] == "\\":
+ i += 2
+ continue
+ if raw[i] == quote:
+ return i + 1
+ i += 1
+ return i
+
+
+# Extensions where a backtick opens a string: Go raw strings and JS/TS template
+# literals. Elsewhere a backtick is punctuation (markdown fences in doc comments
+# and heredocs), and a phantom frame there can re-close later, balanced, where
+# the containment cannot see it.
+_BACKTICK_STRING_SUFFIXES = frozenset(
+ {".go", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts"}
+)
+
+# Characters after which a ``/`` starts a regex rather than a division. Excludes
+# ``)``, ``]`` and alphanumerics, where a division's left operand ends.
+_REGEX_CAN_START_AFTER = frozenset("(,=:[!&|?{};+-*%~^<>\n")
+
+# ...and the keywords a regex can follow, which would otherwise read as a
+# division's left operand (``return /[`]/.test(x)``).
+_REGEX_CAN_START_AFTER_WORD = frozenset(
+ {
+ "return", "case", "typeof", "yield", "await", "throw", "in", "of",
+ "new", "delete", "instanceof", "do", "else", "void",
+ }
+)
+
+
+def _regex_position(raw: str, i: int, prev: str) -> bool:
+ """Whether the ``/`` at ``i`` starts a regex rather than a division."""
+ if prev in _REGEX_CAN_START_AFTER:
+ return True
+ if not (prev.isalnum() or prev == "_"):
+ return False
+ word = ""
+ j = i - 1
+ while j >= 0 and raw[j].isspace():
+ j -= 1
+ while j >= 0 and (raw[j].isalnum() or raw[j] == "_"):
+ word = raw[j] + word
+ j -= 1
+ return word in _REGEX_CAN_START_AFTER_WORD
+
+
+def _skip_regex(raw: str, i: int) -> int:
+ """Index past a regex literal starting at ``i``, or ``i`` if it is not one.
+
+ A backtick inside a regex would otherwise open a phantom template literal
+ that a later backtick closes, leaving a balanced stack the containment in
+ ``_string_masked_lines`` cannot see while the mask eats the lines between.
+
+ Regex literals cannot span lines, so a run with no closing ``/`` on the same
+ line is not one and is left alone.
+ """
+ j = i + 1
+ in_class = False
+ while j < len(raw):
+ c = raw[j]
+ if c == "\\":
+ j += 2
+ continue
+ if c == "[":
+ in_class = True
+ elif c == "]":
+ in_class = False
+ elif c == "/" and not in_class:
+ return j + 1
+ j += 1
+ return i
+
+
+class _StringWalk:
+ """The state ``_walk_string_state`` carries from one line to the next.
+
+ Each ``_scan_*`` method owns one lexical state. It consumes characters until
+ the state changes or the line ends and returns the index to resume at, so
+ the per-line loop only dispatches on a transition, never per character.
+ """
+
+ __slots__ = ("backticks", "comments", "delim", "in_block", "open_line", "stack", "strings")
+
+ def __init__(self, backticks: bool) -> None:
+ self.backticks = backticks
+ self.strings: set[int] = set()
+ self.comments: set[int] = set()
+ self.delim: str | None = None
+ self.in_block = False
+ # Where the open ``delim`` run or ``/* */`` block started; the two are
+ # mutually exclusive, so one variable serves both.
+ self.open_line = 0
+ self.stack: list[int | None] = []
+
+ def walk_line(self, n: int, raw: str) -> None:
+ stack = self.stack
+ # The three states are mutually exclusive: frames open only at code level.
+ if self.delim is not None or (stack and stack[-1] is None):
+ self.strings.add(n)
+ elif self.in_block:
+ self.comments.add(n)
+ elif not stack and not _QUOTEISH_RE.search(raw):
+ # Nothing here can change state; skipping keeps the scan cheap.
+ return
+ i = 0
+ # Last non-space character seen at CODE level, for the regex test below.
+ prev = "\n"
+ while i < len(raw):
+ if stack:
+ if stack[-1] is None:
+ i = self._scan_template_text(raw, i)
+ else:
+ i, prev = self._scan_interpolation(raw, i, prev)
+ elif self.delim is not None:
+ i = self._scan_to_close(raw, i, self.delim)
+ elif self.in_block:
+ i = self._scan_to_close(raw, i, "*/")
+ else:
+ i, prev = self._scan_code(raw, i, prev, n)
+
+ def _scan_to_close(self, raw: str, i: int, closer: str) -> int:
+ """Inside a ``delim`` run or a ``/* */`` block: find its terminator."""
+ j = raw.find(closer, i)
+ if j == -1:
+ return len(raw)
+ self.delim, self.in_block, self.open_line = None, False, 0
+ return j + len(closer)
+
+ def _scan_template_text(self, raw: str, i: int) -> int:
+ """Inside the string part of a backtick literal."""
+ stack = self.stack
+ while i < len(raw):
+ c = raw[i]
+ if c == "\\":
+ # An escaped backtick does not close the literal; misreading it
+ # inverts parity with a balanced stack the containment misses.
+ i += 2
+ elif raw.startswith("${", i):
+ stack.append(1)
+ return i + 2
+ elif c == "`":
+ stack.pop()
+ return i + 1
+ else:
+ i += 1
+ return i
+
+ def _scan_interpolation(self, raw: str, i: int, prev: str) -> tuple[int, str]:
+ """Inside a ``${...}``: code, until its braces balance or a backtick nests.
+
+ Ordinary code tokens apply here too, including the regex probe
+ (``${s.replace(/[`]/g, "")}``).
+ """
+ stack = self.stack
+ while i < len(raw):
+ c = raw[i]
+ if c == "{":
+ stack[-1] += 1
+ elif c == "}":
+ stack[-1] -= 1
+ if stack[-1] <= 0:
+ stack.pop()
+ return i + 1, c
+ elif c == "`":
+ stack.append(None)
+ return i + 1, c
+ elif raw.startswith("//", i):
+ return len(raw), prev
+ elif c == "/" and _regex_position(raw, i, prev):
+ j = _skip_regex(raw, i)
+ if j > i:
+ i, prev = j, "/"
+ continue
+ elif c in ('"', "'"):
+ i = _skip_quoted(raw, i)
+ prev = '"'
+ continue
+ if not c.isspace():
+ prev = c
+ i += 1
+ return i, prev
+
+ def _scan_code(self, raw: str, i: int, prev: str, n: int) -> tuple[int, str]:
+ """At code level: skip ordinary strings and regexes until a frame opens."""
+ backticks = self.backticks
+ while i < len(raw):
+ if raw.startswith('"""', i) or raw.startswith("'''", i):
+ self.delim, self.open_line = raw[i : i + 3], n
+ return i + 3, prev
+ c = raw[i]
+ if backticks and c == "`":
+ self.stack.append(None)
+ return i + 1, prev
+ if raw.startswith("/*", i):
+ self.in_block, self.open_line = True, n
+ return i + 2, prev
+ if c == "#" or raw.startswith("//", i):
+ return len(raw), prev
+ if c == "/" and _regex_position(raw, i, prev):
+ j = _skip_regex(raw, i)
+ if j > i:
+ i, prev = j, "/"
+ continue
+ if c in ('"', "'"):
+ i = _skip_quoted(raw, i)
+ prev = '"'
+ continue
+ if not c.isspace():
+ prev = c
+ i += 1
+ return i, prev
+
+ def result(self) -> tuple[set[int], set[int], bool]:
+ # A run still open at EOF means the walk lost track (usually another
+ # language's delimiter inside an untracked string). Discard it: a
+ # spurious name in a list beats hiding every definition below.
+ strings, comments = self.strings, self.comments
+ if self.delim is not None:
+ strings = {n for n in strings if n < self.open_line}
+ elif self.in_block:
+ comments = {n for n in comments if n < self.open_line}
+ return strings, comments, bool(self.stack)
+
+
+def _walk_string_state(
+ lines: tuple[str, ...], *, backticks: bool
+) -> tuple[set[int], set[int], bool]:
+ """(lines starting inside a string, inside a block comment, literal left open).
+
+ The sets are kept apart: a string body proves the enclosing expression is
+ still open, a comment between two declarations proves nothing.
+
+ ``stack`` models template-literal nesting: a ``None`` frame is the string
+ part of a backtick literal, an ``int`` frame the brace depth inside a
+ ``${...}`` interpolation. An interpolation holds code, so its lines are not
+ masked and a backtick there opens a nested literal. A flat counter would
+ read that backtick as closing the outer literal, then apply code rules
+ (``#``, ``//``) to string content. Escaped backticks and regexes are the
+ other triggers of that shape, and worse, since they leave the stack balanced.
+ """
+ walk = _StringWalk(backticks)
+ for n, raw in enumerate(lines, 1):
+ walk.walk_line(n, raw)
+ return walk.result()
+
+
+def _has_backtick_strings(file_path: str) -> bool:
+ """Whether a backtick opens a string in this file's language."""
+ dot = file_path.rfind(".")
+ return dot != -1 and file_path[dot:].lower() in _BACKTICK_STRING_SUFFIXES
+
+
+class _Masked(NamedTuple):
+ """1-based line numbers, split by what is hiding them.
+
+ ``all`` is precomputed: every caller wants it, and the walk is cached.
+ """
+
+ strings: frozenset[int]
+ comments: frozenset[int]
+ all: frozenset[int]
+
+
+@lru_cache(maxsize=8)
+def _string_masked_lines(lines: tuple[str, ...], backticks: bool = True) -> _Masked:
+ """1-based line numbers that START inside a multi-line string or comment.
+
+ Without this, ``def``/``class`` examples inside docstrings become symbol ids
+ that resolve to nothing.
+
+ Deliberately a lexer-lite: Python triple quotes, backtick template literals
+ and Go raw strings, C-style ``/* */`` blocks; stops at ``#`` / ``//``. Known
+ ceilings:
+
+ * C# verbatim and Rust raw strings are not tracked: rare, and their
+ contents seldom look like definitions.
+ * Inside ``${...}``, ``/* */`` and ``#`` are not handled, so a ``}`` in a
+ comment there can close the frame early; such cases end unbalanced and
+ hit the fallback below.
+ * Markdown fences and inline code spans are masked too (a ``def`` in a
+ fence is not a definition).
+
+ Cached on the line tuple: a per-character walk that the homonym path would
+ otherwise repeat per truncated body. The fallback bounds it at two walks.
+ """
+ strings, comments, template_left_open = _walk_string_state(
+ lines, backticks=backticks
+ )
+ if template_left_open:
+ # A literal never closed, which would silently mask to EOF. Re-walk
+ # without backticks: under-masking costs a spurious name, not a real one.
+ strings, comments, _ = _walk_string_state(lines, backticks=False)
+ return _Masked(
+ frozenset(strings), frozenset(comments), frozenset(strings | comments)
+ )
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py b/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py
index 7469b8251..80447f217 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py
@@ -1,146 +1,87 @@
-"""Question identifier extraction + WikiSymbol hydration for retrieval hits.
+"""Question identifier extraction + symbol anchoring for retrieval hits.
-The pieces that turn a ranked file into LLM-ready symbol context: pull the
-identifiers a question names, read real signatures/source from disk, and
-promote question-matched symbols to the top of each hit's symbol list.
+Pull the identifiers a question names, anchor the files that define them, and
+inline the union of a homonym's bodies. Hydration, live source reads, masking
+and the withheld-definition scan live in sibling modules and are re-exported
+here, since callers and tests import them from this module.
"""
from __future__ import annotations
import re
-from functools import lru_cache
+from collections.abc import Awaitable, Callable
from pathlib import Path
-from typing import Any, NamedTuple
+from typing import Any
from sqlalchemy import select
from repowise.core.persistence.models import WikiSymbol
-from repowise.server.mcp_server._page_paths import hit_file_path
-from repowise.server.mcp_server._query_terms import content_terms, split_humps
from repowise.server.mcp_server._verify import verify_and_heal
from repowise.server.mcp_server.tool_answer.config import (
- _DEFINES_MAX_FILES,
- _DEFINES_PER_CANDIDATE,
_ENRICH_TOP_N_HITS,
_HIGH_CONFIDENCE_SCORE_FLOOR,
_HOMONYM_UNION_BODY_MAX_LINES,
_HOMONYM_UNION_CHAR_BUDGET,
_HOMONYM_UNION_PROSE_DEF_CEILING,
- _MATCHED_SYMBOL_SOURCE_LINES,
- _MAX_RICH_SIG_LINES,
- _MAX_SYMBOLS_PER_HIT,
- _MAX_SYMBOLS_TOP_HIT,
- _RELEVANCE_DOC_CHARS,
- _RELEVANCE_DOC_WEIGHT,
- _RELEVANCE_NAME_WEIGHT,
- _RELEVANCE_SIG_WEIGHT,
- _RELEVANT_EXCERPT_MAX_SYMBOLS,
_STOPWORDS,
- _SYNTH_FULL_BODY_MAX_SYMBOLS,
- _SYNTH_FULL_SOURCE_LINES,
+)
+from repowise.server.mcp_server.tool_answer.hydration import ( # noqa: F401 re-exported
+ _DEFINE_KIND_RANK,
+ _STEM_SUFFIXES,
+ _hydrate_candidate_defines,
+ _hydrate_symbols_for_hits,
+ _question_names_symbol,
+ _stem,
+ _stem_hit,
+ _symbol_relevance,
+ _text_stems,
+)
+from repowise.server.mcp_server.tool_answer.source import ( # noqa: F401 re-exported
+ _SIG_TERMINATOR_RE,
+ _read_repo_text,
+ _read_signature_from_source,
+ _read_symbol_source,
+)
+from repowise.server.mcp_server.tool_answer.string_mask import ( # noqa: F401 re-exported
+ _BACKTICK_STRING_SUFFIXES,
+ _QUOTEISH_RE,
+ _REGEX_CAN_START_AFTER,
+ _REGEX_CAN_START_AFTER_WORD,
+ _has_backtick_strings,
+ _Masked,
+ _regex_position,
+ _skip_quoted,
+ _skip_regex,
+ _string_masked_lines,
+ _walk_string_state,
+)
+from repowise.server.mcp_server.tool_answer.withheld import ( # noqa: F401 re-exported
+ _BRACE_MEMBER,
+ _DECL_MODIFIERS,
+ _FIRST_WORD_RE,
+ _NOT_A_DEFINITION,
+ _RESERVED_NAMES,
+ _UNBOUNDED_INDENT,
+ _WITHHELD_DEF_PATTERNS,
+ _WITHHELD_MAX_SYMBOLS,
+ _indent_width,
+ _match_definition,
+ withheld_definitions,
)
from repowise.server.mcp_server.tool_search import _prose_dominates
-# Suffixes stripped so a question's word reaches the identifier that answers it
-# ("routing" -> the `route` symbol). Longest first; never stems below 4 chars.
-_STEM_SUFFIXES = ("tion", "ing", "ion", "es", "ed", "er", "s")
-
-
-def _stem(token: str) -> str:
- for suffix in _STEM_SUFFIXES:
- if token.endswith(suffix) and len(token) - len(suffix) >= 4:
- return token[: -len(suffix)]
- return token
-
-
-def _text_stems(text: str) -> set[str]:
- """Stemmed content tokens of *text*, hump- and separator-split."""
- return {
- _stem(tok.lower())
- for tok in re.split(r"[^A-Za-z0-9]+", split_humps(text))
- if len(tok) >= 3 and tok.lower() not in _STOPWORDS
- }
-
-
-def _stem_hit(term: str, tokens: set[str]) -> bool:
- """Whether *term* names one of *tokens*, allowing a shared 4-char root."""
- if term in tokens:
- return True
- return any(
- min(len(term), len(tok)) >= 4 and (term.startswith(tok) or tok.startswith(term))
- for tok in tokens
- )
-
-
-def _question_names_symbol(row, qids_lower: set[str]) -> bool:
- """Whether an identifier from the question names this symbol.
-
- Substring-matching the whole qualified name marked every symbol in a package
- whose path shares a word with the question, which flattened the promotion to
- a no-op. Matching is against the symbol's own name, its full qualified name,
- or its parent: asking about a class should still reach its methods.
- """
- if not qids_lower:
- return False
- name_lower = (row.name or "").lower()
- parent_lower = (row.parent_name or "").lower()
- return (
- name_lower in qids_lower
- or (row.qualified_name or "").lower() in qids_lower
- or (bool(parent_lower) and parent_lower in qids_lower)
- or any(
- q in name_lower
- for q in qids_lower
- if len(q) >= 5 # avoid spurious substring matches on short tokens
- )
- )
-
-
-def _symbol_relevance(entry: dict, terms: set[str]) -> int:
- """How strongly a symbol's own text answers the question's content terms.
-
- Reads only what hydration already loaded, so it adds no I/O to the call.
- """
- if not terms:
- return 0
- name_tokens = _text_stems(entry.get("name") or "")
- sig_tokens = _text_stems(entry.get("signature") or "")
- # Docstrings are the bulk of the text to tokenize and the weakest signal, so
- # they are only read once a term has missed the name and the signature.
- doc_tokens: set[str] | None = None
- score = 0
- for term in terms:
- if _stem_hit(term, name_tokens):
- score += _RELEVANCE_NAME_WEIGHT
- elif _stem_hit(term, sig_tokens):
- score += _RELEVANCE_SIG_WEIGHT
- else:
- if doc_tokens is None:
- doc_tokens = _text_stems(
- (entry.get("docstring") or "")[:_RELEVANCE_DOC_CHARS]
- )
- if _stem_hit(term, doc_tokens):
- score += _RELEVANCE_DOC_WEIGHT
- return score
-
def _extract_question_identifiers(question: str) -> set[str]:
"""Pull out Python-looking identifiers the question names explicitly.
- Targets: snake_case (``_local_reachability_density``), CamelCase
- (``NearestCentroid``), dotted paths (``BaseLabelPropagation.fit``).
- Filtered to ≥3 chars, non-stopwords, non-pure-lowercase-English (unless
- they contain an underscore or a digit — otherwise every common word
- matches). The result drives question-aware symbol promotion in
- ``_hydrate_symbols_for_hits``.
+ snake_case, CamelCase and dotted paths, at least 3 chars and not
+ stopwords. Drives question-aware promotion in ``_hydrate_symbols_for_hits``.
"""
import re
ids: set[str] = set()
- # Match bare identifiers and dotted paths: first char letter/underscore,
- # rest alnum/underscore, optionally with dotted continuations.
for tok in re.findall(r"[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*", question):
- # Split dotted paths into both the full thing and the leaf.
+ # A dotted path yields both the full thing and each part.
parts = tok.split(".")
candidates = [tok, *parts]
for c in candidates:
@@ -148,12 +89,7 @@ def _extract_question_identifiers(question: str) -> set[str]:
continue
if c.lower() in _STOPWORDS:
continue
- # Heuristic: keep if it contains an uppercase letter anywhere
- # (covers CamelCase and sentence-initial capitalised nouns like
- # ``Version`` that are typically class names in Python), a
- # digit, or an underscore. Pure-lowercase English words like
- # ``method`` / ``class`` / ``dtype`` are dropped — they are
- # poor promotion signals and match too broadly.
+ # Pure-lowercase words (``method``, ``class``) match too broadly.
has_upper = any(ch.isupper() for ch in c)
has_under = "_" in c
has_digit = any(ch.isdigit() for ch in c)
@@ -167,24 +103,11 @@ def union_defers_to_synthesis(
) -> bool:
"""True when an answer-by-union should fall through to synthesis.
- Answer-by-union is the right reply for a small set of genuine parallel
- implementations the question is actually about (``_severity_for`` has 4
- across the biomarkers). It is the WRONG reply when a prose question merely
- *mentions* a generic method that happens to have many definitions: measured,
- "how does a wiki page get its provider_name during indexing?" dumped 12
- unrelated provider stubs as a confidence=high answer, and a ``to_dict``
- mention dumped 28. Two signals must both hold before deferring, so the
- narrowest population is affected:
-
- * ``_prose_dominates`` — the query reads as prose, not a bare symbol lookup.
- A bare ``provider_name`` (prose does not dominate) still unions: that
- caller explicitly asked for every definition.
- * the def count exceeds ``_HOMONYM_UNION_PROSE_DEF_CEILING`` — past a
- handful, the name is a generic method, not a small parallel-impl set.
+ A union answers a small set of parallel implementations, not a prose
+ question that merely mentions a generic many-def method. Both must hold:
- Small genuine unions and explicit lookups are untouched; only a prose
- question naming a many-def generic method falls through to synthesis (which
- grounds in the file the question is really about).
+ * ``_prose_dominates``: a bare symbol lookup still unions.
+ * The def count exceeds ``_HOMONYM_UNION_PROSE_DEF_CEILING``.
"""
if not union_groups:
return False
@@ -197,21 +120,13 @@ def union_defers_to_synthesis(
def is_symbol_lookup_question(question: str, question_ids: set[str]) -> bool:
"""True when the question IS the symbol names, not prose that mentions them.
- ``ModelAdmin`` is a lookup; "how does ModelAdmin dispatch a request" is
- prose that merely names one. The distinction matters wherever the question
- is whether a served BODY is the answer: for a lookup it is, so truncating
- it is a loss on its own; for prose the body is evidence for a claim, and
- truncation alone says little (22% of truncations withhold nothing the
- response leans on).
+ For a lookup the served body is the answer, so truncating it is a loss on
+ its own; for prose the body is evidence for a claim.
- **Stricter than ``_prose_dominates``, deliberately.** That predicate counts
- ``[A-Za-z0-9_]+`` tokens, so a question written in Cyrillic, Japanese or
- Chinese tokenises to nothing but its identifiers and reads as a bare
- lookup — and repowise ships an output-language feature, so those callers
- exist. It also misreads dense English ("Why does ModelAdmin call
- get_queryset, get_form and save_model?" is 4 identifiers in 7 tokens).
- Removing the identifiers and asking whether any word character survives is
- script-independent and says what "bare lookup" actually means.
+ Stricter than ``_prose_dominates``, which counts ASCII tokens and so reads
+ non-Latin-script questions and identifier-dense English as bare lookups.
+ Removing the identifiers and checking for any surviving word character is
+ script-independent.
"""
if not question_ids:
return False
@@ -223,114 +138,12 @@ def is_symbol_lookup_question(question: str, question_ids: set[str]) -> bool:
return not _prose_dominates(question, list(question_ids))
-def _read_repo_text(repo_root: Path | None, file_path: str) -> str | None:
- """Read a repo file's live text, refusing paths outside the root.
-
- The single disk read shared by the bounds gate and the signature/body
- slices below, so a hydrated file is read once rather than once per helper.
- """
- if repo_root is None:
- return None
- try:
- abs_path = (repo_root / file_path).resolve()
- abs_path.relative_to(repo_root.resolve())
- return abs_path.read_text(encoding="utf-8", errors="replace")
- except (OSError, ValueError):
- return None
-
-
-def _read_symbol_source(
- repo_root: Path | None,
- file_path: str,
- start_line: int,
- end_line: int,
- max_lines: int = _MATCHED_SYMBOL_SOURCE_LINES,
- *,
- text: str | None = None,
-) -> str | None:
- """Return the literal source body for a symbol, bounded to max_lines.
-
- The bounded source is the key ingredient for question-matched symbols.
- The LLM was already getting the file-level summary and a truncated
- docstring; what it was missing was the actual code. With 40 lines of
- the method body in front of it, the synthesis step can answer "how
- does X work" without hedging back to "you should inspect the source".
-
- ``text`` lets a caller that already read the file (the hydrator reads it
- once for the bounds gate) pass the live source in, so a hydrated file is
- read once instead of once per symbol.
- """
- if start_line < 1:
- return None
- if text is None:
- text = _read_repo_text(repo_root, file_path)
- if text is None:
- return None
- lines = text.splitlines()
- if start_line > len(lines):
- return None
- hi = end_line if end_line and end_line >= start_line else start_line + max_lines
- hi = min(hi, start_line + max_lines, len(lines))
- body = "\n".join(lines[start_line - 1 : hi])
- return body
-
-
-# Where a signature stops when it does not end in a Python colon: after the
-# closing paren of the parameter list (with an optional return annotation), at a
-# trailing brace, or at a semicolon (an abstract/interface member).
-_SIG_TERMINATOR_RE = re.compile(r"\)\s*(?:->[^:{]*)?\s*[:{]|\{\s*$|;\s*$")
-
-
-def _read_signature_from_source(
- repo_root: Path | None, file_path: str, start_line: int, *, text: str | None = None
-) -> str | None:
- """Read the symbol's actual signature line from disk.
-
- Returns the def/class line (or its multi-line continuation) verbatim from
- the source file. Captures everything WikiSymbol.signature strips:
- * base classes for `class Foo(Bar, Baz):`
- * decorators (one line above the def)
- * full type annotations across line continuations
-
- ``text`` reuses the caller's already-read source (see _read_symbol_source).
- None on any failure — caller falls back to the stored signature.
- """
- if text is None:
- text = _read_repo_text(repo_root, file_path)
- if text is None:
- return None
- lines = text.splitlines()
- if not lines or start_line < 1 or start_line > len(lines):
- return None
- # Walk forward up to _MAX_RICH_SIG_LINES until we close the parenthesis
- # group (Python signatures often span multiple lines for type hints).
- sig_lines: list[str] = []
- paren_depth = 0
- for i in range(start_line - 1, min(start_line - 1 + _MAX_RICH_SIG_LINES, len(lines))):
- line = lines[i]
- sig_lines.append(line.strip())
- paren_depth += line.count("(") - line.count(")")
- stripped = line.rstrip()
- # "ends with a colon" alone leaves a one-line body (``def go(self): pass``)
- # and every brace language (``func f() error {``, ``render() {``) with no
- # terminator at all, so the signature absorbs the lines after it.
- if paren_depth <= 0 and (
- stripped.endswith(":") or _SIG_TERMINATOR_RE.search(stripped)
- ):
- break
- if not sig_lines:
- return None
- return " ".join(sig_lines)
-
-
def _extract_value_answer(hits: list[dict], question_ids: set[str]) -> dict | None:
"""Verbatim-assignment answer for value-shaped questions (the C1 fast path).
- When a question names an identifier and the hydrator matched a
- constant/variable symbol in the top hits, the symbol's signature IS the
- answer — the verbatim assignment line read from live source. No LLM
- call, nothing to hedge, nothing to invent. Exact name matches win over
- substring matches.
+ A matched constant/variable's signature is the verbatim assignment from
+ live source, so it is the answer, with no LLM call. Exact name matches win
+ over substring matches.
"""
qids_lower = {q.lower() for q in question_ids}
candidates: list[dict] = []
@@ -351,8 +164,7 @@ def _extract_value_answer(hits: list[dict], question_ids: set[str]) -> dict | No
"line": s.get("start_line"),
"answer": f"{sig} ({path}:{s.get('start_line')})",
}
- # Multi-line values (dicts/arrays): the hydrator attached the
- # live body — include it so the agent never needs a follow-up.
+ # Multi-line values: include the live body the hydrator attached.
excerpt = s.get("source_excerpt")
if excerpt and excerpt.strip() != sig.strip():
entry["value_source"] = excerpt
@@ -385,29 +197,19 @@ async def _anchor_symbol_hits(
) -> tuple[list[dict], dict[str, Any]]:
"""Inject the defining file of a question-named indexed symbol into hits.
- BM25 / vector retrieval misses deep-path files even when the named symbol
- is indexed — "explain DecisionExtractor.extract_all" ranks the pipeline
- orchestrators above ``analysis/decisions/extractor.py`` and never surfaces
- the definition, so synthesis hedges and ``symbol_bodies`` can't fire. When
- a question identifier resolves to a single indexed function / method /
- class, prepend (or boost) its defining file as the dominant hit so the
- answer grounds in the actual definition.
+ Fuzzy retrieval can miss a deep-path file even when the named symbol is
+ indexed. A question identifier resolving to one indexed def makes its file
+ the dominant hit, so the answer grounds in the definition.
- Homonyms (N>=2 defs of one name) split three ways:
+ Homonyms (several defs of one name) split three ways:
- * The question names the parent / qualifies the name so exactly one def
- survives → anchor that def (as before).
- * The question does NOT qualify the name → the whole def set is returned in
- ``homonyms["union"]`` so the caller can inline the UNION of bodies instead
- of bailing to a best_guesses pointer list (the pointer list is exactly
- what triggers the agent's get_symbol/get_context drill). This is the fix
- for the retrieval-MISS class (``_severity_for`` x 4) - the defs are never
- in the fuzzy candidate set, so an exact-name index scan is the only thing
- that surfaces them.
- * The question qualifies the name (``Parent.leaf``) but NO def matches that
- qualifier → recorded in ``homonyms["qualified_miss"]`` so the caller can
- return not-found instead of synthesizing from a same-named symbol
- elsewhere (a precise query must never degrade to a confident wrong answer).
+ * The question narrows it to exactly one def: anchor that def.
+ * The question does not qualify the name: the defs go to
+ ``homonyms["union"]`` so the caller inlines the union of bodies, which
+ fuzzy retrieval would never surface.
+ * The question qualifies the name (``Parent.leaf``) but no def matches:
+ ``homonyms["qualified_miss"]``, so the caller answers not-found rather
+ than from a same-named symbol elsewhere.
Returns ``(hits, homonyms)``; ``hits`` is re-sorted by score (mutated in
place). ``homonyms = {"union": {name: [def_dict, ...]}, "qualified_miss":
@@ -417,8 +219,6 @@ async def _anchor_symbol_hits(
if not question_ids:
return hits, homonyms
qids_lower = {q.lower() for q in question_ids}
- # Qualifiers the question used (dotted forms like ``decisionextractor.extract_all``).
- qualifiers = {q for q in qids_lower if "." in q}
res = await session.execute(
select(WikiSymbol).where(
WikiSymbol.repository_id == repo_id,
@@ -428,19 +228,15 @@ async def _anchor_symbol_hits(
)
by_name: dict[str, list] = {}
for row in res.scalars().all():
- # A pathless row names nothing the reply can point at. Dropping it here
- # keeps it out of every route below: the anchor, and both union branches
- # that would otherwise read the live file at an empty path.
+ # A pathless row names nothing the reply can point at: drop it from
+ # every route below.
if not row.file_path:
continue
by_name.setdefault(row.name, []).append(row)
- # Verify bounds against the live file before any body is sliced from a
- # stored range. Both the answer-by-union bodies (grounding=exact_symbol,
- # confidence=high) and the anchored tier-0 symbol_bodies serve live source at
- # these bounds, so a drifted row would otherwise ground the strongest-trust
- # answer in the wrong lines. Cheap gate first (string check); a re-parse fires
- # only on a genuine miss and heals the row. One live read per file, cached.
+ # Verify bounds before slicing: union and anchored bodies are served at
+ # high trust, so a drifted row would ground them in the wrong lines.
+ # One live read per file, cached.
_text_cache: dict[str, str | None] = {}
async def _verified_dict(row) -> dict:
@@ -457,6 +253,36 @@ async def _anchor_symbol_hits(
d["_approx"] = True
return d
+ chosen = await _choose_anchor_symbols(by_name, qids_lower, _verified_dict, homonyms)
+ if not chosen:
+ return hits, homonyms
+ await _anchor_chosen_symbols(hits, chosen, _verified_dict)
+ hits.sort(key=lambda h: h.get("score", 0.0), reverse=True)
+ return hits, homonyms
+
+
+def _question_names_parent(cand, qids_lower: set[str]) -> bool:
+ """Whether the question names this def's parent, or a qualifier of it."""
+ if (cand.parent_name or "").lower() in qids_lower:
+ return True
+ qualified = (cand.qualified_name or "").lower()
+ name = (cand.name or "").lower()
+ return any(q in qualified for q in qids_lower if len(q) >= 4 and q != name)
+
+
+async def _choose_anchor_symbols(
+ by_name: dict[str, list],
+ qids_lower: set[str],
+ verified_dict: Callable[[Any], Awaitable[dict]],
+ homonyms: dict[str, Any],
+) -> list:
+ """The one def per question-named symbol to anchor on.
+
+ Names with several defs that cannot be narrowed to one are recorded in
+ *homonyms* instead, as a union or a qualified miss.
+ """
+ # Qualifiers the question used (dotted forms like ``decisionextractor.extract_all``).
+ qualifiers = {q for q in qids_lower if "." in q}
chosen: list = []
for name, cands in by_name.items():
if len(cands) == 1:
@@ -464,16 +290,7 @@ async def _anchor_symbol_hits(
continue
# Disambiguate a homonym when the question names its parent or the
# parent appears in the qualified name.
- narrowed = [
- c
- for c in cands
- if (c.parent_name or "").lower() in qids_lower
- or any(
- q in (c.qualified_name or "").lower()
- for q in qids_lower
- if len(q) >= 4 and q != (c.name or "").lower()
- )
- ]
+ narrowed = [c for c in cands if _question_names_parent(c, qids_lower)]
if len(narrowed) == 1:
chosen.append(narrowed[0])
continue
@@ -483,54 +300,37 @@ async def _anchor_symbol_hits(
if narrowed:
# Qualifier matched >1 def: union of the narrowed set (still all
# genuine candidates for the qualified name).
- homonyms["union"][name] = [await _verified_dict(c) for c in narrowed]
+ homonyms["union"][name] = [await verified_dict(c) for c in narrowed]
elif targeted:
# Qualifier present but matched nothing: do not guess.
homonyms["qualified_miss"].append(name)
else:
# Bare homonym, no qualifier: union of every def.
- homonyms["union"][name] = [await _verified_dict(c) for c in cands]
+ homonyms["union"][name] = [await verified_dict(c) for c in cands]
+ return chosen
- if not chosen:
- return hits, homonyms
+async def _anchor_chosen_symbols(
+ hits: list[dict],
+ chosen: list,
+ verified_dict: Callable[[Any], Awaitable[dict]],
+) -> None:
+ """Boost (or insert) each chosen def's file and stash the def on it."""
by_path = {h.get("target_path"): h for h in hits}
top_score = max((h.get("score", 0.0) for h in hits), default=0.0)
- # Above the current top so an exact symbol match dominates the dominance
- # gate (an exact name+parent hit is stronger evidence than a prose match).
+ # Above the current top: an exact symbol match is stronger than prose.
anchor_score = max(top_score + 2.0, _HIGH_CONFIDENCE_SCORE_FLOOR + 1.0)
for sym in chosen:
fp = sym.file_path
if not fp:
- # The same guard the concept-anchoring twin applies to its winner.
- # An anchor scores above every real hit by construction, so a
- # pathless one takes rank 1 and serves a row carrying nothing but a
- # score — no path, title, summary or excerpt — while displacing a
- # real hit from the synthesis window.
+ # A pathless anchor would take rank 1 as an empty row.
continue
- target = by_path.get(fp)
- if target is None:
- target = {
- "page_id": f"file_page:{fp}",
- "target_path": fp,
- "title": fp,
- "summary": "",
- "snippet": "",
- "page_type": "file_page",
- "score": anchor_score,
- "_symbol_anchored": True,
- }
- hits.insert(0, target)
- by_path[fp] = target
- else:
- target["score"] = max(target.get("score", 0.0), anchor_score)
- target["_symbol_anchored"] = True
- # Stash the exact symbol the question named so symbol_bodies serves it
- # directly — the fuzzy hydration cap drops a far-down method when the
- # parent class name floods every sibling's qualified-name match. Serve
- # verified bounds only: an unrelocatable (approximate) symbol still
- # boosts its file's rank, but is not stashed for a live-body slice.
- vd = await _verified_dict(sym)
+ target = _boost_or_insert_file_hit(hits, by_path, fp, anchor_score)
+ target["_symbol_anchored"] = True
+ # Stash the named symbol so symbol_bodies serves it even when the
+ # hydration cap would drop it. Verified bounds only; an approximate
+ # symbol still boosts its file.
+ vd = await verified_dict(sym)
if vd.get("_approx"):
continue
target.setdefault("_anchor_symbols", []).append(
@@ -541,8 +341,32 @@ async def _anchor_symbol_hits(
"end_line": vd["end_line"],
}
)
- hits.sort(key=lambda h: h.get("score", 0.0), reverse=True)
- return hits, homonyms
+
+
+def _boost_or_insert_file_hit(
+ hits: list[dict], by_path: dict, path: str, score: float
+) -> dict:
+ """The hit for *path* raised to at least *score*, or a new one at the front.
+
+ Shared by the symbol and concept anchors, which differ only in the flags
+ they set on the returned hit.
+ """
+ target = by_path.get(path)
+ if target is None:
+ target = {
+ "page_id": f"file_page:{path}",
+ "target_path": path,
+ "title": path,
+ "summary": "",
+ "snippet": "",
+ "page_type": "file_page",
+ "score": score,
+ }
+ hits.insert(0, target)
+ by_path[path] = target
+ else:
+ target["score"] = max(target.get("score", 0.0), score)
+ return target
def attach_truncation_contract(
@@ -550,35 +374,15 @@ def attach_truncation_contract(
) -> None:
"""Mark an inlined body that was cut, and name what the cut withheld.
- Every place that inlines a symbol body owes the consumer the same three
- keys when the indexed body outruns what was served: ``truncated``, a
- ``continuation`` naming the exact range holding the remainder, and the
- ``withheld_symbols`` that range covers.
-
- Say WHAT was withheld, not just that something was. A bare truncated flag
- plus a get_symbol pointer was followed zero times across the agent runs
- measured, so the consumer needs the names in hand to decide whether it is
- missing anything it cares about, and to continue inside this tool rather
- than falling back to Read.
-
- Both callers need it for the same reason and one of them needs it more: the
- homonym union payload returns BEFORE synthesis, so it is served in no-LLM
- mode and never reaches any of the confidence gates. Held in one function
- because two copies of this contract drifting apart is a live risk: the
- truncation keys are read by the confidence cascade, and the two sites have
- co-changed ten times.
+ Every site that inlines a body owes the same keys when the indexed body
+ outruns what was served: ``truncated``, a ``continuation`` range, and the
+ ``withheld_symbols`` in it, so the consumer knows what it is missing. One
+ function, because the confidence cascade reads these keys and two copies
+ would drift.
- ``indexed_end`` is the end line the index recorded, ``end_served`` the last
- line actually inlined. A falsy ``indexed_end`` means the index recorded no
- end at all, which is never a cut: it is tested explicitly rather than left
- to ``indexed_end > end_served``, which would only agree with it while
- ``end_served`` stays non-negative.
-
- ``indexed_end`` is trusted to lie within the live file, which is
- ``check_symbol_bounds``'s job rather than this one's: it now clamps to
- ``len(lines)`` on every return, so a stored end that overshoots cannot reach
- here and flag a body served WHOLE as truncated (D8). Clamping again here
- would be a second owner and a second disk read.
+ A falsy ``indexed_end`` means the index recorded no end, which is never a
+ cut. ``indexed_end`` is trusted to lie within the live file:
+ ``check_symbol_bounds`` owns that clamp.
"""
if indexed_end and indexed_end > end_served:
entry["truncated"] = True
@@ -624,10 +428,8 @@ def build_homonym_union_bodies(
start = d.get("start_line") or 0
end = d.get("end_line") or 0
symbol_id = f"{path}::{name}"
- # Bounds that failed live verification (symbol moved and could not be
- # re-located): don't inline a slice at unreliable lines under a
- # confidence=high envelope. Hand the agent a get_symbol pointer, which
- # verifies on its own path.
+ # Unverified bounds: a get_symbol pointer instead of a slice at
+ # unreliable lines under a high-confidence envelope.
body = (
None
if d.get("_approx")
@@ -670,24 +472,14 @@ async def _concept_anchor_hits(
) -> list[dict]:
"""Anchor the file whose rationale COMMENT explains a number-bearing question.
- The symbol anchor above rescues questions that NAME an indexed symbol. This
- rescues the other retrieval-miss class: a why/value question that pins a
- literal number to a *described behaviour* (a cap / limit / batch size) but
- names no symbol. Fuzzy retrieval lands on a same-vocabulary file and never
- surfaces the one whose comment justifies the number, so it never enters the
- candidate set and the agent re-reads. We grep tracked source for comment
- lines carrying the number + a content noun, score the candidates with the
- existing rationale miner, and inject the winner so retrieval includes it and
- its comment reaches ``code_rationale``.
+ For a question that pins a literal number to a described behaviour (a cap,
+ a limit) but names no symbol. Greps comments carrying the number, scores
+ them with the rationale miner, and injects the winning file so its comment
+ reaches ``code_rationale``.
- Fires only when the question pins a literal number (the high-precision case;
- the prototype showed naive number-free grep is too noisy) and the winning
- file is not already the top retrieval hit (i.e. retrieval genuinely missed
- it). When the winner is already top, the existing confidence machinery decides
- the label - we deliberately do NOT force it past the dominance gate, which
- generalized only to the questions it was tuned on. The mined rationale + its
- line are stashed on the hit so the downstream ``code_rationale`` surfacing
- serves the exact comment without a second grep.
+ Fires only on number-bearing questions (number-free grep is too noisy) and
+ only when retrieval missed the winner; otherwise the confidence machinery
+ decides. The mined rationale is stashed on the hit to avoid a second grep.
Returns ``hits`` re-sorted by score (mutated in place). Best-effort: any
failure leaves ``hits`` untouched.
@@ -702,14 +494,11 @@ async def _concept_anchor_hits(
if repo_root is None or not question:
return hits
- # Precision gate: only number-bearing questions. A bare "why is X limited"
- # would grep the whole cap-family vocabulary and over-fire.
+ # Precision gate: only number-bearing questions.
if not _salient_numbers(question):
return hits
- # The grep spawns a subprocess and mine reads files off disk - both blocking.
- # Run them in a worker thread so they never stall the server's event loop
- # (this can run inside a stdio MCP server driving the JSON-RPC transport).
+ # Blocking grep and file reads: keep them off the server's event loop.
def _grep_and_mine() -> dict | None:
candidates = grep_comment_candidates(repo_root, question)
if not candidates:
@@ -723,963 +512,20 @@ async def _concept_anchor_hits(
winner_path = winner.get("path")
if not winner_path:
return hits
- # Retrieval-miss gate: only anchor when retrieval did NOT already lead with
- # the winner. If it is already the top hit, leave the confidence label to the
- # existing dominance/confidence machinery - forcing it past the gate only ever
- # helped the questions it was tuned against. The mined comment still reaches
- # the agent via the gated path's code_rationale.
+ # Retrieval-miss gate: if the winner already leads, leave it to the
+ # confidence machinery.
if hits and hits[0].get("target_path") == winner_path:
return hits
near_line = (winner.get("lines") or [0])[0]
by_path = {h.get("target_path"): h for h in hits}
top_score = max((h.get("score", 0.0) for h in hits), default=0.0)
- # Above the current top so the comment-justified file dominates the
- # dominance gate and synthesis runs instead of gating low.
+ # Above the current top, so synthesis runs instead of gating low.
anchor_score = max(top_score + 1.5, _HIGH_CONFIDENCE_SCORE_FLOOR + 0.5)
- target = by_path.get(winner_path)
- if target is None:
- target = {
- "page_id": f"file_page:{winner_path}",
- "target_path": winner_path,
- "title": winner_path,
- "summary": "",
- "snippet": "",
- "page_type": "file_page",
- "score": anchor_score,
- }
- hits.insert(0, target)
- by_path[winner_path] = target
- else:
- target["score"] = max(target.get("score", 0.0), anchor_score)
+ target = _boost_or_insert_file_hit(hits, by_path, winner_path, anchor_score)
target["_concept_anchored"] = True
target["_concept_near_line"] = near_line
- # Stash the mined comment so the code_rationale surfacing can serve it
- # verbatim on any exit path - including the high path, where the comment IS
- # the answer the agent would otherwise re-read for.
+ # Served verbatim as ``code_rationale`` on any exit path.
target["_concept_rationale"] = winner
hits.sort(key=lambda h: h.get("score", 0.0), reverse=True)
return hits
-
-
-# Definition kinds worth naming in `defines`, best first. A file is
-# characterised by what it declares, so a class outranks a bare function and
-# both outrank a variable. Kinds absent from this map are not emitted at all:
-# imports and re-exports would fill the budget with names that answer nothing.
-_DEFINE_KIND_RANK = {
- "class": 0,
- "interface": 1,
- "struct": 1,
- "enum": 1,
- "type": 2,
- "function": 3,
- "method": 4,
- "constant": 5,
-}
-
-
-async def _hydrate_candidate_defines(
- session,
- repo_id: str,
- hits: list[dict],
- question_ids: set[str] | None = None,
-) -> None:
- """Mutate *hits* in place: attach ``_defines`` to the candidate-pool files.
-
- ``candidates`` names the files retrieval ranked and, before this, said
- nothing about any of them. An agent handed ``django/shortcuts.py`` and
- nothing else has exactly one move available, which is to go and Grep it; the
- Layer B taxonomy judged 89% of post-answer searches to be that move. Naming
- the definitions a file contains turns "search this file" into "read this
- line", and often answers a where-is-it question outright.
-
- Deliberately cheap and deliberately shallow:
-
- * **One batched query**, on ``(repository_id, file_path)`` which the
- ``uq_wiki_symbol`` index already covers, over at most
- ``_DEFINES_MAX_FILES`` paths. No live file reads, no bounds verification.
- * **Names and start lines only.** No signature, no docstring, no body. Those
- already have homes (``retrieval[].key_symbols``, ``symbol_bodies``) and
- this block must not compete with them for the payload's byte budget.
- * **Line numbers are index-recorded, not verified.** Unlike ``get_symbol``,
- nothing here checks the stored bounds against the live file. They are a
- navigation hint; the serializer's field documentation says so.
-
- Ordering within a file: question-named symbols first (they are what the
- agent came for), then by declaration kind, then by position. Dunders and
- private names are dropped unless the question named them.
- """
- qids = {q.lower() for q in (question_ids or set())}
-
- paths: list[str] = []
- seen: set[str] = set()
- for h in hits:
- p = hit_file_path(h)
- if not p or p in seen:
- continue
- seen.add(p)
- paths.append(p)
- if len(paths) >= _DEFINES_MAX_FILES:
- break
- if not paths:
- return
-
- res = await session.execute(
- select(
- WikiSymbol.file_path,
- WikiSymbol.name,
- WikiSymbol.kind,
- WikiSymbol.start_line,
- ).where(
- WikiSymbol.repository_id == repo_id,
- WikiSymbol.file_path.in_(paths),
- )
- )
-
- by_file: dict[str, list[tuple[int, int, str, int]]] = {}
- for file_path, name, kind, start_line in res.all():
- rank = _DEFINE_KIND_RANK.get((kind or "").lower())
- if rank is None or not name:
- continue
- matched = name.lower() in qids
- if not matched and name.startswith("_"):
- continue
- by_file.setdefault(file_path, []).append(
- (0 if matched else 1, rank, name, start_line or 0)
- )
-
- for path, rows in by_file.items():
- rows.sort(key=lambda r: (r[0], r[1], r[3]))
- picked: list[tuple[str, int]] = []
- taken: set[str] = set()
- for _m, _r, name, start in rows:
- if name in taken:
- continue
- taken.add(name)
- picked.append((name, start))
- if len(picked) >= _DEFINES_PER_CANDIDATE:
- break
- by_file[path] = picked # type: ignore[assignment]
-
- for h in hits:
- p = hit_file_path(h)
- if p and by_file.get(p):
- h["_defines"] = by_file[p]
-
-
-async def _hydrate_symbols_for_hits(
- session,
- repo_id: str,
- hits: list[dict],
- ctx: Any = None,
- question_ids: set[str] | None = None,
- question: str = "",
-) -> None:
- """Mutate `hits` in place: attach `symbols` list to top-N file_page hits.
-
- Question-aware promotion: if ``question_ids`` contains identifiers that
- match symbols in the retrieved files, those symbols move to the top of
- their file's symbol list, carry a longer docstring, and get a source
- excerpt (``source_excerpt``). This is the difference between the LLM
- seeing ``class LocalOutlierFactor`` at the file top (and hedging on a
- question about ``_local_reachability_density``) vs. seeing the actual
- method body and answering it.
-
- Top hit gets ``_MAX_SYMBOLS_TOP_HIT`` slots; secondaries get the smaller
- ``_MAX_SYMBOLS_PER_HIT``. Symbols not matching a question id carry the
- short 120-char docstring; matched symbols carry 400 chars + source body.
-
- ``question`` decides which symbols fill those slots when the file holds more
- than fit, and earns the leading few a source body: a question phrased in
- prose names no identifier, so nothing matches and nothing would carry code.
- """
- question_ids = question_ids or set()
- # Case-folded copy for matching.
- qids_lower = {q.lower() for q in question_ids}
- # Once per call: the question's terms, stemmed to match identifier roots.
- term_stems = {_stem(t) for t in content_terms(question)}
-
- # Identify the top file_page hits in retrieval-rank order. `hits` is
- # already sorted by descending score upstream.
- enrich_paths: list[str] = []
- for h in hits:
- if (
- h.get("target_path")
- and h.get("page_type") == "file_page"
- and len(enrich_paths) < _ENRICH_TOP_N_HITS
- ):
- enrich_paths.append(h["target_path"])
- if not enrich_paths:
- return
-
- res = await session.execute(
- select(WikiSymbol)
- .where(
- WikiSymbol.repository_id == repo_id,
- WikiSymbol.file_path.in_(enrich_paths),
- )
- .order_by(WikiSymbol.file_path, WikiSymbol.start_line)
- )
- by_file: dict[str, list[dict]] = {}
- repo_root = Path(str(ctx.path)) if ctx and ctx.path else None
- session_factory = getattr(ctx, "session_factory", None)
- # One live read per hydrated file, shared by the bounds gate and the
- # signature/body slices. None when unreadable (missing/outside root).
- text_cache: dict[str, str | None] = {}
- for row in res.scalars().all():
- if row.file_path not in text_cache:
- text_cache[row.file_path] = _read_repo_text(repo_root, row.file_path)
- text = text_cache[row.file_path]
- # Trust contract (shared with get_symbol): verify the stored bounds
- # against the live file before slicing a signature or body out of it.
- # Drift (an edit above the def, or an update lag) otherwise turns into a
- # garbled signature / body served as if fresh. On a re-parse correction
- # the row is healed; when the symbol can't be re-located we fall back to
- # the stored signature and skip the live body — a stored-but-consistent
- # signature beats a live slice at the wrong lines.
- if text is not None:
- check = await verify_and_heal(session_factory, row, text)
- start_line, end_line, verified = check.start_line, check.end_line, check.verified
- else:
- start_line, end_line, verified = row.start_line, row.end_line, False
- # Constants/variables: the stored signature IS the verbatim assignment
- # line. The disk re-read below walks forward looking for a ":"-closed
- # def line and would join unrelated following lines for assignments.
- if row.kind in ("constant", "variable") or not verified:
- rich_sig = None
- else:
- rich_sig = _read_signature_from_source(
- repo_root, row.file_path, start_line, text=text
- )
- matched = _question_names_symbol(row, qids_lower)
- entry: dict[str, Any] = {
- "name": row.name,
- "kind": row.kind,
- "signature": rich_sig or row.signature,
- "docstring": row.docstring or "",
- "start_line": start_line,
- "end_line": end_line,
- "_matched": matched,
- "_verified": verified,
- }
- # Scored once here, not in the sort key, so a dense file pays for it per
- # symbol rather than per comparison.
- entry["_relevance"] = _symbol_relevance(entry, term_stems)
- if matched and verified:
- src = _read_symbol_source(
- repo_root, row.file_path, start_line, end_line, text=text
- )
- if src:
- entry["source_excerpt"] = src
- by_file.setdefault(row.file_path, []).append(entry)
-
- # Sort: matched symbols first, then by relevance to the question, then in
- # start_line order. Cap per file — top hit gets more slots than secondary
- # hits. This decides WHICH symbols are kept; the kept slice is put back into
- # reading order below, so consumers still see document order.
- for i, h in enumerate(hits):
- path = h.get("target_path")
- if path not in by_file:
- continue
- syms = by_file[path]
- syms.sort(key=lambda s: (not s["_matched"], -s["_relevance"], s["start_line"]))
- cap = _MAX_SYMBOLS_TOP_HIT if i == 0 else _MAX_SYMBOLS_PER_HIT
- # Force-include the exact symbol the question named (via anchoring) so a
- # class-name flood — where every sibling method "matches" through the
- # parent's qualified name — can't evict the method the user asked about
- # from the synthesis context. Without this the LLM never sees the body
- # and hedges, which is exactly the failure anchoring exists to prevent.
- anchor_names = {a.get("name") for a in (h.get("_anchor_symbols") or [])}
- kept: list[dict] = [s for s in syms if s["name"] in anchor_names][:cap]
- # Then the rest of the matched symbols, then unmatched, up to the cap.
- kept.extend(s for s in syms if s["_matched"] and s not in kept)
- kept = kept[:cap]
- for s in syms:
- if s in kept:
- continue
- if len(kept) >= cap:
- break
- kept.append(s)
- # A prose question names no identifier, so nothing is `_matched` and the
- # slate would carry signatures only. Give the leading few symbols the
- # question scored against a body, so the excerpts hold the code the
- # question is about. `kept` is still in priority order here.
- bodied = 0
- for s in kept:
- if bodied >= _RELEVANT_EXCERPT_MAX_SYMBOLS:
- break
- if s.get("source_excerpt") or not s["_relevance"] or not s["_verified"]:
- continue
- src = _read_symbol_source(
- repo_root,
- path,
- s["start_line"],
- s.get("end_line") or 0,
- text=text_cache.get(path),
- )
- if src:
- s["source_excerpt"] = src
- bodied += 1
- # Upgrade the top question-relevant symbols to the inline-body depth
- # BEFORE the reading-order sort, while `kept` is still in priority order
- # (anchors, then matched, then unmatched). The default 40-line excerpt
- # truncates a docstring-heavy definition before its answer-bearing logic,
- # so synthesis hedges on the exact symbol whose full 120-line body the
- # response inlines in symbol_bodies. Reading the leading few at the same
- # depth keeps the LLM's view and the served body consistent. Bounded so a
- # class-name flood can't balloon the prompt; the rest keep the excerpt.
- upgraded = 0
- for s in kept:
- if upgraded >= _SYNTH_FULL_BODY_MAX_SYMBOLS:
- break
- if not s.get("_matched") or not s.get("source_excerpt"):
- continue
- fuller = _read_symbol_source(
- repo_root,
- path,
- s["start_line"],
- s.get("end_line") or 0,
- max_lines=_SYNTH_FULL_SOURCE_LINES,
- text=text_cache.get(path),
- )
- if fuller:
- s["source_excerpt"] = fuller
- upgraded += 1
- # Sort final slice by start_line for natural reading order.
- kept.sort(key=lambda s: s["start_line"])
- h["symbols"] = kept
-
-
-# ---------------------------------------------------------------------------
-# What a truncated body withheld
-# ---------------------------------------------------------------------------
-
-# Leading keywords that decorate a declaration without being one. Shared by the
-# keyword and brace-method shapes below.
-_DECL_MODIFIERS = (
- r"pub|public|private|protected|internal|open|final|static|abstract|override|"
- r"virtual|sealed|export|default|async|suspend|inline|readonly|declare|"
- r"unsafe|extern|partial|data|operator|const"
-)
-
-# Definition-line shapes, tried in order; each yields ``indent`` / ``kind`` /
-# ``name``. A Python-only regex was the first cut and it made this whole feature
-# inert on TS/Go/Java — which, at a 120-line body cap, is exactly where
-# truncation bites hardest, since a TS class or a React component is the shape
-# that overruns the cap in the first place.
-_WITHHELD_DEF_PATTERNS: tuple[re.Pattern[str], ...] = (
- # Python: def / async def / class.
- re.compile(
- r"^(?P<indent>[ \t]*)(?:async[ \t]+)?(?P<kind>def|class)[ \t]+"
- r"(?P<name>[A-Za-z_]\w*)"
- ),
- # Go methods, whose name follows the receiver: ``func (s *Store) Write(``.
- re.compile(
- r"^(?P<indent>[ \t]*)(?P<kind>func)[ \t]+\([^)]*\)[ \t]*"
- r"(?P<name>[A-Za-z_]\w*)"
- ),
- # Declaration keyword + name: Go func/type, Rust fn/struct/trait/impl/enum,
- # Java/C#/Kotlin class/interface/record, TS class/interface/type/enum.
- re.compile(
- rf"^(?P<indent>[ \t]*)(?:(?:{_DECL_MODIFIERS})[ \t]+)*"
- r"(?P<kind>func|fn|class|struct|interface|enum|trait|impl|record|type)"
- r"[ \t]+(?P<name>[A-Za-z_$][\w$]*)"
- ),
- # JS/TS function declarations, including ``export default`` and generators.
- # The separator after ``function`` is REQUIRED (or a generator star). With
- # ``[ \t]*`` the keyword matched as a mere prefix of a longer identifier, so
- # ``function_name: Mapped[str] = ...`` reported a symbol called ``_name``:
- # measured at 507 fabricated entries across this repo, the single largest
- # source of ids that resolve to nothing.
- re.compile(
- r"^(?P<indent>[ \t]*)(?:export[ \t]+)?(?:default[ \t]+)?(?:async[ \t]+)?"
- r"(?P<kind>function)(?:[ \t]*\*[ \t]*|[ \t]+)(?P<name>[A-Za-z_$][\w$]*)"
- ),
- # JS/TS arrow bindings: ``export const Panel = (props) => {``. The arrow has
- # to belong to the binding itself, with only an optional return annotation
- # between: allowing slack before it turns every local initialised from a
- # callback-taking call (``const pages = all.filter((p) => ...)``) into a
- # "definition" with a symbol_id that resolves to nothing.
- re.compile(
- r"^(?P<indent>[ \t]*)(?:export[ \t]+)?(?P<kind>const|let|var)[ \t]+"
- r"(?P<name>[A-Za-z_$][\w$]*)[^=]*=[ \t]*(?:async[ \t]+)?"
- r"(?:\([^)]*\)(?:[ \t]*:[^=]*?)?|[A-Za-z_$][\w$]*)[ \t]*=>"
- ),
- # Brace-language members: `` public void run(String a) {``, `` render() {``.
- # Between the parameter list and the brace only a return type or a throws
- # clause may appear -- no parens, or an assertion call in a test file
- # (``expect(x).toMatchObject({``) reads as a method declaration. The brace
- # itself is optional so Allman style (``void Beta()`` with ``{`` on the next
- # line, the C# default) is reachable; the caller supplies the next line.
- re.compile(
- rf"^(?P<indent>[ \t]*)(?:(?:{_DECL_MODIFIERS})[ \t]+)*"
- r"(?:[A-Za-z_$][\w$<>,.\[\]]*[ \t]+)?(?P<name>[A-Za-z_$][\w$]*)[ \t]*"
- r"\([^;{]*\)(?P<tail>[^;{()]*)(?P<brace>\{)?[ \t]*$"
- ),
-)
-_BRACE_MEMBER = len(_WITHHELD_DEF_PATTERNS) - 1
-
-# The brace-member shape above also matches control flow (``if (x) {``), a
-# statement whose keyword the optional type group swallows (``raise
-# ValueError(f"... {x}")`` reports ``ValueError``), and any call taking a
-# callback (``describe("x", () => {``, ``it("x", function () {``). Both the
-# matched NAME and the line's own first word are checked, because the type group
-# hides the keyword from a name-only guard.
-_NOT_A_DEFINITION = frozenset(
- {
- "if", "for", "while", "switch", "catch", "else", "do", "try", "return",
- "with", "using", "lock", "foreach", "case", "synchronized", "await",
- "yield", "new", "typeof", "in", "of", "when", "unless", "match",
- "raise", "throw", "assert", "del", "delete", "print", "elif", "except",
- "finally", "import", "from", "global", "nonlocal", "pass", "break",
- "continue", "go", "defer", "select", "range", "constructor",
- }
-)
-
-_FIRST_WORD_RE = re.compile(r"[A-Za-z_$][\w$]*")
-
-# Words no language lets you NAME a definition, so a match producing one is a
-# parse accident whatever shape it came from: ``fn is not None`` reads as Rust's
-# ``fn <name>`` and reported a symbol called ``is``. Kept strictly to reserved
-# words -- ``match``, ``range`` and ``print`` are all real function names.
-_RESERVED_NAMES = frozenset(
- {
- "is", "not", "and", "or", "in", "if", "else", "elif", "for", "while",
- "return", "none", "true", "false", "null", "undefined", "class", "def",
- "import", "from", "as", "with", "pass", "lambda", "del", "global",
- "raise", "try", "except", "finally", "yield", "await", "assert",
- "break", "continue", "nonlocal", "var", "let", "const", "function",
- }
-)
-
-
-def _match_definition(raw: str, next_raw: str = "") -> re.Match[str] | None:
- """First definition shape *raw* matches, or None.
-
- ``next_raw`` is the following source line, consulted only for Allman-style
- braces where the declaration and its ``{`` sit on separate lines.
- """
- for i, pattern in enumerate(_WITHHELD_DEF_PATTERNS):
- m = pattern.match(raw)
- if not m:
- continue
- if m.group("name").lower() in _RESERVED_NAMES:
- continue
- if i == _BRACE_MEMBER:
- head = _FIRST_WORD_RE.match(raw.strip())
- if m.group("name").lower() in _NOT_A_DEFINITION:
- continue
- if head and head.group(0).lower() in _NOT_A_DEFINITION:
- continue
- # An anonymous function passed as an argument, in either syntax.
- if "=>" in raw[: m.end()] or "function" in raw[: m.end()]:
- continue
- # Go's third spelling of the same thing: ``func(req *http.Request)
- # (*http.Response, error) {``. There is no space after ``func``, so
- # the optional return-type group matches empty and the name group
- # takes the keyword itself, yielding an unresolvable ``path::func``.
- # This cannot be handled by either general-purpose set above:
- # ``_RESERVED_NAMES`` is tested for every pattern and ``def func():``
- # is a real Python definition (41 of them in django alone), while
- # ``_NOT_A_DEFINITION`` is also tested against the line's FIRST
- # word, which is ``func`` on every named Go function too.
- #
- # Requiring ``func`` to open the line is what keeps it to the Go
- # literal: ``int func(int a) {`` is a real definition named ``func``
- # in C, C++, Java, C# and Kotlin, and a name-only test suppresses
- # all five.
- if m.group("name") == "func" and head and head.group(0) == "func":
- continue
- # Allman: the brace is on the next line. A declaration never ends in
- # a comma, but an argument on its own line inside a multi-line call
- # does -- and when the following argument is a dict literal, the
- # next line really is ``{`` (``bool(matched_nums),`` then ``{``).
- if not m.group("brace") and (
- next_raw.strip() != "{" or raw.rstrip().endswith(",")
- ):
- continue
- return m
- return None
-
-
-# Anything that could open a string or a comment. A line with none of these
-# cannot change the scanner's state, so it skips the character walk.
-_QUOTEISH_RE = re.compile(r"""["'`#]|/[*/]""")
-
-
-def _skip_quoted(raw: str, i: int) -> int:
- """Index just past the single- or double-quoted run starting at ``i``."""
- quote, i = raw[i], i + 1
- while i < len(raw):
- if raw[i] == "\\":
- i += 2
- continue
- if raw[i] == quote:
- return i + 1
- i += 1
- return i
-
-
-# Extensions where a backtick actually opens a string: Go raw strings and the
-# JS/TS template-literal family. Everywhere else a backtick is punctuation --
-# Rust doc comments, Ruby heredocs and Python docstrings all carry markdown
-# fences and shell quotes -- so masking on it can only ever be a misfire.
-#
-# Not a style choice, a measured one. Before this gate, a markdown fence inside
-# an ordinary Rust string opened a phantom frame that a stray backtick 260 lines
-# later re-closed, and `goose/crates/goose-cli/src/session/export.rs` lost FIVE
-# real `pub fn` definitions from `withheld_symbols` with the containment below
-# provably inert. Three more files in the corpus did the same (two Ruby
-# heredocs, one Rust raw string).
-_BACKTICK_STRING_SUFFIXES = frozenset(
- {".go", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts"}
-)
-
-# Characters after which a ``/`` starts a REGEX rather than a division. Notably
-# excludes ``)``, ``]`` and anything alphanumeric, which are the positions where
-# a division's left operand ends -- so Go's and Python's ``a / b`` is never
-# mistaken for a regex.
-_REGEX_CAN_START_AFTER = frozenset("(,=:[!&|?{};+-*%~^<>\n")
-
-# ...and the keywords a regex can follow, which end in an alphanumeric and so
-# would otherwise read as a division's left operand. `return /[`]/.test(x)` is
-# the one that matters here: it opens a phantom frame exactly like the
-# character-class case `_skip_regex` exists for.
-_REGEX_CAN_START_AFTER_WORD = frozenset(
- {
- "return", "case", "typeof", "yield", "await", "throw", "in", "of",
- "new", "delete", "instanceof", "do", "else", "void",
- }
-)
-
-
-def _regex_position(raw: str, i: int, prev: str) -> bool:
- """Whether the ``/`` at ``i`` starts a regex rather than a division."""
- if prev in _REGEX_CAN_START_AFTER:
- return True
- if not (prev.isalnum() or prev == "_"):
- return False
- word = ""
- j = i - 1
- while j >= 0 and raw[j].isspace():
- j -= 1
- while j >= 0 and (raw[j].isalnum() or raw[j] == "_"):
- word = raw[j] + word
- j -= 1
- return word in _REGEX_CAN_START_AFTER_WORD
-
-
-def _skip_regex(raw: str, i: int) -> int:
- """Index past a regex literal starting at ``i``, or ``i`` if it is not one.
-
- Only exists because a backtick inside a regex otherwise opens a phantom
- template literal. When a later backtick closes that phantom again the walk
- ends with an EMPTY stack, so the containment in ``_string_masked_lines``
- never fires and the mask silently eats everything between.
-
- Isolated on one real file: ``mui/packages/markdown/parseMarkdown.js:203``,
- ``return matches[1].replace(/`/g, '');``. Without this branch that file
- masks 61 lines instead of 52, stays balanced, and hides the real
- ``function getDescription`` at line 206.
-
- Regex literals cannot span lines, so a run with no closing ``/`` on the same
- line is not one and is left alone.
- """
- j = i + 1
- in_class = False
- while j < len(raw):
- c = raw[j]
- if c == "\\":
- j += 2
- continue
- if c == "[":
- in_class = True
- elif c == "]":
- in_class = False
- elif c == "/" and not in_class:
- return j + 1
- j += 1
- return i
-
-
-def _walk_string_state(
- lines: tuple[str, ...], *, backticks: bool
-) -> tuple[set[int], set[int], bool]:
- """(lines starting inside a string, inside a block comment, literal left open).
-
- The two sets are kept apart because callers need to tell them apart and this
- is a hot path: a string body proves the enclosing expression is still open,
- a comment between two declarations proves nothing.
-
- ``stack`` models template-literal nesting: a ``None`` frame is the string
- part of a backtick literal, an ``int`` frame is the unclosed-brace depth
- inside a ``${...}`` interpolation. An interpolation holds CODE, so its lines
- are not masked and a backtick inside one opens its own nested literal rather
- than closing the outer one.
-
- That nesting is not optional sophistication, and the reason is not the one
- it looks like. Nesting alone does NOT unbalance a flat open/close counter --
- four synthetic nested fixtures all re-balance. What breaks is the
- combination: a flat counter reads the nested literal's OPENING backtick as
- closing the outer one, which puts the walk at code level inside what is
- really string content, and a code-level rule then eats the rest of the line
- along with the real closing backtick. On mui's
- ``DisabledDefaultClasses.tsx`` that rule is ``#`` firing on the CSS colour
- ``#fff``, and the outer literal then never closes.
-
- The same shape has two other triggers, and those two are worse because they
- leave the stack BALANCED, which the containment in ``_string_masked_lines``
- cannot see: an escaped backtick (handled here) and a backtick inside a regex
- (handled by ``_skip_regex``). The nesting case usually ends UNBALANCED and
- so is caught by the fallback anyway -- interpolation tracking is here for
- precision, measured as 1,926 fabrications against 60 on the 16 corpus files
- where a flat walk and this one disagree.
- """
- strings: set[int] = set()
- comments: set[int] = set()
- delim: str | None = None
- in_block = False
- # Line the currently-open ``delim`` run or ``/* */`` block started on. The
- # two are mutually exclusive (a frame is only ever opened at code level), so
- # one variable serves both.
- open_line = 0
- stack: list[int | None] = []
- for n, raw in enumerate(lines, 1):
- # The three states are mutually exclusive: a frame is only ever pushed
- # at code level, so an if/elif chain is faithful to the walk below.
- if delim is not None or (stack and stack[-1] is None):
- strings.add(n)
- elif in_block:
- comments.add(n)
- elif not stack and not _QUOTEISH_RE.search(raw):
- # Nothing on this line can open a string or comment, so the
- # character walk below cannot change state. Most lines are this
- # line, and skipping them is what keeps a whole-file scan cheap.
- continue
- i = 0
- # Last non-space character seen at CODE level, for the regex test below.
- prev = "\n"
- while i < len(raw):
- if stack:
- if stack[-1] is None:
- if raw[i] == "\\":
- # An escaped backtick does not close the literal. Without
- # this, `` `x\`y` `` closes early and re-opens on the real
- # terminator, inverting the parity for the rest of the
- # file with a balanced stack the containment cannot see.
- i += 2
- elif raw.startswith("${", i):
- stack.append(1)
- i += 2
- elif raw[i] == "`":
- stack.pop()
- i += 1
- else:
- i += 1
- continue
- # Inside ${...}, so the ordinary code tokens apply again --
- # including the regex probe, without which a backtick in a
- # character class here (``${s.replace(/[`]/g, "")}``) opens the
- # same phantom frame _skip_regex exists to prevent.
- if raw[i] == "{":
- stack[-1] += 1
- elif raw[i] == "}":
- stack[-1] -= 1
- if stack[-1] <= 0:
- stack.pop()
- elif raw[i] == "`":
- stack.append(None)
- elif raw.startswith("//", i):
- break
- elif raw[i] == "/" and _regex_position(raw, i, prev):
- j = _skip_regex(raw, i)
- if j > i:
- i, prev = j, "/"
- continue
- elif raw[i] in ('"', "'"):
- i = _skip_quoted(raw, i)
- prev = '"'
- continue
- if not raw[i].isspace():
- prev = raw[i]
- i += 1
- continue
- if delim is not None:
- if raw.startswith(delim, i):
- delim, open_line, i = None, 0, i + len(delim)
- else:
- i += 1
- continue
- if in_block:
- if raw.startswith("*/", i):
- in_block, open_line, i = False, 0, i + 2
- else:
- i += 1
- continue
- if raw.startswith('"""', i) or raw.startswith("'''", i):
- delim, open_line, i = raw[i : i + 3], n, i + 3
- continue
- if backticks and raw[i] == "`":
- stack.append(None)
- i += 1
- continue
- if raw.startswith("/*", i):
- in_block, open_line, i = True, n, i + 2
- continue
- if raw[i] == "#" or raw.startswith("//", i):
- break
- if raw[i] == "/" and _regex_position(raw, i, prev):
- j = _skip_regex(raw, i)
- if j > i:
- i, prev = j, "/"
- continue
- if raw[i] in ('"', "'"):
- i = _skip_quoted(raw, i)
- prev = '"'
- continue
- if not raw[i].isspace():
- prev = raw[i]
- i += 1
- # A run still open at EOF is a walk that lost track, not a file with an
- # unterminated construct, and masking to EOF hides every definition below
- # it. The template-literal stack has had this containment since the backtick
- # work; ``delim`` and ``/* */`` never did, and both fire on the same shape --
- # a delimiter belonging to another language, sitting inside a string this
- # walk cannot see. Measured on Rust: ``${0%/*}`` in a raw string masked 103
- # lines and cost 6 real ``fn``; ``description = """#`` masked 3,002 and cost
- # 10. Discarding the trailing run under-masks instead, which costs a
- # spurious name in a list rather than an absent real one.
- if delim is not None:
- strings = {n for n in strings if n < open_line}
- elif in_block:
- comments = {n for n in comments if n < open_line}
- return strings, comments, bool(stack)
-
-
-def _has_backtick_strings(file_path: str) -> bool:
- """Whether a backtick opens a string in this file's language."""
- dot = file_path.rfind(".")
- return dot != -1 and file_path[dot:].lower() in _BACKTICK_STRING_SUFFIXES
-
-
-class _Masked(NamedTuple):
- """1-based line numbers, split by what is hiding them.
-
- ``all`` is precomputed rather than unioned per call: every caller wants it,
- and this is a cached whole-file walk.
- """
-
- strings: frozenset[int]
- comments: frozenset[int]
- all: frozenset[int]
-
-
-@lru_cache(maxsize=8)
-def _string_masked_lines(lines: tuple[str, ...], backticks: bool = True) -> _Masked:
- """1-based line numbers that START inside a multi-line string or comment.
-
- Repowise's own docstrings are full of indented ``def``/``class`` examples,
- and without this every one of them becomes a ``symbol_id`` the note tells the
- agent to fetch and that resolves to nothing.
-
- Deliberately a lexer-lite: it tracks Python triple quotes, backtick template
- literals / Go raw strings, and C-style ``/* */`` blocks, and stops at ``#`` /
- ``//``. Known ceilings, all measured rather than assumed:
-
- * C# verbatim strings (``@"..."``) and Rust raw strings (``r#"..."#``) are
- not tracked. C# because the exposure is 27 multi-line literals in 4,284
- files; Rust because its 35,484 interior lines yielded 0 fabrications --
- Go raw strings hold GraphQL, which matches the definition patterns, while
- Rust raw strings hold TOML, which does not.
- * Inside a ``${...}`` interpolation, ``/* */`` and ``#`` are not handled, so
- a ``}`` inside a block comment there can close the frame early. Every
- constructed case ended unbalanced and was caught by the fallback below.
- * Markdown fenced blocks and inline code spans are now masked too, which is
- wanted (a ``def`` inside a ```` ``` ```` fence is not a definition) but is
- a behaviour change worth knowing about.
-
- Cached on the line tuple: this is a per-character Python loop over the whole
- file (68 ms on a 424 KB one), and the homonym-union path calls it once per
- truncated body, re-masking the same file each time. The fallback below can
- double that on a file whose literals do not balance, because the fast-path
- line skip is disabled while a frame is open -- measured at 8x on a
- synthetic 1.2 MB file with one stray backtick on line 1. Bounded at two
- walks, and the cache means it is paid once per file.
- """
- strings, comments, template_left_open = _walk_string_state(
- lines, backticks=backticks
- )
- if template_left_open:
- # The lexer-lite lost track: a template literal opened and never closed,
- # so every line below it is masked to EOF and every definition there is
- # silently suppressed. That failure is invisible -- no error, just
- # missing symbols -- so it must not be the one we ship. Fall back to the
- # pre-backtick walk for this file, which under-masks instead: the cost is
- # a spurious name in a list, not an absent real one.
- strings, comments, _ = _walk_string_state(lines, backticks=False)
- return _Masked(
- frozenset(strings), frozenset(comments), frozenset(strings | comments)
- )
-
-
-def _indent_width(raw: str) -> int:
- return len(raw) - len(raw.lstrip())
-
-
-# Stand-in indent for "the cut is inside something still open", used when every
-# withheld line is a string body or a bracket tail and none carries a real one.
-_UNBOUNDED_INDENT = 1 << 30
-
-
-# Cap on how many withheld definitions are surfaced. This block exists to let
-# the agent CONTINUE inside the tool rather than fall back to Read, so it has to
-# stay small enough that it never competes with the answer for the window: names
-# and signatures only, never bodies.
-_WITHHELD_MAX_SYMBOLS: int = 8
-
-
-def withheld_definitions(
- repo_root: Path | None, continuation: str | None
-) -> list[dict]:
- """Definitions that live in the range a truncated body did NOT serve.
-
- ``continuation`` is the ``path:first-last`` pointer already attached to a
- truncated ``symbol_bodies`` entry, so this reads exactly the lines the
- payload admits it withheld.
-
- Why this exists rather than just flagging the truncation: measured on the
- transcripts on disk, when a body is truncated the withheld range contains a
- symbol the answer goes on to talk about **78% of the time**, and the
- responses are at ``confidence: high`` in most of those. A flag alone does
- not help the consumer, and the ``get_symbol`` pointer the payload already
- carries was followed ZERO times across the runs measured. Names and
- signatures are cheap and keep the agent inside the tool.
-
- Returns ``[{name, kind, line, symbol_id, signature}]``, the boundary-cut
- symbol first, empty on any failure (a probe that cannot read must not
- manufacture doubt).
- """
- if not continuation:
- return []
- path, _, span = continuation.rpartition(":")
- first, _, last = span.partition("-")
- if not path or not first.isdigit() or not last.isdigit():
- return []
- lo, hi = int(first), int(last)
- text = _read_repo_text(repo_root, path)
- if text is None:
- return []
- lines = text.splitlines()
- if lo < 1 or lo > len(lines):
- return []
- mask = _string_masked_lines(tuple(lines), _has_backtick_strings(path))
- masked = mask.all
-
- def _entry(line_no: int, m: re.Match[str], *, cut: bool = False) -> dict:
- name = m.group("name")
- # The brace-member shape has no keyword to report, so it is named for
- # what it is rather than mislabelled as a Python `def`.
- kind = m.groupdict().get("kind") or "member"
- sig = _read_signature_from_source(repo_root, path, line_no, text=text)
- e = {
- "name": name,
- "kind": kind,
- "line": line_no,
- "symbol_id": f"{path}::{name}",
- "signature": (sig or f"{kind} {name}").strip(),
- }
- if cut:
- e["body_continues"] = True
- return e
-
- out: list[dict] = []
-
- # The symbol whose body is CUT BY the boundary, which is the case that
- # motivated this whole helper and the one a naive implementation misses.
- # In the reference defect the served range ended at 166 and `_validate`
- # starts at 164: its `def` line was served, so it does not appear anywhere
- # in the withheld range, while the line that actually causes the bug (176)
- # sits inside it. Reporting only defs that START after the cut would say
- # nothing about the symbol the answer is about.
- #
- # Taking the nearest preceding definition unconditionally is wrong, though:
- # a symbol that ENDED before the cut was served whole, and reporting it as
- # continuing puts a fully-served name at the head of the note and into the
- # get_symbol pointer. A definition at indent I reaches line ``lo`` only if
- # every non-blank line from it up to the first non-blank withheld line is
- # indented deeper than I, so walking backwards while tracking the running
- # minimum indent decides it exactly, in one pass and with no re-scan.
- #
- # The anchor obeys the same two exclusions as the walk. Taking the first
- # non-blank withheld line flatly is what put the walk one line short of
- # reality: when the cut lands ON a multi-line signature's own ``) -> dict:``
- # (or on a flush-left docstring line), the anchor reads as column 0, the
- # walk dies at once, and the payload ships truncated with NO withheld
- # symbols -- gate 8 inert on exactly the long entry points that truncate.
- _end = min(hi, len(lines))
- _usable = [
- n
- for n in range(lo, _end + 1)
- if lines[n - 1].strip()
- and n not in masked
- and lines[n - 1].strip()[0] not in ")]}{"
- ]
- if lo in mask.strings:
- # The cut is INSIDE a multi-line string, so the expression holding that
- # string -- and everything enclosing it -- is still open at ``lo``.
- # Without this the anchor reads from the first line BELOW the string,
- # usually a top-level declaration at column 0, and the walk dies at once
- # (D9: 8 real definitions lost across cli/cli and mui). A block COMMENT
- # cannot stand in for this: one sitting between two methods would report
- # the preceding method as continuing when it has already ended.
- anchor = _UNBOUNDED_INDENT
- elif _usable:
- anchor = _indent_width(lines[_usable[0] - 1])
- elif any(lines[n - 1].strip() for n in range(lo, _end + 1)):
- # Every withheld line is a string body or a bracket tail, so whatever
- # encloses the cut is certainly still open: let any preceding
- # definition qualify.
- anchor = _UNBOUNDED_INDENT
- else:
- anchor = None
- if anchor is not None:
- min_indent = anchor
- for back in range(lo - 1, 0, -1):
- if min_indent <= 0:
- break # nothing can be shallower, so nothing can still be open
- raw = lines[back - 1]
- stripped = raw.strip()
- if not stripped:
- continue
- # A line opening with a closing bracket is the tail of a multi-line
- # construct, not a statement at its own indent. Folding it is what
- # made this miss the live reference case: ``get_answer``'s signature
- # spans lines and ends ``) -> dict:`` at column 0, so the running
- # minimum hit zero on the signature's own closing paren and the walk
- # gave up two lines short of the ``async def`` it was looking for.
- # An Allman brace (``{`` alone) is likewise part of the declaration
- # above it, not a statement.
- if stripped[0] in ")]}{":
- continue
- ind = _indent_width(raw)
- nxt = lines[back] if back < len(lines) else ""
- m = None if back in masked else _match_definition(raw, nxt)
- if m is not None and ind < min_indent:
- out.append(_entry(back, m, cut=True))
- break
- min_indent = min(min_indent, ind)
-
- seen = {d["name"] for d in out}
- for offset, raw in enumerate(lines[lo - 1 : _end]):
- line_no = lo + offset
- if line_no in masked:
- continue
- nxt = lines[line_no] if line_no < len(lines) else ""
- m = _match_definition(raw, nxt)
- if m is None or m.group("name") in seen:
- continue
- seen.add(m.group("name"))
- out.append(_entry(line_no, m))
- if len(out) >= _WITHHELD_MAX_SYMBOLS:
- break
- return out
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/withheld.py b/packages/server/src/repowise/server/mcp_server/tool_answer/withheld.py
new file mode 100644
index 000000000..f61ed31a3
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/withheld.py
@@ -0,0 +1,312 @@
+"""Name the definitions a truncated symbol body did not serve.
+
+Definition-line shapes across languages, and the scan over a withheld range
+that reports them while skipping lines the string mask says are not code.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator
+from pathlib import Path
+
+from repowise.server.mcp_server.tool_answer.source import (
+ _read_repo_text,
+ _read_signature_from_source,
+)
+from repowise.server.mcp_server.tool_answer.string_mask import (
+ _has_backtick_strings,
+ _Masked,
+ _string_masked_lines,
+)
+
+# Leading keywords that decorate a declaration without being one. Shared by the
+# keyword and brace-method shapes below.
+_DECL_MODIFIERS = (
+ r"pub|public|private|protected|internal|open|final|static|abstract|override|"
+ r"virtual|sealed|export|default|async|suspend|inline|readonly|declare|"
+ r"unsafe|extern|partial|data|operator|const"
+)
+
+# Definition-line shapes across languages, tried in order; each yields
+# ``indent`` / ``kind`` / ``name``.
+_WITHHELD_DEF_PATTERNS: tuple[re.Pattern[str], ...] = (
+ # Python: def / async def / class.
+ re.compile(
+ r"^(?P<indent>[ \t]*)(?:async[ \t]+)?(?P<kind>def|class)[ \t]+"
+ r"(?P<name>[A-Za-z_]\w*)"
+ ),
+ # Go methods, whose name follows the receiver: ``func (s *Store) Write(``.
+ re.compile(
+ r"^(?P<indent>[ \t]*)(?P<kind>func)[ \t]+\([^)]*\)[ \t]*"
+ r"(?P<name>[A-Za-z_]\w*)"
+ ),
+ # Declaration keyword + name: Go func/type, Rust fn/struct/trait/impl/enum,
+ # Java/C#/Kotlin class/interface/record, TS class/interface/type/enum.
+ re.compile(
+ rf"^(?P<indent>[ \t]*)(?:(?:{_DECL_MODIFIERS})[ \t]+)*"
+ r"(?P<kind>func|fn|class|struct|interface|enum|trait|impl|record|type)"
+ r"[ \t]+(?P<name>[A-Za-z_$][\w$]*)"
+ ),
+ # JS/TS function declarations, including ``export default`` and generators.
+ # The separator after ``function`` is required, or ``function_name: ...``
+ # would report a symbol called ``_name``.
+ re.compile(
+ r"^(?P<indent>[ \t]*)(?:export[ \t]+)?(?:default[ \t]+)?(?:async[ \t]+)?"
+ r"(?P<kind>function)(?:[ \t]*\*[ \t]*|[ \t]+)(?P<name>[A-Za-z_$][\w$]*)"
+ ),
+ # JS/TS arrow bindings: ``export const Panel = (props) => {``. The arrow must
+ # belong to the binding, or ``const pages = all.filter((p) => ...)`` matches.
+ re.compile(
+ r"^(?P<indent>[ \t]*)(?:export[ \t]+)?(?P<kind>const|let|var)[ \t]+"
+ r"(?P<name>[A-Za-z_$][\w$]*)[^=]*=[ \t]*(?:async[ \t]+)?"
+ r"(?:\([^)]*\)(?:[ \t]*:[^=]*?)?|[A-Za-z_$][\w$]*)[ \t]*=>"
+ ),
+ # Brace-language members: `` public void run(String a) {``, `` render() {``.
+ # No parens after the parameter list, or ``expect(x).toMatchObject({``
+ # matches. The brace is optional for Allman style; the caller supplies the
+ # next line.
+ re.compile(
+ rf"^(?P<indent>[ \t]*)(?:(?:{_DECL_MODIFIERS})[ \t]+)*"
+ r"(?:[A-Za-z_$][\w$<>,.\[\]]*[ \t]+)?(?P<name>[A-Za-z_$][\w$]*)[ \t]*"
+ r"\([^;{]*\)(?P<tail>[^;{()]*)(?P<brace>\{)?[ \t]*$"
+ ),
+)
+_BRACE_MEMBER = len(_WITHHELD_DEF_PATTERNS) - 1
+
+# Words that make a brace-member match control flow or a statement rather than
+# a definition. Checked against the matched name and the line's first word,
+# since the optional type group can swallow the keyword (``raise X(...{``).
+_NOT_A_DEFINITION = frozenset(
+ {
+ "if", "for", "while", "switch", "catch", "else", "do", "try", "return",
+ "with", "using", "lock", "foreach", "case", "synchronized", "await",
+ "yield", "new", "typeof", "in", "of", "when", "unless", "match",
+ "raise", "throw", "assert", "del", "delete", "print", "elif", "except",
+ "finally", "import", "from", "global", "nonlocal", "pass", "break",
+ "continue", "go", "defer", "select", "range", "constructor",
+ }
+)
+
+_FIRST_WORD_RE = re.compile(r"[A-Za-z_$][\w$]*")
+
+# Words no language lets you name a definition (``fn is not None`` is not Rust).
+# Strictly reserved words: ``match``, ``range`` and ``print`` are real names.
+_RESERVED_NAMES = frozenset(
+ {
+ "is", "not", "and", "or", "in", "if", "else", "elif", "for", "while",
+ "return", "none", "true", "false", "null", "undefined", "class", "def",
+ "import", "from", "as", "with", "pass", "lambda", "del", "global",
+ "raise", "try", "except", "finally", "yield", "await", "assert",
+ "break", "continue", "nonlocal", "var", "let", "const", "function",
+ }
+)
+
+
+def _match_definition(raw: str, next_raw: str = "") -> re.Match[str] | None:
+ """First definition shape *raw* matches, or None.
+
+ ``next_raw`` is the following source line, consulted only for Allman-style
+ braces where the declaration and its ``{`` sit on separate lines.
+ """
+ for i, pattern in enumerate(_WITHHELD_DEF_PATTERNS):
+ m = pattern.match(raw)
+ if not m or m.group("name").lower() in _RESERVED_NAMES:
+ continue
+ if i == _BRACE_MEMBER and _not_a_brace_member(raw, next_raw, m):
+ continue
+ return m
+ return None
+
+
+def _not_a_brace_member(raw: str, next_raw: str, m: re.Match[str]) -> bool:
+ """Whether a brace-member match is a statement or call that only looks like one."""
+ head = _FIRST_WORD_RE.match(raw.strip())
+ if m.group("name").lower() in _NOT_A_DEFINITION:
+ return True
+ if head and head.group(0).lower() in _NOT_A_DEFINITION:
+ return True
+ # An anonymous function passed as an argument, in either syntax.
+ if "=>" in raw[: m.end()] or "function" in raw[: m.end()]:
+ return True
+ # A Go function literal, ``func(req *http.Request) (...) {``. Neither set
+ # above fits: ``func`` is a real name elsewhere (``int func(int a) {``), so
+ # only a line that opens with it is the Go literal.
+ if m.group("name") == "func" and head and head.group(0) == "func":
+ return True
+ # Allman: the brace is on the next line. A trailing comma means a call
+ # argument followed by a dict literal, not a declaration.
+ return not m.group("brace") and (
+ next_raw.strip() != "{" or raw.rstrip().endswith(",")
+ )
+
+
+def _indent_width(raw: str) -> int:
+ return len(raw) - len(raw.lstrip())
+
+
+# Stand-in indent for "the cut is inside something still open", used when every
+# withheld line is a string body or a bracket tail and none carries a real one.
+_UNBOUNDED_INDENT = 1 << 30
+
+
+# Cap on withheld definitions surfaced: names and signatures only, small enough
+# never to compete with the answer.
+_WITHHELD_MAX_SYMBOLS: int = 8
+
+
+def withheld_definitions(
+ repo_root: Path | None, continuation: str | None
+) -> list[dict]:
+ """Definitions that live in the range a truncated body did NOT serve.
+
+ ``continuation`` is the ``path:first-last`` pointer already attached to a
+ truncated ``symbol_bodies`` entry, so this reads exactly the lines the
+ payload admits it withheld.
+
+ More than a truncation flag: the withheld range often holds the symbol the
+ answer is about, and names and signatures are cheap.
+
+ Returns ``[{name, kind, line, symbol_id, signature}]``, the boundary-cut
+ symbol first, empty on any failure (a probe that cannot read must not
+ manufacture doubt).
+ """
+ span = _parse_continuation(continuation)
+ if span is None:
+ return []
+ path, lo, hi = span
+ text = _read_repo_text(repo_root, path)
+ if text is None:
+ return []
+ lines = text.splitlines()
+ if lo < 1 or lo > len(lines):
+ return []
+ mask = _string_masked_lines(tuple(lines), _has_backtick_strings(path))
+ end = min(hi, len(lines))
+
+ out: list[dict] = []
+ anchor = _cut_anchor_indent(lines, lo, end, mask)
+ cut = _definition_cut_by_boundary(lines, lo, anchor, mask.all)
+ if cut is not None:
+ out.append(_withheld_entry(repo_root, path, text, *cut, cut=True))
+
+ seen = {d["name"] for d in out}
+ for line_no, m in _definitions_starting_in(lines, lo, end, mask.all):
+ if m.group("name") in seen:
+ continue
+ seen.add(m.group("name"))
+ out.append(_withheld_entry(repo_root, path, text, line_no, m))
+ if len(out) >= _WITHHELD_MAX_SYMBOLS:
+ break
+ return out
+
+
+def _parse_continuation(continuation: str | None) -> tuple[str, int, int] | None:
+ """``(path, first, last)`` from a ``path:first-last`` pointer, or None."""
+ if not continuation:
+ return None
+ path, _, span = continuation.rpartition(":")
+ first, _, last = span.partition("-")
+ if not path or not first.isdigit() or not last.isdigit():
+ return None
+ return path, int(first), int(last)
+
+
+def _withheld_entry(
+ repo_root: Path | None,
+ path: str,
+ text: str,
+ line_no: int,
+ m: re.Match[str],
+ *,
+ cut: bool = False,
+) -> dict:
+ name = m.group("name")
+ # The brace-member shape has no keyword to report.
+ kind = m.groupdict().get("kind") or "member"
+ sig = _read_signature_from_source(repo_root, path, line_no, text=text)
+ e = {
+ "name": name,
+ "kind": kind,
+ "line": line_no,
+ "symbol_id": f"{path}::{name}",
+ "signature": (sig or f"{kind} {name}").strip(),
+ }
+ if cut:
+ e["body_continues"] = True
+ return e
+
+
+def _cut_anchor_indent(lines: list[str], lo: int, end: int, mask: _Masked) -> int | None:
+ """The indent a definition above ``lo`` must undercut to still be open there.
+
+ None when every withheld line is blank, so nothing can be continuing.
+ """
+ # Same exclusions as the walk: a cut on a signature's ``) -> dict:`` or a
+ # flush-left docstring line would otherwise read as column 0.
+ usable = [
+ n
+ for n in range(lo, end + 1)
+ if lines[n - 1].strip()
+ and n not in mask.all
+ and lines[n - 1].strip()[0] not in ")]}{"
+ ]
+ if lo in mask.strings:
+ # A cut inside a multi-line string leaves everything enclosing it open.
+ # Not so for a block comment, which can sit between two methods.
+ return _UNBOUNDED_INDENT
+ if usable:
+ return _indent_width(lines[usable[0] - 1])
+ if any(lines[n - 1].strip() for n in range(lo, end + 1)):
+ # Only string bodies and bracket tails: whatever encloses the cut is open.
+ return _UNBOUNDED_INDENT
+ return None
+
+
+def _definition_cut_by_boundary(
+ lines: list[str], lo: int, anchor: int | None, masked: frozenset[int]
+) -> tuple[int, re.Match[str]] | None:
+ """The definition above ``lo`` whose body the cut splits, if any.
+
+ Its ``def`` line was served, so it appears nowhere in the withheld range,
+ yet its body continues into it. Not simply the nearest preceding definition:
+ one that ended before the cut was served whole. A definition at indent I
+ reaches ``lo`` only if every non-blank line after it is deeper than I, so a
+ backward walk tracking the minimum indent decides it in one pass.
+ """
+ if anchor is None:
+ return None
+ min_indent = anchor
+ for back in range(lo - 1, 0, -1):
+ if min_indent <= 0:
+ break # nothing can be shallower, so nothing can still be open
+ raw = lines[back - 1]
+ stripped = raw.strip()
+ if not stripped:
+ continue
+ # A closing-bracket line is the tail of a multi-line construct (e.g. a
+ # signature's ``) -> dict:``), and an Allman ``{`` belongs to the
+ # declaration above: neither is a statement at its own indent.
+ if stripped[0] in ")]}{":
+ continue
+ ind = _indent_width(raw)
+ nxt = lines[back] if back < len(lines) else ""
+ m = None if back in masked else _match_definition(raw, nxt)
+ if m is not None and ind < min_indent:
+ return back, m
+ min_indent = min(min_indent, ind)
+ return None
+
+
+def _definitions_starting_in(
+ lines: list[str], lo: int, end: int, masked: frozenset[int]
+) -> Iterator[tuple[int, re.Match[str]]]:
+ """Every unmasked definition line in ``lo..end``, in order."""
+ for offset, raw in enumerate(lines[lo - 1 : end]):
+ line_no = lo + offset
+ if line_no in masked:
+ continue
+ nxt = lines[line_no] if line_no < len(lines) else ""
+ m = _match_definition(raw, nxt)
+ if m is not None:
+ yield line_no, m
diff --git a/tests/unit/server/mcp/test_answer_backtick_masking.py b/tests/unit/server/mcp/test_answer_backtick_masking.py
index 18031b8c3..9ce04943a 100644
--- a/tests/unit/server/mcp/test_answer_backtick_masking.py
+++ b/tests/unit/server/mcp/test_answer_backtick_masking.py
@@ -506,7 +506,9 @@ def test_the_fallback_costs_at_most_one_extra_walk(tmp_path) -> None:
disabled while a template frame is open and a pathological file therefore
pays a full per-character walk twice.
"""
- from repowise.server.mcp_server.tool_answer import symbols as mod
+ # Patch the module that owns the walk: a re-export's binding is not the
+ # one _string_masked_lines looks up.
+ from repowise.server.mcp_server.tool_answer import string_mask as mod
calls = []
real = mod._walk_string_state
diff --git a/tests/unit/server/mcp/test_answer_string_state.py b/tests/unit/server/mcp/test_answer_string_state.py
new file mode 100644
index 000000000..e0f180c7a
--- /dev/null
+++ b/tests/unit/server/mcp/test_answer_string_state.py
@@ -0,0 +1,68 @@
+"""Characterization of the string/comment scanner behind withheld-symbol masking.
+
+Pins ``_walk_string_state``'s raw three-part result, one lexical construct per
+case, so a restructured scanner must reproduce the exact line sets.
+"""
+
+import pytest
+
+from repowise.server.mcp_server.tool_answer.symbols import _walk_string_state
+
+
+def _walk(src: str, *, backticks: bool = True):
+ strings, comments, left_open = _walk_string_state(
+ tuple(src.split("\n")), backticks=backticks
+ )
+ return sorted(strings), sorted(comments), left_open
+
+
+ ("src", "backticks", "expected"),
+ [
+ # Python triple quotes, both spellings; the opening line is code.
+ ('def f():\n """Doc.\n def fake():\n """\n return 1', True, ([3, 4], [], False)),
+ ("x = '''\ndef fake():\n'''\ny = 1", True, ([2, 3], [], False)),
+ # Opened and closed on one line: nothing starts inside it.
+ ('x = """one line"""\ndef real():', True, ([], [], False)),
+ # A triple quote inside an ordinary string or a comment opens nothing.
+ ("x = '\"\"\"'\ndef real():", True, ([], [], False)),
+ ('x = "a\\"\\"\\"b"\ndef real():', True, ([], [], False)),
+ ('# a """ in a comment\ndef real():', True, ([], [], False)),
+ ('// a /* in a comment\nfunc real() {', True, ([], [], False)),
+ # C-style block comment, closed mid-line with code after it.
+ ("/* one\n * two\n */ int x = 1;\nint y;", False, ([], [2, 3], False)),
+ # Code after a closing triple quote is scanned again.
+ ('a = """\nb\n""" + "/*"\nc = 1', True, ([2, 3], [], False)),
+ # A template literal is only a string where backticks are strings.
+ ("const q = `\nfunction fake() {\n`;\nfunction real() {", True, ([2, 3], [], False)),
+ ("const q = `\nfunction fake() {\n`;\nfunction real() {", False, ([], [], False)),
+ # An interpolation holds code: its own lines are not masked, and a
+ # backtick inside it nests rather than closing the outer literal.
+ ("const a = `x ${\n f(`inner\n `)\n} y\n`;\nz", True, ([3, 5], [], False)),
+ # Braces inside an interpolation are counted, not taken as its end.
+ ("const a = `${ {k: 1}.k }\nstill`;\nz", True, ([2], [], False)),
+ # A quoted string or a line comment inside an interpolation.
+ ('const a = `${ "`" }\nstill`;\nz', True, ([2], [], False)),
+ ("const a = `${ x // `\n}\nrest`;\nz", True, ([3], [], False)),
+ # An escaped backtick does not close the literal.
+ ("const a = `x\\`y\nstill`;\nz", True, ([2], [], False)),
+ # A backtick inside a regex literal opens nothing, after punctuation
+ # or after a keyword a regex can follow.
+ ("s.replace(/[`]/g, '')\nfunction real() {", True, ([], [], False)),
+ ("return /`/.test(x)\nfunction real() {", True, ([], [], False)),
+ ("const a = `${s.replace(/[`]/g, '')}\nstill`;\nz", True, ([2], [], False)),
+ # A division is not a regex, so the backtick after it does open one.
+ ("n = a / b; q = `\nbody\n`", True, ([2, 3], [], False)),
+ # Runs left open at EOF: triple quotes and block comments are
+ # discarded from the line that opened them; a template is reported.
+ ('x = 1\ny = """\nz\nw', True, ([], [], False)),
+ ("x = 1\n/* never\nclosed", True, ([], [], False)),
+ ('a = """\nb\n"""\nc = """\nd', True, ([2, 3], [], False)),
+ ("const a = `\nnever closed", True, ([2], [], True)),
+ ("const a = `${\nf(", True, ([], [], True)),
+ # A line with nothing quote-like is skipped without a walk.
+ ("plain = 1\nother = 2", True, ([], [], False)),
+ ],
+)
+def test_walk_string_state(src, backticks, expected) -> None:
+ assert _walk(src, backticks=backticks) == expected