Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(mcp): ground data-shape questions in the field set mined from source (#779)

Raghav Chamadiya199293a6a2c9
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py
index b82f672d5..e10beb1fb 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py
@@ -101,6 +101,10 @@ from repowise.server.mcp_server.tool_answer.config import (
_SYSTEM_PROMPT,
_USER_TEMPLATE,
)
+from repowise.server.mcp_server.tool_answer.data_shape import (
+ _is_data_shape_question,
+ mine_data_shape,
+)
from repowise.server.mcp_server.tool_answer.retrieval import (
_apply_domain_penalty,
_candidate_justification,
@@ -289,6 +293,67 @@ def _gather_body_candidates(
return candidates
+def _build_data_shape_payload(grounded: dict, t0: float, repository) -> dict:
+ """Shape a grounded data-shape result into a get_answer response.
+
+ ``grounded`` is :func:`mine_data_shape`'s return. Cite the exact source
+ lines the fields were lifted from; a docstring shape is authoritative
+ (confidence high, no verification Read), a usage-mined shape is medium.
+ """
+ ident = grounded["identifier"]
+ fields = grounded["fields"]
+ sources = grounded["sources"]
+ citations = sorted({s["file"] for s in sources})
+ field_list = ", ".join(f"`{f}`" for f in fields)
+ doc_src = next((s for s in sources if s["kind"] == "docstring"), None)
+ if grounded["grounding"] == "docstring":
+ where = f"{doc_src['file']}:{doc_src['line']}" if doc_src else citations[0]
+ answer = (
+ f"Each entry in `{ident}` has {len(fields)} field(s): {field_list}. "
+ f"This is the documented shape at {where}; cite it directly, no "
+ "verification Read needed."
+ )
+ note = (
+ "Grounded in the documented field shape mined from source (the "
+ "quoted keys in the docstring/comment at the cited line). "
+ "data_shape.sources lists every field's origin line."
+ )
+ else:
+ first = sources[0]
+ answer = (
+ f"Each entry in `{ident}` is accessed with {len(fields)} key(s): "
+ f"{field_list}. These are the keys consumers actually pull off the "
+ f"parsed value (e.g. {first['file']}:{first['line']}); this is mined "
+ "from usage, not a declared schema, so verify if you need the full set."
+ )
+ note = (
+ "Grounded in the key accesses mined from consumer source (no "
+ "documented shape was found). Medium confidence: these are the keys "
+ "the code reads, which may be a subset of the stored fields."
+ )
+ payload: dict = {
+ "answer": answer,
+ "citations": citations,
+ "confidence": grounded["confidence"],
+ "grounding": "data_shape",
+ "data_shape": {
+ "identifier": ident,
+ "fields": fields,
+ "sources": sources,
+ },
+ "fallback_targets": citations,
+ "retrieval": [],
+ "note": note,
+ "_meta": _build_meta(
+ timing_ms=(time.perf_counter() - t0) * 1000,
+ hint=_answer_hint(grounded["confidence"], len(citations)),
+ repository=repository,
+ targets=citations,
+ ),
+ }
+ return payload
+
+
@mcp.tool()
async def get_answer(
question: str,
@@ -334,6 +399,22 @@ async def get_answer(
repository = await _get_repo(session)
repo_id = repository.id
+ # --- Data-shape fast path ----------------------------------------------
+ # "what fields does each entry in <blob> contain" is answered by mining the
+ # field set straight from source (a documented {...} shape, else consistent
+ # key accesses) instead of gating to a best_guesses pointer list — the exact
+ # payload that triggers the agent's Read/get_symbol drill. Runs before the
+ # cache and retrieval: it's deterministic from live source, cheap, and reads
+ # the field set directly (retrieval scatters across every file that touches
+ # the blob and misses the one file that documents it). Returns None (falls
+ # through) unless the fields are genuinely grounded, so it can never invent a
+ # shape.
+ ds_ids = _extract_question_identifiers(question)
+ if _is_data_shape_question(question, ds_ids):
+ grounded = await asyncio.to_thread(mine_data_shape, getattr(ctx, "path", None), ds_ids)
+ if grounded is not None:
+ return _build_data_shape_payload(grounded, t0, repository)
+
# --- Cache lookup --------------------------------------------------------
# Scope: ignore the (rare) `scope` argument in the cache key for now;
# scoped queries are uncommon and including scope would balloon hit rate
@@ -1045,9 +1126,7 @@ async def get_answer(
hits, limit=2, summary_chars=160, symbols_for_expanded=False
)
else:
- retrieval_view = _serialize_hits(
- hits, limit=_GATED_RETURN_HITS, lean_symbols=True
- )
+ retrieval_view = _serialize_hits(hits, limit=_GATED_RETURN_HITS, lean_symbols=True)
payload = {
"answer": answer_text,
"citations": citations,
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py b/packages/server/src/repowise/server/mcp_server/tool_answer/config.py
index 81085a4aa..d5647336f 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/config.py
@@ -66,6 +66,30 @@ _HOMONYM_UNION_CHAR_BUDGET = 12000
# Line cap per union body — same rationale as _INLINE_BODY_MAX_LINES.
_HOMONYM_UNION_BODY_MAX_LINES = 120
+# Data-shape grounding. "what fields does each entry in <blob> contain" is
+# answered directly by mining the field set from source instead of gating to a
+# best_guesses pointer list (the pointer list is exactly what triggers the
+# agent's Read/get_symbol drill). Two grounding sources, precision-ordered:
+# * a documented {...} shape in a docstring/comment near the identifier
+# (authoritative -> confidence high);
+# * consistent key accesses (VAR.get("f") / VAR["f"]) on the value bound from
+# the identifier (access-mined -> confidence medium).
+# Every reported field is a quoted token lifted verbatim from source, so the
+# path cannot synthesise a field with no source backing (no confidently-wrong
+# shapes). Returns nothing (falls through to normal retrieval) unless a shape
+# is genuinely grounded.
+# Cap on files scanned for the shape. A specific blob name can be referenced
+# across dozens of files (every consumer), and the one file that *documents* the
+# shape need not sort first, so the cap is generous - source files are small and
+# the doc scan must not miss the documenting file. Non-test files are scanned
+# first (fixtures document nothing), so the cap mostly trims trailing tests.
+_DATA_SHAPE_MAX_FILES = 30 # cap the identifier grep fan-out
+_DATA_SHAPE_DOC_WINDOW = 6 # lines below an identifier mention to scan for a {...}
+_DATA_SHAPE_ACCESS_WINDOW = 40 # lines below a binding to mine VAR key accesses
+_DATA_SHAPE_MIN_FIELDS = 2 # never answer from a single-field shape (too weak to trust)
+_DATA_SHAPE_MIN_IDENT_LEN = 6 # identifier must be specific, not a bare generic name
+_DATA_SHAPE_GREP_TIMEOUT_S = 6.0
+
# Sort priority by symbol kind. Classes first because "what does X do" /
# "which class inherits from Y" questions resolve at the class level. Then
# top-level functions, then methods (which usually only matter once the
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/data_shape.py b/packages/server/src/repowise/server/mcp_server/tool_answer/data_shape.py
new file mode 100644
index 000000000..b105cd077
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/data_shape.py
@@ -0,0 +1,408 @@
+"""Ground data-shape questions ("what fields does each entry in X contain?").
+
+A data-shape question names a data blob / row / record and asks for its field
+set. Fuzzy retrieval scatters across every file that *touches* the blob and
+gates low, so the tool hands back a best_guesses pointer list and the agent
+drills with Read/get_symbol to find the fields itself. But the answer usually
+lives verbatim in source: a documented ``{...}`` shape in a docstring near the
+identifier, and/or the concrete keys consumers pull off the parsed value
+(``partner.get("co_change_count")``). This module mines that field set directly
+so the tool answers in one call.
+
+Precision-first: every reported field is a quoted token lifted from source, so a
+field with no source backing can never be synthesised. Two grounding sources,
+precision-ordered: a documented brace shape (authoritative -> high) beats mined
+key accesses (usage-inferred -> medium). Returns ``None`` (the caller falls
+through to normal retrieval) unless a shape is genuinely grounded.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+
+from repowise.server.mcp_server.tool_answer.config import (
+ _DATA_SHAPE_ACCESS_WINDOW,
+ _DATA_SHAPE_DOC_WINDOW,
+ _DATA_SHAPE_GREP_TIMEOUT_S,
+ _DATA_SHAPE_MAX_FILES,
+ _DATA_SHAPE_MIN_FIELDS,
+ _DATA_SHAPE_MIN_IDENT_LEN,
+)
+
+# --- Question detection ---------------------------------------------------
+
+# Nouns that, when present alongside a named identifier, mark a question as
+# asking for a data shape rather than a mechanism ("how does X work").
+_SHAPE_NOUNS = frozenset(
+ {
+ "field",
+ "fields",
+ "key",
+ "keys",
+ "column",
+ "columns",
+ "schema",
+ "attribute",
+ "attributes",
+ "property",
+ "properties",
+ "shape",
+ "structure",
+ }
+)
+# Container nouns ("each entry in X", "what does each record hold") only count as
+# a data-shape signal when paired with a containment verb, so "entry point" and
+# "list items" don't over-fire.
+_CONTAINER_NOUNS = frozenset(
+ {"entry", "entries", "element", "elements", "record", "records", "item", "items"}
+)
+_CONTAINMENT_VERBS = ("contain", "consist", "comprise", "hold", "look like", "made of", "made up")
+
+_WORD = re.compile(r"[a-z_]+")
+
+
+def _is_data_shape_question(question: str, question_ids: set[str]) -> bool:
+ """Whether ``question`` asks for the field set of a named data blob.
+
+ Requires (a) a named identifier (something to ground on) and (b) a lexical
+ data-shape cue: a shape noun (field/key/column/schema/...), or a container
+ noun (entry/record/element/...) paired with a containment verb. Mechanism
+ questions ("how does X work") carry neither and fall through. The cue is a
+ cheap gate only; the miner is the real precision gate (it returns nothing
+ unless the fields are grounded in source), so a false-positive cue is safe.
+ """
+ if not question or not question_ids:
+ return False
+ low = question.lower()
+ words = set(_WORD.findall(low))
+ if words & _SHAPE_NOUNS:
+ return True
+ return bool(words & _CONTAINER_NOUNS) and any(v in low for v in _CONTAINMENT_VERBS)
+
+
+# --- Source mining --------------------------------------------------------
+
+_QUOTED_FIELD = re.compile(r"""['"]([A-Za-z_][A-Za-z0-9_]*)['"]""")
+_BRACE_GROUP = re.compile(r"\{([^{}]*)\}")
+_COMMENT_LEAD = ("#", "*", "//", "--")
+
+
+def _neg_str(s: str) -> tuple[int, ...]:
+ """Sort key that inverts string order (smaller string -> larger key).
+
+ Lets a ``max()`` tie-break toward the alphabetically-first path.
+ """
+ return tuple(-ord(c) for c in s)
+
+
+def _specific_identifiers(question_ids: set[str]) -> list[str]:
+ """Keep only identifiers specific enough to ground a data shape on.
+
+ A bare short/generic token ("id", "row") greps the whole tree and mines an
+ incoherent field union. Require a real data-name shape: long enough, and
+ either snake_case or CamelCase (a name a schema/blob actually carries).
+ Longest first so the most specific blob name is tried before its prefixes.
+ """
+ kept = [
+ q
+ for q in question_ids
+ if len(q) >= _DATA_SHAPE_MIN_IDENT_LEN and ("_" in q or any(c.isupper() for c in q))
+ ]
+ return sorted(kept, key=len, reverse=True)
+
+
+# Pathspecs limiting the grep to source and excluding heavy/vendored dirs. The
+# exclusions matter for the --no-index fallback (a non-git tree has no ignore
+# file); they're harmless on a tracked grep (those dirs are gitignored anyway).
+_GREP_PATHSPECS = (
+ "*.py",
+ "*.ts",
+ "*.tsx",
+ ":!**/node_modules/**",
+ ":!**/.venv/**",
+ ":!**/dist/**",
+ ":!**/build/**",
+)
+_TEST_PATH = re.compile(r"(^|/)tests?(/|$)|(^|/)test_[^/]*$|_test\.[^/]+$|\.spec\.[^/]+$")
+
+
+def _order_candidates(files: list[str]) -> list[str]:
+ """Order files so the one that *documents* the shape is scanned first.
+
+ Test/fixture files build shapes but never document them, so they sort last
+ (a fixture's incidental dict literal must not be the field set we mine when
+ a real doc exists). Within each group, shallower paths first, then name -
+ a declared schema usually lives nearer the package root than its consumers.
+ """
+ return sorted(
+ files,
+ key=lambda p: (bool(_TEST_PATH.search(p)), p.count("/"), p),
+ )
+
+
+def _run_grep(repo_root: Path, args: list[str], identifier: str) -> subprocess.CompletedProcess | None:
+ """Run a bounded, transport-safe ``git grep`` variant; ``None`` on failure."""
+ try:
+ return subprocess.run(
+ # --no-pager + stdin=DEVNULL are load-bearing: this can run inside a
+ # stdio MCP server whose stdin IS the JSON-RPC pipe, so a pager that
+ # reads stdin would deadlock the transport.
+ ["git", "--no-pager", "grep", *args, "-l", "-I", "-F", "-w", "-e", identifier, "--", *_GREP_PATHSPECS],
+ cwd=str(repo_root),
+ capture_output=True,
+ text=True,
+ stdin=subprocess.DEVNULL,
+ timeout=_DATA_SHAPE_GREP_TIMEOUT_S,
+ )
+ except Exception:
+ return None
+
+
+def _grep_identifier_files(repo_root: Path, identifier: str) -> list[str]:
+ """Source files naming ``identifier`` (whole word), repo-relative.
+
+ Tracked ``git grep`` first (fast, skips ignored/vendored). If the tree isn't
+ a git checkout (returncode 128), retry with ``--no-index`` so the tool still
+ grounds its answer on a non-git tree - both are fast C greps, never the
+ per-file Python read that wedges on a large tree. Returns the full match set;
+ the caller orders (doc files first) then caps, so the documenting file is
+ never dropped by an unlucky order.
+ """
+ proc = _run_grep(repo_root, [], identifier)
+ if proc is not None and proc.returncode == 128:
+ # Not a git repository - grep the filesystem directly.
+ proc = _run_grep(repo_root, ["--no-index"], identifier)
+ if proc is None or proc.returncode not in (0, 1):
+ return []
+ return [ln.strip().replace("\\", "/") for ln in proc.stdout.splitlines() if ln.strip()]
+
+
+def _read_lines(repo_root: Path, rel_path: str) -> list[str] | None:
+ """Read a repo-relative file's lines, refusing any path outside the root."""
+ try:
+ abs_path = (repo_root / rel_path).resolve()
+ abs_path.relative_to(repo_root.resolve())
+ return abs_path.read_text(encoding="utf-8", errors="replace").splitlines()
+ except (OSError, ValueError):
+ return None
+
+
+def _docstring_line_set(lines: list[str]) -> set[int]:
+ """1-based line numbers that fall inside a triple-quoted docstring.
+
+ Approximate (toggles on triple-quote parity, not a real parser) but enough
+ to tell a documented ``{...}`` shape from an executable dict literal. A
+ one-line docstring (two markers on one line) counts as inside.
+ """
+ in_doc = False
+ doc: set[int] = set()
+ for idx, line in enumerate(lines, 1):
+ cnt = line.count('"""') + line.count("'''")
+ if cnt % 2 == 1:
+ doc.add(idx) # the boundary line itself is part of the docstring
+ in_doc = not in_doc
+ elif in_doc:
+ doc.add(idx)
+ elif cnt >= 2:
+ doc.add(idx) # a self-contained one-line docstring
+ return doc
+
+
+def _fields_in_order(text: str) -> list[str]:
+ """Quoted field-name tokens inside ``text``, in order, deduped, >=2 chars."""
+ seen: list[str] = []
+ for m in _QUOTED_FIELD.finditer(text):
+ tok = m.group(1)
+ if len(tok) >= 2 and tok not in seen:
+ seen.append(tok)
+ return seen
+
+
+def _mention_lines(lines: list[str], identifier: str) -> list[int]:
+ """1-based line numbers where ``identifier`` appears as a whole word."""
+ pat = re.compile(r"\b" + re.escape(identifier) + r"\b")
+ return [idx for idx, line in enumerate(lines, 1) if pat.search(line)]
+
+
+def _doc_shape_in_file(
+ lines: list[str], identifier: str, mentions: list[int]
+) -> tuple[list[str], int] | None:
+ """A documented ``{...}`` field shape near an identifier mention, if any.
+
+ A brace group counts only when it is in documentation context (inside a
+ docstring, on a comment line, or wrapped in RST/markdown backticks), carries
+ >= _DATA_SHAPE_MIN_FIELDS quoted tokens, and sits within
+ _DATA_SHAPE_DOC_WINDOW lines of a mention of the identifier. Returns
+ ``(fields, line)`` for the first such brace, or ``None``.
+ """
+ text = "\n".join(lines)
+ doc_lines = _docstring_line_set(lines)
+ for m in _BRACE_GROUP.finditer(text):
+ start_line = text.count("\n", 0, m.start()) + 1
+ line_str = lines[start_line - 1] if start_line - 1 < len(lines) else ""
+ stripped = line_str.lstrip()
+ doc_ctx = start_line in doc_lines or stripped.startswith(_COMMENT_LEAD) or "`" in line_str
+ if not doc_ctx:
+ continue
+ fields = _fields_in_order(m.group(1))
+ if len(fields) < _DATA_SHAPE_MIN_FIELDS:
+ continue
+ # The mention must sit near the brace, on either side: the identifier is
+ # named then its shape documented (``blob is a list of {...}``), or the
+ # shape is commented just above the field declaration (``# {...}`` then
+ # ``field: type``). Both are common documentation styles.
+ if not any(abs(start_line - ml) <= _DATA_SHAPE_DOC_WINDOW for ml in mentions):
+ continue
+ return fields, start_line
+ return None
+
+
+def _accessed_fields_in_file(
+ lines: list[str], identifier: str, mentions: list[int]
+) -> list[tuple[str, int]]:
+ """Field keys pulled off the value bound from the identifier in this file.
+
+ Finds a variable bound to the identifier on a mention line (``for VAR in
+ ...ident...`` or ``VAR = ...ident...``), then mines ``VAR.get("f")`` /
+ ``VAR["f"]`` within the next _DATA_SHAPE_ACCESS_WINDOW lines. Also mines
+ direct ``ident.get("f")`` / ``ident["f"]`` accesses (when the identifier is
+ itself the dict). Returns ``(field, line)`` pairs in first-seen order.
+ """
+ bind_re = re.compile(
+ r"\bfor\s+([A-Za-z_]\w*)\s+in\b.*\b"
+ + re.escape(identifier)
+ + r"\b|\b([A-Za-z_]\w*)\s*=\s*.*\b"
+ + re.escape(identifier)
+ + r"\b"
+ )
+ # Variables to mine accesses on, each scoped to the line it was bound at.
+ scoped: list[tuple[str, int]] = []
+ mention_set = set(mentions)
+ for idx, line in enumerate(lines, 1):
+ if idx not in mention_set:
+ continue
+ m = bind_re.search(line)
+ if m:
+ var = m.group(1) or m.group(2)
+ if var and var != identifier:
+ scoped.append((var, idx))
+ # The identifier itself, mined across every one of its mention lines.
+ scoped.extend((identifier, ml) for ml in mentions)
+
+ found: list[tuple[str, int]] = []
+ seen: set[str] = set()
+ for var, start in scoped:
+ get_re = re.compile(r"\b" + re.escape(var) + r"\s*\.\s*get\(\s*['\"]([A-Za-z_]\w*)['\"]")
+ sub_re = re.compile(r"\b" + re.escape(var) + r"\s*\[\s*['\"]([A-Za-z_]\w*)['\"]\s*\]")
+ hi = min(start + _DATA_SHAPE_ACCESS_WINDOW, len(lines))
+ for i in range(start - 1, hi):
+ for rx in (get_re, sub_re):
+ for m in rx.finditer(lines[i]):
+ tok = m.group(1)
+ if len(tok) >= 2 and tok not in seen:
+ seen.add(tok)
+ found.append((tok, i + 1))
+ return found
+
+
+def mine_data_shape(repo_root: Path | None, question_ids: set[str]) -> dict | None:
+ """Ground the field set of a named data blob directly from source.
+
+ Blocking (subprocess grep + file reads) - call via ``asyncio.to_thread``.
+ Tries each specific identifier the question named; the first that grounds
+ wins. Returns ``None`` when nothing is grounded so the caller falls through
+ to normal retrieval.
+
+ Return shape::
+
+ {
+ "identifier": str,
+ "fields": [str, ...], # ordered, deduped, all from source
+ "grounding": "docstring" | "access",
+ "confidence": "high" | "medium",
+ "sources": [{"file", "line", "kind"}], # kind in {docstring, access}
+ }
+ """
+ if repo_root is None:
+ return None
+ try:
+ root = Path(str(repo_root))
+ except Exception:
+ return None
+
+ for identifier in _specific_identifiers(question_ids):
+ files = _grep_identifier_files(root, identifier)
+ if not files:
+ continue
+ # Scan documenting files first, then cap: the file that documents the
+ # shape need not sort first among dozens of consumers.
+ files = _order_candidates(files)[:_DATA_SHAPE_MAX_FILES]
+
+ doc_hits: list[tuple[frozenset, list[str], str, int]] = []
+ access_fields: list[str] = []
+ access_sources: list[dict] = []
+ access_seen: set[str] = set()
+
+ for rel in files:
+ lines = _read_lines(root, rel)
+ if not lines:
+ continue
+ mentions = _mention_lines(lines, identifier)
+ if not mentions:
+ continue
+ doc = _doc_shape_in_file(lines, identifier, mentions)
+ if doc is not None:
+ fields, line = doc
+ doc_hits.append((frozenset(fields), fields, rel, line))
+ for field, line in _accessed_fields_in_file(lines, identifier, mentions):
+ if field not in access_seen:
+ access_seen.add(field)
+ access_fields.append(field)
+ access_sources.append({"file": rel, "line": line, "kind": "access"})
+
+ # A documented shape is authoritative. When several files document a
+ # shape, take the most common set (mode), tie-broken by size then path,
+ # so one file's incidental brace can't override a repeated schema doc.
+ if doc_hits:
+ by_set: dict[frozenset, list[tuple[list[str], str, int]]] = {}
+ for fset, fields, rel, line in doc_hits:
+ by_set.setdefault(fset, []).append((fields, rel, line))
+
+ def _rank(item: tuple[frozenset, list[tuple[list[str], str, int]]]):
+ fset, group = item
+ # Most files agreeing wins; then more fields; then the
+ # alphabetically-first documenting path (deterministic).
+ first_path = min(t[1] for t in group)
+ return (len(group), len(fset), _neg_str(first_path))
+
+ best_set, _ = max(by_set.items(), key=_rank)
+ group = sorted(by_set[best_set], key=lambda t: t[1])
+ fields = group[0][0]
+ sources = [{"file": rel, "line": line, "kind": "docstring"} for _f, rel, line in group]
+ # Corroborating access sites (same-named fields) strengthen the cite
+ # trail without changing the authoritative doc field set.
+ for src in access_sources:
+ if any(src["file"] == s["file"] for s in sources):
+ continue
+ sources.append(src)
+ return {
+ "identifier": identifier,
+ "fields": fields,
+ "grounding": "docstring",
+ "confidence": "high",
+ "sources": sources,
+ }
+
+ # No documented shape - fall back to consistent key accesses.
+ if len(access_fields) >= _DATA_SHAPE_MIN_FIELDS:
+ return {
+ "identifier": identifier,
+ "fields": access_fields,
+ "grounding": "access",
+ "confidence": "medium",
+ "sources": access_sources,
+ }
+
+ return None
diff --git a/tests/unit/server/mcp/test_answer_data_shape.py b/tests/unit/server/mcp/test_answer_data_shape.py
new file mode 100644
index 000000000..b38f75ce1
--- /dev/null
+++ b/tests/unit/server/mcp/test_answer_data_shape.py
@@ -0,0 +1,241 @@
+"""Data-shape grounding: answer "what fields does X contain" from source.
+
+A data-shape question names a data blob and asks for its field set. Instead of
+gating to a best_guesses pointer list (which triggers the agent's Read/get_symbol
+drill), the tool mines the field set straight from source: a documented ``{...}``
+shape near the identifier (authoritative -> high) or the concrete keys consumers
+pull off the parsed value (usage-mined -> medium).
+
+These tests use synthetic repos with varied identifiers and shapes so nothing is
+tuned to any one real question. The precision contract is the load-bearing part:
+every reported field must be a quoted token from source, and an identifier with
+no groundable shape must fall through (return None), never a fabricated field.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from repowise.server.mcp_server.tool_answer.data_shape import (
+ _is_data_shape_question,
+ mine_data_shape,
+)
+
+
+def _write(root: Path, rel: str, body: str) -> None:
+ p = root / rel
+ p.parent.mkdir(parents=True, exist_ok=True)
+ p.write_text(body, encoding="utf-8")
+
+
+# --- Question detection ---------------------------------------------------
+
+
+ "question,ids,expected",
+ [
+ (
+ "What fields does each entry in co_change_partners_json contain?",
+ {"co_change_partners_json"},
+ True,
+ ),
+ ("what keys are in the blame_record blob", {"blame_record"}, True),
+ ("what columns does the health_metric_row have", {"health_metric_row"}, True),
+ ("describe the schema of GitCommitMeta", {"GitCommitMeta"}, True),
+ ("what does each record in owner_stats consist of", {"owner_stats"}, True),
+ # Mechanism questions carry no shape cue -> not data-shape.
+ ("how does co_change_partners_json get populated", {"co_change_partners_json"}, False),
+ ("where is the coupling graph assembled", set(), False),
+ # A shape cue with no named identifier can't ground -> not data-shape.
+ ("what fields are typical in a config", set(), False),
+ ],
+)
+def test_detection(question, ids, expected):
+ assert _is_data_shape_question(question, ids) is expected
+
+
+# --- Documented shape (authoritative -> high) -----------------------------
+
+
+def test_docstring_brace_shape_high_confidence(tmp_path):
+ """A {...} field shape in a docstring near the identifier grounds high."""
+ _write(
+ tmp_path,
+ "pkg/models.py",
+ '''\
+class GitMeta:
+ """Row model.
+
+ ``blame_record_json`` is the raw column: a JSON list of
+ ``{"author", "commit_sha", "line_count"}`` blame records.
+ """
+
+ blame_record_json: str
+''',
+ )
+ out = mine_data_shape(tmp_path, {"blame_record_json"})
+ assert out is not None
+ assert out["grounding"] == "docstring"
+ assert out["confidence"] == "high"
+ assert out["fields"] == ["author", "commit_sha", "line_count"]
+ assert out["sources"][0]["file"] == "pkg/models.py"
+ assert out["sources"][0]["kind"] == "docstring"
+
+
+def test_comment_brace_shape(tmp_path):
+ """A shape documented in a line comment also grounds (not just docstrings)."""
+ _write(
+ tmp_path,
+ "pkg/store.py",
+ """\
+def load(raw):
+ # owner_stats rows look like {"owner", "files_touched", "last_seen"}
+ return parse(raw)
+""",
+ )
+ out = mine_data_shape(tmp_path, {"owner_stats"})
+ assert out is not None
+ assert out["grounding"] == "docstring"
+ assert set(out["fields"]) == {"owner", "files_touched", "last_seen"}
+
+
+def test_executable_dict_literal_is_not_authoritative(tmp_path):
+ """A bare dict literal in code (no doc context) is not a documented shape.
+
+ It has quoted keys, but it's one construction site, not a declared schema.
+ The access-mining path may still pick up keys, but the doc path must not
+ mislabel an executable literal as authoritative/high.
+ """
+ _write(
+ tmp_path,
+ "pkg/build.py",
+ """\
+def build(widget_payload, count):
+ result = {"kind": widget_payload, "n": count, "ok": True}
+ return result
+""",
+ )
+ out = mine_data_shape(tmp_path, {"widget_payload"})
+ # Nothing is documented and the identifier is a scalar arg (no key access),
+ # so this must fall through rather than claim {kind,n,ok} as its shape.
+ assert out is None
+
+
+# --- Access-mined shape (usage -> medium) ---------------------------------
+
+
+def test_access_mined_shape_medium_when_no_doc(tmp_path):
+ """With no documented shape, consistent key accesses ground at medium."""
+ _write(
+ tmp_path,
+ "pkg/consume.py",
+ """\
+def summarize(partner_records):
+ for entry in iter_records(partner_records):
+ name = entry.get("file_path")
+ weight = entry["co_change_count"]
+ stamp = entry.get("last_seen")
+ return name, weight, stamp
+""",
+ )
+ out = mine_data_shape(tmp_path, {"partner_records"})
+ assert out is not None
+ assert out["grounding"] == "access"
+ assert out["confidence"] == "medium"
+ assert set(out["fields"]) == {"file_path", "co_change_count", "last_seen"}
+
+
+def test_direct_subscript_on_identifier(tmp_path):
+ """When the identifier IS the dict variable, direct key access grounds it."""
+ _write(
+ tmp_path,
+ "pkg/direct.py",
+ """\
+def read(config_blob):
+ host = config_blob["host"]
+ port = config_blob.get("port")
+ return host, port
+""",
+ )
+ out = mine_data_shape(tmp_path, {"config_blob"})
+ assert out is not None
+ assert out["grounding"] == "access"
+ assert set(out["fields"]) == {"host", "port"}
+
+
+# --- Precision guards -----------------------------------------------------
+
+
+def test_no_shape_returns_none(tmp_path):
+ """An identifier that names no blob shape falls through (no fabrication)."""
+ _write(tmp_path, "pkg/plain.py", "def process_records(x):\n return x + 1\n")
+ assert mine_data_shape(tmp_path, {"process_records"}) is None
+
+
+def test_single_field_is_too_weak(tmp_path):
+ """A one-field shape is below the min-fields floor -> abstain."""
+ _write(
+ tmp_path,
+ "pkg/thin.py",
+ """\
+def read(thin_blob):
+ # thin_blob entries are {"only_field"}
+ return thin_blob
+""",
+ )
+ assert mine_data_shape(tmp_path, {"thin_blob"}) is None
+
+
+def test_generic_short_identifier_skipped(tmp_path):
+ """Short/generic identifiers are not specific enough to ground on."""
+ _write(
+ tmp_path,
+ "pkg/rows.py",
+ """\
+def go(row):
+ a = row.get("x")
+ b = row.get("y")
+ return a, b
+""",
+ )
+ # "row" is below the specificity floor (too short, no underscore/camel).
+ assert mine_data_shape(tmp_path, {"row"}) is None
+
+
+def test_absent_identifier_returns_none(tmp_path):
+ """An identifier that appears nowhere in source grounds nothing."""
+ _write(tmp_path, "pkg/a.py", "x = 1\n")
+ assert mine_data_shape(tmp_path, {"nonexistent_blob_xyz"}) is None
+
+
+def test_doc_beats_access(tmp_path):
+ """When both a doc shape and accesses exist, the doc shape wins (high)."""
+ _write(
+ tmp_path,
+ "pkg/both.py",
+ '''\
+class M:
+ """``event_payload_json`` is a list of
+ ``{"kind", "ts", "actor"}`` event records."""
+
+ event_payload_json: str
+
+
+def consume(event_payload_json):
+ for e in parse(event_payload_json):
+ k = e.get("kind")
+ t = e.get("ts")
+ return k, t
+''',
+ )
+ out = mine_data_shape(tmp_path, {"event_payload_json"})
+ assert out is not None
+ assert out["grounding"] == "docstring"
+ assert out["confidence"] == "high"
+ assert out["fields"] == ["kind", "ts", "actor"]
+
+
+def test_none_repo_root_is_safe():
+ assert mine_data_shape(None, {"anything_json"}) is None