Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(mcp): ground data-shape questions in the field set mined from source (#779)
Raghav Chamadiya199293a6a2c9
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.pyindex b82f672d5..e10beb1fb 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py@@ -101,6 +101,10 @@ from repowise.server.mcp_server.tool_answer.config import (_SYSTEM_PROMPT,_USER_TEMPLATE,)+from repowise.server.mcp_server.tool_answer.data_shape import (+ _is_data_shape_question,+ mine_data_shape,+)from repowise.server.mcp_server.tool_answer.retrieval import (_apply_domain_penalty,_candidate_justification,@@ -289,6 +293,67 @@ def _gather_body_candidates(return candidates+def _build_data_shape_payload(grounded: dict, t0: float, repository) -> dict:+ """Shape a grounded data-shape result into a get_answer response.++ ``grounded`` is :func:`mine_data_shape`'s return. Cite the exact source+ lines the fields were lifted from; a docstring shape is authoritative+ (confidence high, no verification Read), a usage-mined shape is medium.+ """+ ident = grounded["identifier"]+ fields = grounded["fields"]+ sources = grounded["sources"]+ citations = sorted({s["file"] for s in sources})+ field_list = ", ".join(f"`{f}`" for f in fields)+ doc_src = next((s for s in sources if s["kind"] == "docstring"), None)+ if grounded["grounding"] == "docstring":+ where = f"{doc_src['file']}:{doc_src['line']}" if doc_src else citations[0]+ answer = (+ f"Each entry in `{ident}` has {len(fields)} field(s): {field_list}. "+ f"This is the documented shape at {where}; cite it directly, no "+ "verification Read needed."+ )+ note = (+ "Grounded in the documented field shape mined from source (the "+ "quoted keys in the docstring/comment at the cited line). "+ "data_shape.sources lists every field's origin line."+ )+ else:+ first = sources[0]+ answer = (+ f"Each entry in `{ident}` is accessed with {len(fields)} key(s): "+ f"{field_list}. These are the keys consumers actually pull off the "+ f"parsed value (e.g. {first['file']}:{first['line']}); this is mined "+ "from usage, not a declared schema, so verify if you need the full set."+ )+ note = (+ "Grounded in the key accesses mined from consumer source (no "+ "documented shape was found). Medium confidence: these are the keys "+ "the code reads, which may be a subset of the stored fields."+ )+ payload: dict = {+ "answer": answer,+ "citations": citations,+ "confidence": grounded["confidence"],+ "grounding": "data_shape",+ "data_shape": {+ "identifier": ident,+ "fields": fields,+ "sources": sources,+ },+ "fallback_targets": citations,+ "retrieval": [],+ "note": note,+ "_meta": _build_meta(+ timing_ms=(time.perf_counter() - t0) * 1000,+ hint=_answer_hint(grounded["confidence"], len(citations)),+ repository=repository,+ targets=citations,+ ),+ }+ return payload++@mcp.tool()async def get_answer(question: str,@@ -334,6 +399,22 @@ async def get_answer(repository = await _get_repo(session)repo_id = repository.id+ # --- Data-shape fast path ----------------------------------------------+ # "what fields does each entry in <blob> contain" is answered by mining the+ # field set straight from source (a documented {...} shape, else consistent+ # key accesses) instead of gating to a best_guesses pointer list — the exact+ # payload that triggers the agent's Read/get_symbol drill. Runs before the+ # cache and retrieval: it's deterministic from live source, cheap, and reads+ # the field set directly (retrieval scatters across every file that touches+ # the blob and misses the one file that documents it). Returns None (falls+ # through) unless the fields are genuinely grounded, so it can never invent a+ # shape.+ ds_ids = _extract_question_identifiers(question)+ if _is_data_shape_question(question, ds_ids):+ grounded = await asyncio.to_thread(mine_data_shape, getattr(ctx, "path", None), ds_ids)+ if grounded is not None:+ return _build_data_shape_payload(grounded, t0, repository)+# --- Cache lookup --------------------------------------------------------# Scope: ignore the (rare) `scope` argument in the cache key for now;# scoped queries are uncommon and including scope would balloon hit rate@@ -1045,9 +1126,7 @@ async def get_answer(hits, limit=2, summary_chars=160, symbols_for_expanded=False)else:- retrieval_view = _serialize_hits(- hits, limit=_GATED_RETURN_HITS, lean_symbols=True- )+ retrieval_view = _serialize_hits(hits, limit=_GATED_RETURN_HITS, lean_symbols=True)payload = {"answer": answer_text,"citations": citations,diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py b/packages/server/src/repowise/server/mcp_server/tool_answer/config.pyindex 81085a4aa..d5647336f 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/config.py@@ -66,6 +66,30 @@ _HOMONYM_UNION_CHAR_BUDGET = 12000# Line cap per union body — same rationale as _INLINE_BODY_MAX_LINES._HOMONYM_UNION_BODY_MAX_LINES = 120+# Data-shape grounding. "what fields does each entry in <blob> contain" is+# answered directly by mining the field set from source instead of gating to a+# best_guesses pointer list (the pointer list is exactly what triggers the+# agent's Read/get_symbol drill). Two grounding sources, precision-ordered:+# * a documented {...} shape in a docstring/comment near the identifier+# (authoritative -> confidence high);+# * consistent key accesses (VAR.get("f") / VAR["f"]) on the value bound from+# the identifier (access-mined -> confidence medium).+# Every reported field is a quoted token lifted verbatim from source, so the+# path cannot synthesise a field with no source backing (no confidently-wrong+# shapes). Returns nothing (falls through to normal retrieval) unless a shape+# is genuinely grounded.+# Cap on files scanned for the shape. A specific blob name can be referenced+# across dozens of files (every consumer), and the one file that *documents* the+# shape need not sort first, so the cap is generous - source files are small and+# the doc scan must not miss the documenting file. Non-test files are scanned+# first (fixtures document nothing), so the cap mostly trims trailing tests.+_DATA_SHAPE_MAX_FILES = 30 # cap the identifier grep fan-out+_DATA_SHAPE_DOC_WINDOW = 6 # lines below an identifier mention to scan for a {...}+_DATA_SHAPE_ACCESS_WINDOW = 40 # lines below a binding to mine VAR key accesses+_DATA_SHAPE_MIN_FIELDS = 2 # never answer from a single-field shape (too weak to trust)+_DATA_SHAPE_MIN_IDENT_LEN = 6 # identifier must be specific, not a bare generic name+_DATA_SHAPE_GREP_TIMEOUT_S = 6.0+# Sort priority by symbol kind. Classes first because "what does X do" /# "which class inherits from Y" questions resolve at the class level. Then# top-level functions, then methods (which usually only matter once thediff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/data_shape.py b/packages/server/src/repowise/server/mcp_server/tool_answer/data_shape.pynew file mode 100644index 000000000..b105cd077--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/data_shape.py@@ -0,0 +1,408 @@+"""Ground data-shape questions ("what fields does each entry in X contain?").++A data-shape question names a data blob / row / record and asks for its field+set. Fuzzy retrieval scatters across every file that *touches* the blob and+gates low, so the tool hands back a best_guesses pointer list and the agent+drills with Read/get_symbol to find the fields itself. But the answer usually+lives verbatim in source: a documented ``{...}`` shape in a docstring near the+identifier, and/or the concrete keys consumers pull off the parsed value+(``partner.get("co_change_count")``). This module mines that field set directly+so the tool answers in one call.++Precision-first: every reported field is a quoted token lifted from source, so a+field with no source backing can never be synthesised. Two grounding sources,+precision-ordered: a documented brace shape (authoritative -> high) beats mined+key accesses (usage-inferred -> medium). Returns ``None`` (the caller falls+through to normal retrieval) unless a shape is genuinely grounded.+"""++from __future__ import annotations++import re+import subprocess+from pathlib import Path++from repowise.server.mcp_server.tool_answer.config import (+ _DATA_SHAPE_ACCESS_WINDOW,+ _DATA_SHAPE_DOC_WINDOW,+ _DATA_SHAPE_GREP_TIMEOUT_S,+ _DATA_SHAPE_MAX_FILES,+ _DATA_SHAPE_MIN_FIELDS,+ _DATA_SHAPE_MIN_IDENT_LEN,+)++# --- Question detection ---------------------------------------------------++# Nouns that, when present alongside a named identifier, mark a question as+# asking for a data shape rather than a mechanism ("how does X work").+_SHAPE_NOUNS = frozenset(+ {+ "field",+ "fields",+ "key",+ "keys",+ "column",+ "columns",+ "schema",+ "attribute",+ "attributes",+ "property",+ "properties",+ "shape",+ "structure",+ }+)+# Container nouns ("each entry in X", "what does each record hold") only count as+# a data-shape signal when paired with a containment verb, so "entry point" and+# "list items" don't over-fire.+_CONTAINER_NOUNS = frozenset(+ {"entry", "entries", "element", "elements", "record", "records", "item", "items"}+)+_CONTAINMENT_VERBS = ("contain", "consist", "comprise", "hold", "look like", "made of", "made up")++_WORD = re.compile(r"[a-z_]+")+++def _is_data_shape_question(question: str, question_ids: set[str]) -> bool:+ """Whether ``question`` asks for the field set of a named data blob.++ Requires (a) a named identifier (something to ground on) and (b) a lexical+ data-shape cue: a shape noun (field/key/column/schema/...), or a container+ noun (entry/record/element/...) paired with a containment verb. Mechanism+ questions ("how does X work") carry neither and fall through. The cue is a+ cheap gate only; the miner is the real precision gate (it returns nothing+ unless the fields are grounded in source), so a false-positive cue is safe.+ """+ if not question or not question_ids:+ return False+ low = question.lower()+ words = set(_WORD.findall(low))+ if words & _SHAPE_NOUNS:+ return True+ return bool(words & _CONTAINER_NOUNS) and any(v in low for v in _CONTAINMENT_VERBS)+++# --- Source mining --------------------------------------------------------++_QUOTED_FIELD = re.compile(r"""['"]([A-Za-z_][A-Za-z0-9_]*)['"]""")+_BRACE_GROUP = re.compile(r"\{([^{}]*)\}")+_COMMENT_LEAD = ("#", "*", "//", "--")+++def _neg_str(s: str) -> tuple[int, ...]:+ """Sort key that inverts string order (smaller string -> larger key).++ Lets a ``max()`` tie-break toward the alphabetically-first path.+ """+ return tuple(-ord(c) for c in s)+++def _specific_identifiers(question_ids: set[str]) -> list[str]:+ """Keep only identifiers specific enough to ground a data shape on.++ A bare short/generic token ("id", "row") greps the whole tree and mines an+ incoherent field union. Require a real data-name shape: long enough, and+ either snake_case or CamelCase (a name a schema/blob actually carries).+ Longest first so the most specific blob name is tried before its prefixes.+ """+ kept = [+ q+ for q in question_ids+ if len(q) >= _DATA_SHAPE_MIN_IDENT_LEN and ("_" in q or any(c.isupper() for c in q))+ ]+ return sorted(kept, key=len, reverse=True)+++# Pathspecs limiting the grep to source and excluding heavy/vendored dirs. The+# exclusions matter for the --no-index fallback (a non-git tree has no ignore+# file); they're harmless on a tracked grep (those dirs are gitignored anyway).+_GREP_PATHSPECS = (+ "*.py",+ "*.ts",+ "*.tsx",+ ":!**/node_modules/**",+ ":!**/.venv/**",+ ":!**/dist/**",+ ":!**/build/**",+)+_TEST_PATH = re.compile(r"(^|/)tests?(/|$)|(^|/)test_[^/]*$|_test\.[^/]+$|\.spec\.[^/]+$")+++def _order_candidates(files: list[str]) -> list[str]:+ """Order files so the one that *documents* the shape is scanned first.++ Test/fixture files build shapes but never document them, so they sort last+ (a fixture's incidental dict literal must not be the field set we mine when+ a real doc exists). Within each group, shallower paths first, then name -+ a declared schema usually lives nearer the package root than its consumers.+ """+ return sorted(+ files,+ key=lambda p: (bool(_TEST_PATH.search(p)), p.count("/"), p),+ )+++def _run_grep(repo_root: Path, args: list[str], identifier: str) -> subprocess.CompletedProcess | None:+ """Run a bounded, transport-safe ``git grep`` variant; ``None`` on failure."""+ try:+ return subprocess.run(+ # --no-pager + stdin=DEVNULL are load-bearing: this can run inside a+ # stdio MCP server whose stdin IS the JSON-RPC pipe, so a pager that+ # reads stdin would deadlock the transport.+ ["git", "--no-pager", "grep", *args, "-l", "-I", "-F", "-w", "-e", identifier, "--", *_GREP_PATHSPECS],+ cwd=str(repo_root),+ capture_output=True,+ text=True,+ stdin=subprocess.DEVNULL,+ timeout=_DATA_SHAPE_GREP_TIMEOUT_S,+ )+ except Exception:+ return None+++def _grep_identifier_files(repo_root: Path, identifier: str) -> list[str]:+ """Source files naming ``identifier`` (whole word), repo-relative.++ Tracked ``git grep`` first (fast, skips ignored/vendored). If the tree isn't+ a git checkout (returncode 128), retry with ``--no-index`` so the tool still+ grounds its answer on a non-git tree - both are fast C greps, never the+ per-file Python read that wedges on a large tree. Returns the full match set;+ the caller orders (doc files first) then caps, so the documenting file is+ never dropped by an unlucky order.+ """+ proc = _run_grep(repo_root, [], identifier)+ if proc is not None and proc.returncode == 128:+ # Not a git repository - grep the filesystem directly.+ proc = _run_grep(repo_root, ["--no-index"], identifier)+ if proc is None or proc.returncode not in (0, 1):+ return []+ return [ln.strip().replace("\\", "/") for ln in proc.stdout.splitlines() if ln.strip()]+++def _read_lines(repo_root: Path, rel_path: str) -> list[str] | None:+ """Read a repo-relative file's lines, refusing any path outside the root."""+ try:+ abs_path = (repo_root / rel_path).resolve()+ abs_path.relative_to(repo_root.resolve())+ return abs_path.read_text(encoding="utf-8", errors="replace").splitlines()+ except (OSError, ValueError):+ return None+++def _docstring_line_set(lines: list[str]) -> set[int]:+ """1-based line numbers that fall inside a triple-quoted docstring.++ Approximate (toggles on triple-quote parity, not a real parser) but enough+ to tell a documented ``{...}`` shape from an executable dict literal. A+ one-line docstring (two markers on one line) counts as inside.+ """+ in_doc = False+ doc: set[int] = set()+ for idx, line in enumerate(lines, 1):+ cnt = line.count('"""') + line.count("'''")+ if cnt % 2 == 1:+ doc.add(idx) # the boundary line itself is part of the docstring+ in_doc = not in_doc+ elif in_doc:+ doc.add(idx)+ elif cnt >= 2:+ doc.add(idx) # a self-contained one-line docstring+ return doc+++def _fields_in_order(text: str) -> list[str]:+ """Quoted field-name tokens inside ``text``, in order, deduped, >=2 chars."""+ seen: list[str] = []+ for m in _QUOTED_FIELD.finditer(text):+ tok = m.group(1)+ if len(tok) >= 2 and tok not in seen:+ seen.append(tok)+ return seen+++def _mention_lines(lines: list[str], identifier: str) -> list[int]:+ """1-based line numbers where ``identifier`` appears as a whole word."""+ pat = re.compile(r"\b" + re.escape(identifier) + r"\b")+ return [idx for idx, line in enumerate(lines, 1) if pat.search(line)]+++def _doc_shape_in_file(+ lines: list[str], identifier: str, mentions: list[int]+) -> tuple[list[str], int] | None:+ """A documented ``{...}`` field shape near an identifier mention, if any.++ A brace group counts only when it is in documentation context (inside a+ docstring, on a comment line, or wrapped in RST/markdown backticks), carries+ >= _DATA_SHAPE_MIN_FIELDS quoted tokens, and sits within+ _DATA_SHAPE_DOC_WINDOW lines of a mention of the identifier. Returns+ ``(fields, line)`` for the first such brace, or ``None``.+ """+ text = "\n".join(lines)+ doc_lines = _docstring_line_set(lines)+ for m in _BRACE_GROUP.finditer(text):+ start_line = text.count("\n", 0, m.start()) + 1+ line_str = lines[start_line - 1] if start_line - 1 < len(lines) else ""+ stripped = line_str.lstrip()+ doc_ctx = start_line in doc_lines or stripped.startswith(_COMMENT_LEAD) or "`" in line_str+ if not doc_ctx:+ continue+ fields = _fields_in_order(m.group(1))+ if len(fields) < _DATA_SHAPE_MIN_FIELDS:+ continue+ # The mention must sit near the brace, on either side: the identifier is+ # named then its shape documented (``blob is a list of {...}``), or the+ # shape is commented just above the field declaration (``# {...}`` then+ # ``field: type``). Both are common documentation styles.+ if not any(abs(start_line - ml) <= _DATA_SHAPE_DOC_WINDOW for ml in mentions):+ continue+ return fields, start_line+ return None+++def _accessed_fields_in_file(+ lines: list[str], identifier: str, mentions: list[int]+) -> list[tuple[str, int]]:+ """Field keys pulled off the value bound from the identifier in this file.++ Finds a variable bound to the identifier on a mention line (``for VAR in+ ...ident...`` or ``VAR = ...ident...``), then mines ``VAR.get("f")`` /+ ``VAR["f"]`` within the next _DATA_SHAPE_ACCESS_WINDOW lines. Also mines+ direct ``ident.get("f")`` / ``ident["f"]`` accesses (when the identifier is+ itself the dict). Returns ``(field, line)`` pairs in first-seen order.+ """+ bind_re = re.compile(+ r"\bfor\s+([A-Za-z_]\w*)\s+in\b.*\b"+ + re.escape(identifier)+ + r"\b|\b([A-Za-z_]\w*)\s*=\s*.*\b"+ + re.escape(identifier)+ + r"\b"+ )+ # Variables to mine accesses on, each scoped to the line it was bound at.+ scoped: list[tuple[str, int]] = []+ mention_set = set(mentions)+ for idx, line in enumerate(lines, 1):+ if idx not in mention_set:+ continue+ m = bind_re.search(line)+ if m:+ var = m.group(1) or m.group(2)+ if var and var != identifier:+ scoped.append((var, idx))+ # The identifier itself, mined across every one of its mention lines.+ scoped.extend((identifier, ml) for ml in mentions)++ found: list[tuple[str, int]] = []+ seen: set[str] = set()+ for var, start in scoped:+ get_re = re.compile(r"\b" + re.escape(var) + r"\s*\.\s*get\(\s*['\"]([A-Za-z_]\w*)['\"]")+ sub_re = re.compile(r"\b" + re.escape(var) + r"\s*\[\s*['\"]([A-Za-z_]\w*)['\"]\s*\]")+ hi = min(start + _DATA_SHAPE_ACCESS_WINDOW, len(lines))+ for i in range(start - 1, hi):+ for rx in (get_re, sub_re):+ for m in rx.finditer(lines[i]):+ tok = m.group(1)+ if len(tok) >= 2 and tok not in seen:+ seen.add(tok)+ found.append((tok, i + 1))+ return found+++def mine_data_shape(repo_root: Path | None, question_ids: set[str]) -> dict | None:+ """Ground the field set of a named data blob directly from source.++ Blocking (subprocess grep + file reads) - call via ``asyncio.to_thread``.+ Tries each specific identifier the question named; the first that grounds+ wins. Returns ``None`` when nothing is grounded so the caller falls through+ to normal retrieval.++ Return shape::++ {+ "identifier": str,+ "fields": [str, ...], # ordered, deduped, all from source+ "grounding": "docstring" | "access",+ "confidence": "high" | "medium",+ "sources": [{"file", "line", "kind"}], # kind in {docstring, access}+ }+ """+ if repo_root is None:+ return None+ try:+ root = Path(str(repo_root))+ except Exception:+ return None++ for identifier in _specific_identifiers(question_ids):+ files = _grep_identifier_files(root, identifier)+ if not files:+ continue+ # Scan documenting files first, then cap: the file that documents the+ # shape need not sort first among dozens of consumers.+ files = _order_candidates(files)[:_DATA_SHAPE_MAX_FILES]++ doc_hits: list[tuple[frozenset, list[str], str, int]] = []+ access_fields: list[str] = []+ access_sources: list[dict] = []+ access_seen: set[str] = set()++ for rel in files:+ lines = _read_lines(root, rel)+ if not lines:+ continue+ mentions = _mention_lines(lines, identifier)+ if not mentions:+ continue+ doc = _doc_shape_in_file(lines, identifier, mentions)+ if doc is not None:+ fields, line = doc+ doc_hits.append((frozenset(fields), fields, rel, line))+ for field, line in _accessed_fields_in_file(lines, identifier, mentions):+ if field not in access_seen:+ access_seen.add(field)+ access_fields.append(field)+ access_sources.append({"file": rel, "line": line, "kind": "access"})++ # A documented shape is authoritative. When several files document a+ # shape, take the most common set (mode), tie-broken by size then path,+ # so one file's incidental brace can't override a repeated schema doc.+ if doc_hits:+ by_set: dict[frozenset, list[tuple[list[str], str, int]]] = {}+ for fset, fields, rel, line in doc_hits:+ by_set.setdefault(fset, []).append((fields, rel, line))++ def _rank(item: tuple[frozenset, list[tuple[list[str], str, int]]]):+ fset, group = item+ # Most files agreeing wins; then more fields; then the+ # alphabetically-first documenting path (deterministic).+ first_path = min(t[1] for t in group)+ return (len(group), len(fset), _neg_str(first_path))++ best_set, _ = max(by_set.items(), key=_rank)+ group = sorted(by_set[best_set], key=lambda t: t[1])+ fields = group[0][0]+ sources = [{"file": rel, "line": line, "kind": "docstring"} for _f, rel, line in group]+ # Corroborating access sites (same-named fields) strengthen the cite+ # trail without changing the authoritative doc field set.+ for src in access_sources:+ if any(src["file"] == s["file"] for s in sources):+ continue+ sources.append(src)+ return {+ "identifier": identifier,+ "fields": fields,+ "grounding": "docstring",+ "confidence": "high",+ "sources": sources,+ }++ # No documented shape - fall back to consistent key accesses.+ if len(access_fields) >= _DATA_SHAPE_MIN_FIELDS:+ return {+ "identifier": identifier,+ "fields": access_fields,+ "grounding": "access",+ "confidence": "medium",+ "sources": access_sources,+ }++ return Nonediff --git a/tests/unit/server/mcp/test_answer_data_shape.py b/tests/unit/server/mcp/test_answer_data_shape.pynew file mode 100644index 000000000..b38f75ce1--- /dev/null+++ b/tests/unit/server/mcp/test_answer_data_shape.py@@ -0,0 +1,241 @@+"""Data-shape grounding: answer "what fields does X contain" from source.++A data-shape question names a data blob and asks for its field set. Instead of+gating to a best_guesses pointer list (which triggers the agent's Read/get_symbol+drill), the tool mines the field set straight from source: a documented ``{...}``+shape near the identifier (authoritative -> high) or the concrete keys consumers+pull off the parsed value (usage-mined -> medium).++These tests use synthetic repos with varied identifiers and shapes so nothing is+tuned to any one real question. The precision contract is the load-bearing part:+every reported field must be a quoted token from source, and an identifier with+no groundable shape must fall through (return None), never a fabricated field.+"""++from __future__ import annotations++from pathlib import Path++import pytest++from repowise.server.mcp_server.tool_answer.data_shape import (+ _is_data_shape_question,+ mine_data_shape,+)+++def _write(root: Path, rel: str, body: str) -> None:+ p = root / rel+ p.parent.mkdir(parents=True, exist_ok=True)+ p.write_text(body, encoding="utf-8")+++# --- Question detection ---------------------------------------------------+++ "question,ids,expected",+ [+ (+ "What fields does each entry in co_change_partners_json contain?",+ {"co_change_partners_json"},+ True,+ ),+ ("what keys are in the blame_record blob", {"blame_record"}, True),+ ("what columns does the health_metric_row have", {"health_metric_row"}, True),+ ("describe the schema of GitCommitMeta", {"GitCommitMeta"}, True),+ ("what does each record in owner_stats consist of", {"owner_stats"}, True),+ # Mechanism questions carry no shape cue -> not data-shape.+ ("how does co_change_partners_json get populated", {"co_change_partners_json"}, False),+ ("where is the coupling graph assembled", set(), False),+ # A shape cue with no named identifier can't ground -> not data-shape.+ ("what fields are typical in a config", set(), False),+ ],+)+def test_detection(question, ids, expected):+ assert _is_data_shape_question(question, ids) is expected+++# --- Documented shape (authoritative -> high) -----------------------------+++def test_docstring_brace_shape_high_confidence(tmp_path):+ """A {...} field shape in a docstring near the identifier grounds high."""+ _write(+ tmp_path,+ "pkg/models.py",+ '''\+class GitMeta:+ """Row model.++ ``blame_record_json`` is the raw column: a JSON list of+ ``{"author", "commit_sha", "line_count"}`` blame records.+ """++ blame_record_json: str+''',+ )+ out = mine_data_shape(tmp_path, {"blame_record_json"})+ assert out is not None+ assert out["grounding"] == "docstring"+ assert out["confidence"] == "high"+ assert out["fields"] == ["author", "commit_sha", "line_count"]+ assert out["sources"][0]["file"] == "pkg/models.py"+ assert out["sources"][0]["kind"] == "docstring"+++def test_comment_brace_shape(tmp_path):+ """A shape documented in a line comment also grounds (not just docstrings)."""+ _write(+ tmp_path,+ "pkg/store.py",+ """\+def load(raw):+ # owner_stats rows look like {"owner", "files_touched", "last_seen"}+ return parse(raw)+""",+ )+ out = mine_data_shape(tmp_path, {"owner_stats"})+ assert out is not None+ assert out["grounding"] == "docstring"+ assert set(out["fields"]) == {"owner", "files_touched", "last_seen"}+++def test_executable_dict_literal_is_not_authoritative(tmp_path):+ """A bare dict literal in code (no doc context) is not a documented shape.++ It has quoted keys, but it's one construction site, not a declared schema.+ The access-mining path may still pick up keys, but the doc path must not+ mislabel an executable literal as authoritative/high.+ """+ _write(+ tmp_path,+ "pkg/build.py",+ """\+def build(widget_payload, count):+ result = {"kind": widget_payload, "n": count, "ok": True}+ return result+""",+ )+ out = mine_data_shape(tmp_path, {"widget_payload"})+ # Nothing is documented and the identifier is a scalar arg (no key access),+ # so this must fall through rather than claim {kind,n,ok} as its shape.+ assert out is None+++# --- Access-mined shape (usage -> medium) ---------------------------------+++def test_access_mined_shape_medium_when_no_doc(tmp_path):+ """With no documented shape, consistent key accesses ground at medium."""+ _write(+ tmp_path,+ "pkg/consume.py",+ """\+def summarize(partner_records):+ for entry in iter_records(partner_records):+ name = entry.get("file_path")+ weight = entry["co_change_count"]+ stamp = entry.get("last_seen")+ return name, weight, stamp+""",+ )+ out = mine_data_shape(tmp_path, {"partner_records"})+ assert out is not None+ assert out["grounding"] == "access"+ assert out["confidence"] == "medium"+ assert set(out["fields"]) == {"file_path", "co_change_count", "last_seen"}+++def test_direct_subscript_on_identifier(tmp_path):+ """When the identifier IS the dict variable, direct key access grounds it."""+ _write(+ tmp_path,+ "pkg/direct.py",+ """\+def read(config_blob):+ host = config_blob["host"]+ port = config_blob.get("port")+ return host, port+""",+ )+ out = mine_data_shape(tmp_path, {"config_blob"})+ assert out is not None+ assert out["grounding"] == "access"+ assert set(out["fields"]) == {"host", "port"}+++# --- Precision guards -----------------------------------------------------+++def test_no_shape_returns_none(tmp_path):+ """An identifier that names no blob shape falls through (no fabrication)."""+ _write(tmp_path, "pkg/plain.py", "def process_records(x):\n return x + 1\n")+ assert mine_data_shape(tmp_path, {"process_records"}) is None+++def test_single_field_is_too_weak(tmp_path):+ """A one-field shape is below the min-fields floor -> abstain."""+ _write(+ tmp_path,+ "pkg/thin.py",+ """\+def read(thin_blob):+ # thin_blob entries are {"only_field"}+ return thin_blob+""",+ )+ assert mine_data_shape(tmp_path, {"thin_blob"}) is None+++def test_generic_short_identifier_skipped(tmp_path):+ """Short/generic identifiers are not specific enough to ground on."""+ _write(+ tmp_path,+ "pkg/rows.py",+ """\+def go(row):+ a = row.get("x")+ b = row.get("y")+ return a, b+""",+ )+ # "row" is below the specificity floor (too short, no underscore/camel).+ assert mine_data_shape(tmp_path, {"row"}) is None+++def test_absent_identifier_returns_none(tmp_path):+ """An identifier that appears nowhere in source grounds nothing."""+ _write(tmp_path, "pkg/a.py", "x = 1\n")+ assert mine_data_shape(tmp_path, {"nonexistent_blob_xyz"}) is None+++def test_doc_beats_access(tmp_path):+ """When both a doc shape and accesses exist, the doc shape wins (high)."""+ _write(+ tmp_path,+ "pkg/both.py",+ '''\+class M:+ """``event_payload_json`` is a list of+ ``{"kind", "ts", "actor"}`` event records."""++ event_payload_json: str+++def consume(event_payload_json):+ for e in parse(event_payload_json):+ k = e.get("kind")+ t = e.get("ts")+ return k, t+''',+ )+ out = mine_data_shape(tmp_path, {"event_payload_json"})+ assert out is not None+ assert out["grounding"] == "docstring"+ assert out["confidence"] == "high"+ assert out["fields"] == ["kind", "ts", "actor"]+++def test_none_repo_root_is_safe():+ assert mine_data_shape(None, {"anything_json"}) is None