Turn any codebase, with its docs, SQL schemas, configs, and PDFs, into a queryable knowledge graph. A /graphify skill for Claude Code, Cursor, Codex, and Gemini CLI: local deterministic AST parsing, every edge explained, no vector store.
Python ai-agentsantigravityastclaude-codecode-analysiscode-searchcodexcursordeveloper-toolsgeminigraphifygraphragknowledge-graphleidenllmmcpopenclawragskillstree-sitter
feat: composite surprise score — cross-type, cross-repo, community distance, peripheral→hub
Safi258eff3e9faf
diff --git a/graphify/analyze.py b/graphify/analyze.pyindex c15b16248..cb414dd43 100644--- a/graphify/analyze.py+++ b/graphify/analyze.py@@ -100,21 +100,100 @@ def _is_concept_node(G: nx.Graph, node_id: str) -> bool:return False+_CODE_EXTENSIONS = {"py", "ts", "tsx", "js", "go", "rs", "java", "rb", "cpp", "c", "h", "cs", "kt", "scala", "php"}+_DOC_EXTENSIONS = {"md", "txt", "rst"}+_PAPER_EXTENSIONS = {"pdf"}+_IMAGE_EXTENSIONS = {"png", "jpg", "jpeg", "webp", "gif", "svg"}+++def _file_category(path: str) -> str:+ ext = path.rsplit(".", 1)[-1].lower() if "." in path else ""+ if ext in _CODE_EXTENSIONS:+ return "code"+ if ext in _PAPER_EXTENSIONS:+ return "paper"+ if ext in _IMAGE_EXTENSIONS:+ return "image"+ return "doc"+++def _top_level_dir(path: str) -> str:+ """Return the first path component — used to detect cross-repo edges."""+ return path.split("/")[0] if "/" in path else path+++def _surprise_score(+ G: nx.Graph,+ u: str,+ v: str,+ data: dict,+ node_community: dict[str, int],+ u_source: str,+ v_source: str,+) -> tuple[int, list[str]]:+ """Score how surprising a cross-file edge is. Returns (score, reasons)."""+ score = 0+ reasons: list[str] = []++ # 1. Confidence weight — uncertain connections are more noteworthy+ conf = data.get("confidence", "EXTRACTED")+ conf_bonus = {"AMBIGUOUS": 3, "INFERRED": 2, "EXTRACTED": 1}.get(conf, 1)+ score += conf_bonus+ if conf in ("AMBIGUOUS", "INFERRED"):+ reasons.append(f"{conf.lower()} connection — not explicitly stated in source")++ # 2. Cross file-type bonus — code↔paper or code↔image is non-obvious+ cat_u = _file_category(u_source)+ cat_v = _file_category(v_source)+ if cat_u != cat_v:+ score += 2+ reasons.append(f"crosses file types ({cat_u} ↔ {cat_v})")++ # 3. Cross-repo bonus — different top-level directory+ if _top_level_dir(u_source) != _top_level_dir(v_source):+ score += 2+ reasons.append("connects across different repos/directories")++ # 4. Cross-community bonus — Leiden says these are structurally distant+ cid_u = node_community.get(u)+ cid_v = node_community.get(v)+ if cid_u is not None and cid_v is not None and cid_u != cid_v:+ score += 1+ reasons.append("bridges separate communities")++ # 5. Peripheral→hub: a low-degree node connecting to a high-degree one+ deg_u = G.degree(u)+ deg_v = G.degree(v)+ if min(deg_u, deg_v) <= 2 and max(deg_u, deg_v) >= 5:+ score += 1+ peripheral = G.nodes[u].get("label", u) if deg_u <= 2 else G.nodes[v].get("label", v)+ hub = G.nodes[v].get("label", v) if deg_u <= 2 else G.nodes[u].get("label", u)+ reasons.append(f"peripheral node `{peripheral}` unexpectedly reaches hub `{hub}`")++ return score, reasons++def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: int) -> list[dict]:"""- Cross-file edges between real code/doc entities.- Excludes concept nodes, file hub nodes, and plain import edges.- Sorted AMBIGUOUS → INFERRED → EXTRACTED.+ Cross-file edges between real code/doc entities, ranked by a composite+ surprise score rather than confidence alone.++ Surprise score accounts for:+ - Confidence (AMBIGUOUS > INFERRED > EXTRACTED)+ - Cross file-type (code↔paper is more surprising than code↔code)+ - Cross-repo (different top-level directory)+ - Cross-community (Leiden says structurally distant)+ - Peripheral→hub (low-degree node reaching a god node)++ Each result includes a 'why' field explaining what makes it non-obvious."""- surprises = []- order = {"AMBIGUOUS": 0, "INFERRED": 1, "EXTRACTED": 2}+ node_community = {n: cid for cid, nodes in communities.items() for n in nodes}+ candidates = []for u, v, data in G.edges(data=True):- # Skip structural scaffolding — not insightsrelation = data.get("relation", "")if relation in ("imports", "imports_from", "contains", "method"):continue- # Skip if either endpoint is a concept or file-level nodeif _is_concept_node(G, u) or _is_concept_node(G, v):continueif _is_file_node(G, u) or _is_file_node(G, v):@@ -123,28 +202,32 @@ def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n:u_source = G.nodes[u].get("source_file", "")v_source = G.nodes[v].get("source_file", "")- if u_source and v_source and u_source != v_source:- # Respect original edge direction stored in _src/_tgt (if present),- # otherwise fall back to u/v which may be in arbitrary order.- src_id = data.get("_src", u)- tgt_id = data.get("_tgt", v)- surprises.append({- "source": G.nodes[src_id].get("label", src_id),- "target": G.nodes[tgt_id].get("label", tgt_id),- "source_files": [- G.nodes[src_id].get("source_file", ""),- G.nodes[tgt_id].get("source_file", ""),- ],- "confidence": data.get("confidence", "EXTRACTED"),- "relation": relation,- })+ if not u_source or not v_source or u_source == v_source:+ continue- surprises.sort(key=lambda x: order.get(x["confidence"], 3))- if surprises:- return surprises[:top_n]+ score, reasons = _surprise_score(G, u, v, data, node_community, u_source, v_source)+ src_id = data.get("_src", u)+ tgt_id = data.get("_tgt", v)+ candidates.append({+ "_score": score,+ "source": G.nodes[src_id].get("label", src_id),+ "target": G.nodes[tgt_id].get("label", tgt_id),+ "source_files": [+ G.nodes[src_id].get("source_file", ""),+ G.nodes[tgt_id].get("source_file", ""),+ ],+ "confidence": data.get("confidence", "EXTRACTED"),+ "relation": relation,+ "why": "; ".join(reasons) if reasons else "cross-file semantic connection",+ })++ candidates.sort(key=lambda x: x["_score"], reverse=True)+ for c in candidates:+ c.pop("_score")++ if candidates:+ return candidates[:top_n]- # Fallback: no semantic cross-file edges found (pure AST corpus).- # Surface cross-community bridge edges as structural surprises instead.return _cross_community_surprises(G, communities, top_n)diff --git a/tests/test_analyze.py b/tests/test_analyze.pyindex 420a84f5d..0ae2ede54 100644--- a/tests/test_analyze.py+++ b/tests/test_analyze.py@@ -4,7 +4,7 @@ import networkx as nxfrom pathlib import Pathfrom graphify.build import build_from_jsonfrom graphify.cluster import cluster-from graphify.analyze import god_nodes, surprising_connections, _is_concept_node, graph_diff+from graphify.analyze import god_nodes, surprising_connections, _is_concept_node, graph_diff, _surprise_score, _file_categoryFIXTURES = Path(__file__).parent / "fixtures"@@ -85,14 +85,58 @@ def test_surprising_connections_single_file_uses_community_bridges():assert len(surprises) > 0-def test_surprising_connections_ambiguous_first():+def test_surprising_connections_ambiguous_scores_higher_than_extracted():+ """AMBIGUOUS edge should score higher than an otherwise identical EXTRACTED edge."""+ G = nx.Graph()+ for nid, label, src in [+ ("a", "Alpha", "repo1/model.py"),+ ("b", "Beta", "repo2/train.py"),+ ("c", "Gamma", "repo1/data.py"),+ ("d", "Delta", "repo2/eval.py"),+ ]:+ G.add_node(nid, label=label, source_file=src, file_type="code")+ G.add_edge("a", "b", relation="calls", confidence="AMBIGUOUS", weight=1.0, source_file="repo1/model.py")+ G.add_edge("c", "d", relation="calls", confidence="EXTRACTED", weight=1.0, source_file="repo1/data.py")+ communities = {0: ["a", "c"], 1: ["b", "d"]}+ nc = {"a": 0, "c": 0, "b": 1, "d": 1}+ score_amb, _ = _surprise_score(G, "a", "b", G.edges["a", "b"], nc, "repo1/model.py", "repo2/train.py")+ score_ext, _ = _surprise_score(G, "c", "d", G.edges["c", "d"], nc, "repo1/data.py", "repo2/eval.py")+ assert score_amb > score_ext+++def test_surprising_connections_cross_type_scores_higher():+ """Code↔paper edge should score higher than code↔code edge."""+ G = nx.Graph()+ for nid, label, src in [+ ("a", "Transformer", "code/model.py"),+ ("b", "FlashAttn", "papers/flash.pdf"),+ ("c", "Trainer", "code/train.py"),+ ("d", "Dataset", "code/data.py"),+ ]:+ G.add_node(nid, label=label, source_file=src, file_type="code")+ G.add_edge("a", "b", relation="references", confidence="EXTRACTED", weight=1.0, source_file="code/model.py")+ G.add_edge("c", "d", relation="calls", confidence="EXTRACTED", weight=1.0, source_file="code/train.py")+ nc = {"a": 0, "b": 1, "c": 0, "d": 0}+ score_cross, reasons_cross = _surprise_score(G, "a", "b", G.edges["a", "b"], nc, "code/model.py", "papers/flash.pdf")+ score_same, _ = _surprise_score(G, "c", "d", G.edges["c", "d"], nc, "code/train.py", "code/data.py")+ assert score_cross > score_same+ assert any("code" in r and "paper" in r for r in reasons_cross)+++def test_surprising_connections_have_why_field():G = make_graph()communities = cluster(G)- surprises = surprising_connections(G, communities)- if len(surprises) >= 2:- order = {"AMBIGUOUS": 0, "INFERRED": 1, "EXTRACTED": 2}- confidences = [order[s["confidence"]] for s in surprises]- assert confidences == sorted(confidences)+ for s in surprising_connections(G, communities):+ assert "why" in s+ assert isinstance(s["why"], str)+ assert len(s["why"]) > 0+++def test_file_category():+ assert _file_category("model.py") == "code"+ assert _file_category("flash.pdf") == "paper"+ assert _file_category("diagram.png") == "image"+ assert _file_category("notes.md") == "doc"def test_is_concept_node_empty_source():