Turn any codebase, with its docs, SQL schemas, configs, and PDFs, into a queryable knowledge graph. A /graphify skill for Claude Code, Cursor, Codex, and Gemini CLI: local deterministic AST parsing, every edge explained, no vector store.
Python ai-agentsantigravityastclaude-codecode-analysiscode-searchcodexcursordeveloper-toolsgeminigraphifygraphragknowledge-graphleidenllmmcpopenclawragskillstree-sitter
feat: core pipeline
Safi77966a762257
diff --git a/.gitignore b/.gitignorenew file mode 100644index 000000000..64cb3e032--- /dev/null+++ b/.gitignore@@ -0,0 +1,9 @@+venv/+__pycache__/+*.pyc+*.egg-info/+.eggs/+dist/+build/+.pytest_cache/+*.sodiff --git a/README.md b/README.mdnew file mode 100644index 000000000..dfc97f265--- /dev/null+++ b/README.md@@ -0,0 +1,65 @@+# graphify++Any input → knowledge graph → clustered communities → interactive HTML + GraphRAG-ready JSON + audit report.++```+┌──────────────────┐ ┌────────────────────────────────────────┐+│ │ │ .graphify/ │+│ /graphify ./raw │ ───▶ │ ├── GRAPH_REPORT.md # primary │+│ │ │ ├── graph.html # interactive │+│ │ │ └── graph.json # GraphRAG-ready│+└──────────────────┘ └────────────────────────────────────────┘+```++## Why this exists++Every other graph tool handles codebases only, builds edges silently (you can't tell what was extracted vs invented), and gives you a graph with no explanation of what it means.++graphify handles any input, tags every edge `[EXTRACTED]`, `[INFERRED]`, or `[AMBIGUOUS]`, scores cluster quality as a plain number (not an emoji), and tells you when your corpus is small enough that you don't need a graph at all.++## Install++```bash+npx skills add safishamsi/graphify/skills/graphify+```++## Usage++```bash+/graphify ./raw # full pipeline+/graphify ./my-repo --mode deep # thorough extraction+/graphify ./docs --no-viz # skip HTML+/graphify ./raw --neo4j # also export Cypher for Neo4j+/graphify query "what connects auth to the database?"+```++Works with any mix of file types:+- `.py / .ts / .js / .go` etc → code (AST + semantic)+- `.md / .txt / .rst` → documents+- `.pdf` → papers (with citation mining)++## What you get++```+.graphify/+├── GRAPH_REPORT.md # Corpus check · God nodes · Surprising connections ·+│ # Community summaries with cohesion scores · Ambiguous edges+├── graph.html # Interactive pyvis — color by community, hover for edge type+└── graph.json # NetworkX node-link format, compatible with MS GraphRAG+```++## What this will NOT do++- Won't guarantee extraction correctness — `[AMBIGUOUS]` edges are yours to review+- Won't claim the graph is useful when it isn't — corpus < 50K words gets a warning+- Won't connect to external services unless you pass `--neo4j`+- Won't visualize graphs > 5,000 nodes — use `--no-viz` at that scale++## Design principles++Informed by Karpathy's /raw folder workflow and his observation that most RAG infrastructure is overkill. The graph earns its complexity.++1. Extraction quality is everything — clustering is downstream of it+2. Show the numbers — cohesion is 0.91, not "good"+3. The best output is what you didn't know — Surprising Connections is not optional+4. Token cost is always visiblediff --git a/requirements.txt b/requirements.txtnew file mode 100644index 000000000..d7595b6fb--- /dev/null+++ b/requirements.txt@@ -0,0 +1,10 @@+networkx>=3.3+graspologic>=3.3+pyvis>=0.3.2+tree-sitter>=0.23.0+tree-sitter-python>=0.23.0+tree-sitter-javascript>=0.23.0+tree-sitter-typescript>=0.23.2+pytest>=8.0+pytest-cov>=5.0+pypdf>=4.0diff --git a/setup.py b/setup.pynew file mode 100644index 000000000..f26d62058--- /dev/null+++ b/setup.py@@ -0,0 +1,2 @@+from setuptools import setup, find_packages+setup(name="graphify", packages=find_packages(where="src"), package_dir={"": "src"})diff --git a/skills/graphify/skill.md b/skills/graphify/skill.mdnew file mode 100644index 000000000..ef1040a26--- /dev/null+++ b/skills/graphify/skill.md@@ -0,0 +1,250 @@+---+name: graphify+description: Any input (code, docs, papers, notes) → knowledge graph → clustered communities → interactive HTML + GraphRAG-ready JSON + audit report+trigger: /graphify+---++# /graphify++Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md.++## Usage++```+/graphify <path> # full pipeline+/graphify <path> --mode deep # thorough extraction, richer relationships+/graphify <path> --cluster-only # rerun clustering on existing .graphify/graph.json+/graphify <path> --no-viz # skip HTML, just files + report+/graphify <path> --neo4j # also generate .graphify/cypher.txt+/graphify query "<question>" # ask a question against an existing graph+```++## What You Must Do When Invoked++Follow these steps in order. Do not skip steps.++### Step 1 — Install dependencies++```bash+pip install networkx graspologic pyvis tree-sitter tree-sitter-python tree-sitter-javascript -q+pip install -e . -q 2>/dev/null || true+```++### Step 2 — Detect files++Run the detector and save results:++```bash+python -c "+import sys, json+sys.path.insert(0, 'src')+from graphify.detector import detect+from pathlib import Path+result = detect(Path('INPUT_PATH'))+print(json.dumps(result, indent=2))+" > .graphify_detect.json+cat .graphify_detect.json+```++Replace INPUT_PATH with the actual path the user provided.++If the `warning` field is not null, show it to the user and ask:+> "⚠ [warning]. Continue anyway? (y/n)"+If they say no, stop and clean up.++### Step 3 — Extract entities and relationships++Read every file listed in `.graphify_detect.json`. For each file extract nodes and edges.++**Confidence rules:**+- `EXTRACTED` — relationship is explicit in the source (import, calls, see §3.2, citation)+- `INFERRED` — reasonable inference you are making (two functions sharing data structure)+- `AMBIGUOUS` — you are not sure; flag for user review++**For code files**: Identify classes, functions, modules. Structural edges (imports, calls, class membership) are EXTRACTED. Semantic/conceptual edges are INFERRED.++**For document/paper files**: Extract concepts, named entities, citations. Citation edges are EXTRACTED. Conceptual similarity edges are INFERRED.++As you read and extract from each file, print progress so the user knows what's happening:+- Before reading each file: print `[1/N] Extracting: filename`+- After finishing all files: print `Extraction complete — N nodes, M edges found`++Output `.graphify_extract.json` in this schema:+```json+{+ "nodes": [+ {+ "id": "file_stem_entity_name",+ "label": "Human Readable Name",+ "file_type": "code|document|paper",+ "source_file": "relative/path/to/file",+ "source_location": "L42 or §3.1 or null"+ }+ ],+ "edges": [+ {+ "source": "node_id",+ "target": "node_id",+ "relation": "imports|calls|implements|references|cites|conceptually_related_to",+ "confidence": "EXTRACTED|INFERRED|AMBIGUOUS",+ "source_file": "relative/path/to/file",+ "source_location": "L42 or §3.1 or null",+ "weight": 1.0+ }+ ],+ "input_tokens": 0,+ "output_tokens": 0+}+```++Write this JSON to `.graphify_extract.json`. Set `input_tokens` and `output_tokens` to your actual token usage.++### Step 4 — Build graph, cluster, analyze, generate outputs++```bash+mkdir -p .graphify+python -c "+import sys, json+sys.path.insert(0, 'src')+from graphify.graph_builder import build_from_json, total_tokens+from graphify.clusterer import cluster, score_all+from graphify.analyzer import god_nodes, surprising_connections+from graphify.reporter import generate+from graphify.exporter import to_json, to_cypher+from pathlib import Path++extraction = json.loads(Path('.graphify_extract.json').read_text())+detection = json.loads(Path('.graphify_detect.json').read_text())++G = build_from_json(extraction)+communities = cluster(G)+cohesion = score_all(G, communities)+tokens = {'input': extraction['input_tokens'], 'output': extraction['output_tokens']}+gods = god_nodes(G)+surprises = surprising_connections(G)+labels = {cid: 'Community ' + str(cid) for cid in communities}++report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH')+Path('.graphify/GRAPH_REPORT.md').write_text(report)+to_json(G, communities, '.graphify/graph.json')++analysis = {'communities': {str(k): v for k, v in communities.items()}, 'cohesion': cohesion, 'gods': gods, 'surprises': surprises}+Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2))+print('Pipeline complete')+"+```++Replace INPUT_PATH with the actual path.++### Step 5 — Label communities++Read `.graphify_analysis.json`. For each community, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading").++Rewrite `.graphify/GRAPH_REPORT.md`, replacing `"Community N"` placeholders with the real names you chose.++### Step 6 — Generate visualization (skip if --no-viz)++```bash+python -c "+import sys, json+sys.path.insert(0, 'src')+from graphify.graph_builder import build_from_json+from graphify.clusterer import cluster+from graphify.visualizer import generate_html+from pathlib import Path++extraction = json.loads(Path('.graphify_extract.json').read_text())+G = build_from_json(extraction)+communities = cluster(G)+generate_html(G, communities, '.graphify/graph.html')+print('graph.html written')+"+```++### Step 7 — Neo4j export (only if --neo4j flag)++```bash+python -c "+import sys, json+sys.path.insert(0, 'src')+from graphify.graph_builder import build_from_json+from graphify.exporter import to_cypher+from pathlib import Path++G = __import__('graphify.graph_builder', fromlist=['build_from_json']).build_from_json(+ json.loads(Path('.graphify_extract.json').read_text()))+to_cypher(G, '.graphify/cypher.txt')+print('cypher.txt written')+"+```++### Step 8 — Clean up and report++```bash+rm -f .graphify_detect.json .graphify_extract.json .graphify_analysis.json+```++Tell the user:+```+Graph complete. Outputs in .graphify/++ GRAPH_REPORT.md — audit trail, clusters, surprising connections+ graph.html — open in browser to explore interactively+ graph.json — GraphRAG-ready, compatible with Microsoft GraphRAG++[paste God Nodes section from GRAPH_REPORT.md]+[paste Surprising Connections section from GRAPH_REPORT.md]+```++## For --cluster-only++Skip Steps 1–3. Load the existing graph from `.graphify/graph.json` and run Steps 4–8:++```bash+python -c "+import sys, json+sys.path.insert(0, 'src')+from graphify.graph_builder import build_from_json+from graphify.clusterer import cluster, score_all+from graphify.analyzer import god_nodes, surprising_connections+from graphify.reporter import generate+from graphify.exporter import to_json+from networkx.readwrite import json_graph+import networkx as nx+from pathlib import Path+import os++# Load existing graph+data = json.loads(Path('.graphify/graph.json').read_text())+G = json_graph.node_link_graph(data, edges='links')++# Rebuild a minimal detection result from graph metadata+total_nodes = G.number_of_nodes()+detection = {'total_files': total_nodes, 'total_words': 99999, 'needs_graph': True, 'warning': None}+tokens = {'input': 0, 'output': 0}++communities = cluster(G)+cohesion = score_all(G, communities)+gods = god_nodes(G)+surprises = surprising_connections(G)+labels = {cid: 'Community ' + str(cid) for cid in communities}++report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH')+Path('.graphify/GRAPH_REPORT.md').write_text(report)+to_json(G, communities, '.graphify/graph.json')+print('Re-clustered successfully')+"+```++Then run Steps 5–8 as normal (label communities, generate viz, clean up, report).++## For /graphify query++Load `.graphify/graph.json`. Find the nodes and edges most relevant to the question using BFS from the most relevant starting node. Answer using only what the graph contains — do not hallucinate edges. If the graph lacks enough information, say so.++## Honesty Rules++- Never invent an edge. If unsure, use AMBIGUOUS.+- Never skip the corpus check warning.+- Always show token cost in the report.+- Never hide cohesion scores behind symbols — show the raw number.diff --git a/src/graphify/__init__.py b/src/graphify/__init__.pynew file mode 100644index 000000000..e9852dcf1--- /dev/null+++ b/src/graphify/__init__.py@@ -0,0 +1 @@+"""graphify — any input → knowledge graph → clustered communities → audit report."""diff --git a/src/graphify/analyzer.py b/src/graphify/analyzer.pynew file mode 100644index 000000000..064681cbb--- /dev/null+++ b/src/graphify/analyzer.py@@ -0,0 +1,158 @@+from __future__ import annotations+import networkx as nx+++def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]:+ """Return the top_n most-connected nodes — the core abstractions."""+ degree = dict(G.degree())+ sorted_nodes = sorted(degree.items(), key=lambda x: x[1], reverse=True)[:top_n]+ return [+ {+ "id": node_id,+ "label": G.nodes[node_id].get("label", node_id),+ "edges": deg,+ }+ for node_id, deg in sorted_nodes+ ]+++def surprising_connections(+ G: nx.Graph,+ communities: dict[int, list[str]] | None = None,+ top_n: int = 5,+) -> list[dict]:+ """+ Find connections that are genuinely surprising — not obvious from file structure.++ Strategy:+ - Multi-file corpora: cross-file edges between real entities (not concept nodes).+ Sorted AMBIGUOUS → INFERRED → EXTRACTED.+ - Single-file / single-source corpora: cross-community edges that bridge+ distant parts of the graph (betweenness centrality on edges).+ These reveal non-obvious structural couplings.++ Concept nodes (empty source_file, or injected semantic annotations) are excluded+ from surprising connections because they are intentional, not discovered.+ """+ # Identify unique source files (ignore empty/null source_file)+ source_files = {+ data.get("source_file", "")+ for _, data in G.nodes(data=True)+ if data.get("source_file", "")+ }+ is_multi_source = len(source_files) > 1++ if is_multi_source:+ return _cross_file_surprises(G, top_n)+ else:+ return _cross_community_surprises(G, communities or {}, top_n)+++def _is_concept_node(G: nx.Graph, node_id: str) -> bool:+ """+ Return True if this node is a manually-injected semantic concept node+ rather than a real entity found in source code.++ Signals:+ - Empty source_file+ - source_file doesn't look like a real file path (no extension)+ """+ data = G.nodes[node_id]+ source = data.get("source_file", "")+ if not source:+ return True+ # Has no file extension → probably a concept label, not a real file+ if "." not in source.split("/")[-1]:+ return True+ return False+++def _cross_file_surprises(G: nx.Graph, top_n: int) -> list[dict]:+ """+ Cross-file edges between real code/doc entities.+ Excludes concept nodes. Sorted AMBIGUOUS first.+ """+ surprises = []+ order = {"AMBIGUOUS": 0, "INFERRED": 1, "EXTRACTED": 2}++ for u, v, data in G.edges(data=True):+ # Skip if either endpoint is a concept node+ if _is_concept_node(G, u) or _is_concept_node(G, v):+ continue++ u_source = G.nodes[u].get("source_file", "")+ v_source = G.nodes[v].get("source_file", "")++ if u_source and v_source and u_source != v_source:+ surprises.append({+ "source": G.nodes[u].get("label", u),+ "target": G.nodes[v].get("label", v),+ "source_files": [u_source, v_source],+ "confidence": data.get("confidence", "EXTRACTED"),+ "relation": data.get("relation", ""),+ })++ surprises.sort(key=lambda x: order.get(x["confidence"], 3))+ return surprises[:top_n]+++def _cross_community_surprises(+ G: nx.Graph,+ communities: dict[int, list[str]],+ top_n: int,+) -> list[dict]:+ """+ For single-source corpora: find edges that bridge different communities.+ These are surprising because Leiden grouped everything else tightly —+ these edges cut across the natural structure.++ Falls back to high-betweenness edges if no community info is provided.+ """+ if not communities:+ # No community info — use edge betweenness centrality+ if G.number_of_edges() == 0:+ return []+ betweenness = nx.edge_betweenness_centrality(G)+ top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n]+ result = []+ for (u, v), score in top_edges:+ data = G.edges[u, v]+ result.append({+ "source": G.nodes[u].get("label", u),+ "target": G.nodes[v].get("label", v),+ "source_files": [+ G.nodes[u].get("source_file", ""),+ G.nodes[v].get("source_file", ""),+ ],+ "confidence": data.get("confidence", "EXTRACTED"),+ "relation": data.get("relation", ""),+ "note": f"Bridges graph structure (betweenness={score:.3f})",+ })+ return result++ # Build node → community map+ node_community = {n: cid for cid, nodes in communities.items() for n in nodes}++ surprises = []+ for u, v, data in G.edges(data=True):+ cid_u = node_community.get(u)+ cid_v = node_community.get(v)+ if cid_u is not None and cid_v is not None and cid_u != cid_v:+ # This edge crosses community boundaries — interesting+ confidence = data.get("confidence", "EXTRACTED")+ surprises.append({+ "source": G.nodes[u].get("label", u),+ "target": G.nodes[v].get("label", v),+ "source_files": [+ G.nodes[u].get("source_file", ""),+ G.nodes[v].get("source_file", ""),+ ],+ "confidence": confidence,+ "relation": data.get("relation", ""),+ "note": f"Bridges community {cid_u} → community {cid_v}",+ })++ # Sort: AMBIGUOUS first, then INFERRED, then EXTRACTED+ order = {"AMBIGUOUS": 0, "INFERRED": 1, "EXTRACTED": 2}+ surprises.sort(key=lambda x: order.get(x["confidence"], 3))+ return surprises[:top_n]diff --git a/src/graphify/ast_extractor.py b/src/graphify/ast_extractor.pynew file mode 100644index 000000000..da3c13958--- /dev/null+++ b/src/graphify/ast_extractor.py@@ -0,0 +1,190 @@+"""+Deterministic structural extraction from Python code using tree-sitter.+Outputs JSON nodes+edges compatible with the graphify extraction schema.++Usage:+ python -m graphify.ast_extractor file1.py [file2.py ...]+ python -m graphify.ast_extractor ./src/+"""+from __future__ import annotations+import json+import re+import sys+from pathlib import Path+++def _make_id(*parts: str) -> str:+ """Build a stable node ID from one or more name parts."""+ combined = "_".join(p.strip("_.") for p in parts if p)+ cleaned = re.sub(r"[^a-zA-Z0-9]+", "_", combined)+ return cleaned.strip("_").lower()+++def extract_python(path: Path) -> dict:+ """Extract classes, functions, and imports from a .py file via tree-sitter AST."""+ try:+ import tree_sitter_python as tspython+ from tree_sitter import Language, Parser+ except ImportError:+ return {"nodes": [], "edges": [], "error": "tree-sitter-python not installed"}++ try:+ language = Language(tspython.language())+ parser = Parser(language)+ source = path.read_bytes()+ tree = parser.parse(source)+ root = tree.root_node+ except Exception as e:+ return {"nodes": [], "edges": [], "error": str(e)}++ stem = path.stem+ str_path = str(path)+ nodes: list[dict] = []+ edges: list[dict] = []+ seen_ids: set[str] = set()++ def add_node(nid: str, label: str, line: int) -> None:+ if nid not in seen_ids:+ seen_ids.add(nid)+ nodes.append({+ "id": nid,+ "label": label,+ "file_type": "code",+ "source_file": str_path,+ "source_location": f"L{line}",+ })++ def add_edge(src: str, tgt: str, relation: str, line: int) -> None:+ # Only add edge if both endpoints exist or src is the file node+ edges.append({+ "source": src,+ "target": tgt,+ "relation": relation,+ "confidence": "EXTRACTED",+ "source_file": str_path,+ "source_location": f"L{line}",+ "weight": 1.0,+ })++ # File-level node — stable ID based on stem only+ file_nid = _make_id(stem)+ add_node(file_nid, path.name, 1)++ def walk(node, parent_class_nid: str | None = None) -> None:+ t = node.type++ if t == "import_statement":+ for child in node.children:+ if child.type in ("dotted_name", "aliased_import"):+ raw = source[child.start_byte:child.end_byte].decode()+ module_name = raw.split(" as ")[0].strip().lstrip(".")+ tgt_nid = _make_id(module_name)+ add_edge(file_nid, tgt_nid, "imports", node.start_point[0] + 1)+ return++ if t == "import_from_statement":+ module_node = node.child_by_field_name("module_name")+ if module_node:+ raw = source[module_node.start_byte:module_node.end_byte].decode().lstrip(".")+ tgt_nid = _make_id(raw)+ add_edge(file_nid, tgt_nid, "imports_from", node.start_point[0] + 1)+ return++ if t == "class_definition":+ name_node = node.child_by_field_name("name")+ if not name_node:+ return+ class_name = source[name_node.start_byte:name_node.end_byte].decode()+ class_nid = _make_id(stem, class_name)+ line = node.start_point[0] + 1+ add_node(class_nid, class_name, line)+ add_edge(file_nid, class_nid, "contains", line)++ # Inheritance+ args = node.child_by_field_name("superclasses")+ if args:+ for arg in args.children:+ if arg.type == "identifier":+ base = source[arg.start_byte:arg.end_byte].decode()+ base_nid = _make_id(stem, base)+ add_edge(class_nid, base_nid, "inherits", line)++ # Walk class body for methods+ body = node.child_by_field_name("body")+ if body:+ for child in body.children:+ walk(child, parent_class_nid=class_nid)+ return++ if t == "function_definition":+ name_node = node.child_by_field_name("name")+ if not name_node:+ return+ func_name = source[name_node.start_byte:name_node.end_byte].decode()+ line = node.start_point[0] + 1+ if parent_class_nid:+ func_nid = _make_id(parent_class_nid, func_name)+ add_node(func_nid, f".{func_name}()", line)+ add_edge(parent_class_nid, func_nid, "method", line)+ else:+ func_nid = _make_id(stem, func_name)+ add_node(func_nid, f"{func_name}()", line)+ add_edge(file_nid, func_nid, "contains", line)+ return++ for child in node.children:+ walk(child, parent_class_nid=None)++ walk(root)++ # Post-process: remove edges whose source or target was never added as a node+ # (dangling import edges pointing to external libraries are fine to keep,+ # but edges between internal entities must be valid)+ valid_ids = seen_ids+ clean_edges = []+ for edge in edges:+ src, tgt = edge["source"], edge["target"]+ # Keep if both endpoints are known, OR if it's an import edge (tgt may be external)+ if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")):+ clean_edges.append(edge)++ return {"nodes": nodes, "edges": clean_edges}+++def extract(paths: list[Path]) -> dict:+ """Extract AST nodes and edges from a list of code files."""+ all_nodes: list[dict] = []+ all_edges: list[dict] = []++ for path in paths:+ if path.suffix == ".py":+ result = extract_python(path)+ all_nodes.extend(result.get("nodes", []))+ all_edges.extend(result.get("edges", []))++ return {+ "nodes": all_nodes,+ "edges": all_edges,+ "input_tokens": 0,+ "output_tokens": 0,+ }+++def collect_files(target: Path) -> list[Path]:+ if target.is_file():+ return [target]+ return sorted(p for p in target.rglob("*.py")+ if not any(part.startswith(".") for part in p.parts))+++if __name__ == "__main__":+ if len(sys.argv) < 2:+ print("Usage: python -m graphify.ast_extractor <file_or_dir> ...", file=sys.stderr)+ sys.exit(1)++ paths: list[Path] = []+ for arg in sys.argv[1:]:+ paths.extend(collect_files(Path(arg)))++ result = extract(paths)+ print(json.dumps(result, indent=2))diff --git a/src/graphify/clusterer.py b/src/graphify/clusterer.pynew file mode 100644index 000000000..813fe63da--- /dev/null+++ b/src/graphify/clusterer.py@@ -0,0 +1,71 @@+from __future__ import annotations+import networkx as nx+from graspologic.partition import leiden++_MAX_COMMUNITY_FRACTION = 0.25 # communities larger than 25% of graph get split+_MIN_SPLIT_SIZE = 10 # only split if community has at least this many nodes+++def cluster(G: nx.Graph) -> dict[int, list[str]]:+ """Run Leiden community detection. Returns {community_id: [node_ids]}.++ Community IDs are stable across runs: 0 = largest community after splitting.+ Oversized communities (> 25% of graph nodes, min 10) are split by running+ a second Leiden pass on the subgraph.+ """+ if G.number_of_nodes() == 0:+ return {}+ if G.number_of_edges() == 0:+ return {i: [n] for i, n in enumerate(sorted(G.nodes))}++ partition: dict[str, int] = leiden(G)+ raw: dict[int, list[str]] = {}+ for node, cid in partition.items():+ raw.setdefault(cid, []).append(node)++ # Split oversized communities+ max_size = max(_MIN_SPLIT_SIZE, int(G.number_of_nodes() * _MAX_COMMUNITY_FRACTION))+ final_communities: list[list[str]] = []+ for nodes in raw.values():+ if len(nodes) > max_size:+ final_communities.extend(_split_community(G, nodes))+ else:+ final_communities.append(nodes)++ # Re-index by size descending for deterministic ordering+ final_communities.sort(key=len, reverse=True)+ return {i: sorted(nodes) for i, nodes in enumerate(final_communities)}+++def _split_community(G: nx.Graph, nodes: list[str]) -> list[list[str]]:+ """Run a second Leiden pass on a community subgraph to split it further."""+ subgraph = G.subgraph(nodes)+ if subgraph.number_of_edges() == 0:+ # No edges — split into individual nodes+ return [[n] for n in sorted(nodes)]+ try:+ sub_partition: dict[str, int] = leiden(subgraph)+ sub_communities: dict[int, list[str]] = {}+ for node, cid in sub_partition.items():+ sub_communities.setdefault(cid, []).append(node)+ if len(sub_communities) <= 1:+ # Leiden couldn't split it — return as-is+ return [sorted(nodes)]+ return [sorted(v) for v in sub_communities.values()]+ except Exception:+ return [sorted(nodes)]+++def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float:+ """Ratio of actual intra-community edges to maximum possible."""+ n = len(community_nodes)+ if n <= 1:+ return 1.0+ subgraph = G.subgraph(community_nodes)+ actual = subgraph.number_of_edges()+ possible = n * (n - 1) / 2+ return round(actual / possible, 2) if possible > 0 else 0.0+++def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]:+ return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()}diff --git a/src/graphify/detector.py b/src/graphify/detector.pynew file mode 100644index 000000000..b2f5e5b33--- /dev/null+++ b/src/graphify/detector.py@@ -0,0 +1,108 @@+from __future__ import annotations+import re+from pathlib import Path+from .models import FileType++CODE_EXTENSIONS = {'.py', '.ts', '.js', '.go', '.rs', '.java', '.cpp', '.c', '.rb', '.swift', '.kt'}+DOC_EXTENSIONS = {'.md', '.txt', '.rst'}+PAPER_EXTENSIONS = {'.pdf'}++CORPUS_WARN_THRESHOLD = 50_000 # words++# Signals that a .md/.txt file is actually a converted academic paper+_PAPER_SIGNALS = [+ re.compile(r'\barxiv\b', re.IGNORECASE),+ re.compile(r'\bdoi\s*:', re.IGNORECASE),+ re.compile(r'\babstract\b', re.IGNORECASE),+ re.compile(r'\bproceedings\b', re.IGNORECASE),+ re.compile(r'\bjournal\b', re.IGNORECASE),+ re.compile(r'\bpreprint\b', re.IGNORECASE),+ re.compile(r'\\cite\{'), # LaTeX citation+ re.compile(r'\[\d+\]'), # Numbered citation [1], [23] (inline)+ re.compile(r'\[\n\d+\n\]'), # Numbered citation spread across lines (markdown conversion)+ re.compile(r'eq\.\s*\d+|equation\s+\d+', re.IGNORECASE),+ re.compile(r'\d{4}\.\d{4,5}'), # arXiv ID like 1706.03762+ re.compile(r'\bwe propose\b', re.IGNORECASE), # common academic phrasing+ re.compile(r'\bliterature\b', re.IGNORECASE), # "from the literature"+]+_PAPER_SIGNAL_THRESHOLD = 3 # need at least this many signals to call it a paper+++def _looks_like_paper(path: Path) -> bool:+ """Heuristic: does this text file read like an academic paper?"""+ try:+ # Only scan first 3000 chars for speed+ text = path.read_text(errors="ignore")[:3000]+ hits = sum(1 for pattern in _PAPER_SIGNALS if pattern.search(text))+ return hits >= _PAPER_SIGNAL_THRESHOLD+ except Exception:+ return False+++def classify_file(path: Path) -> FileType | None:+ ext = path.suffix.lower()+ if ext in CODE_EXTENSIONS:+ return FileType.CODE+ if ext in PAPER_EXTENSIONS:+ return FileType.PAPER+ if ext in DOC_EXTENSIONS:+ # Check if it's a converted paper+ if _looks_like_paper(path):+ return FileType.PAPER+ return FileType.DOCUMENT+ return None+++def extract_pdf_text(path: Path) -> str:+ """Extract plain text from a PDF file using pypdf."""+ try:+ from pypdf import PdfReader+ reader = PdfReader(str(path))+ pages = []+ for page in reader.pages:+ text = page.extract_text()+ if text:+ pages.append(text)+ return "\n".join(pages)+ except Exception:+ return ""+++def count_words(path: Path) -> int:+ try:+ if path.suffix.lower() == ".pdf":+ return len(extract_pdf_text(path).split())+ return len(path.read_text(errors="ignore").split())+ except Exception:+ return 0+++def detect(root: Path) -> dict:+ files: dict[FileType, list[str]] = {+ FileType.CODE: [],+ FileType.DOCUMENT: [],+ FileType.PAPER: [],+ }+ total_words = 0++ for p in sorted(root.rglob("*")):+ if not p.is_file():+ continue+ if any(part.startswith(".") for part in p.relative_to(root).parts):+ continue+ ftype = classify_file(p)+ if ftype:+ files[ftype].append(str(p))+ total_words += count_words(p)++ needs_graph = total_words >= CORPUS_WARN_THRESHOLD+ return {+ "files": {k.value: v for k, v in files.items()},+ "total_files": sum(len(v) for v in files.values()),+ "total_words": total_words,+ "needs_graph": needs_graph,+ "warning": None if needs_graph else (+ f"Corpus is ~{total_words:,} words — fits in a single context window. "+ f"You may not need a graph."+ ),+ }diff --git a/src/graphify/exporter.py b/src/graphify/exporter.pynew file mode 100644index 000000000..f1410d0c0--- /dev/null+++ b/src/graphify/exporter.py@@ -0,0 +1,31 @@+from __future__ import annotations+import json+import networkx as nx+from networkx.readwrite import json_graph+++def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str) -> None:+ node_community = {n: cid for cid, nodes in communities.items() for n in nodes}+ data = json_graph.node_link_data(G, edges="links")+ for node in data["nodes"]:+ node["community"] = node_community.get(node["id"])+ with open(output_path, "w") as f:+ json.dump(data, f, indent=2)+++def to_cypher(G: nx.Graph, output_path: str) -> None:+ lines = ["// Neo4j Cypher import — generated by /graphify", ""]+ for node_id, data in G.nodes(data=True):+ label = data.get("label", node_id).replace("'", "\\'")+ ftype = data.get("file_type", "unknown").capitalize()+ lines.append(f"MERGE (n:{ftype} {{id: '{node_id}', label: '{label}'}});")+ lines.append("")+ for u, v, data in G.edges(data=True):+ rel = data.get("relation", "RELATES_TO").upper().replace(" ", "_").replace("-", "_")+ conf = data.get("confidence", "EXTRACTED")+ lines.append(+ f"MATCH (a {{id: '{u}'}}), (b {{id: '{v}'}}) "+ f"MERGE (a)-[:{rel} {{confidence: '{conf}'}}]->(b);"+ )+ with open(output_path, "w") as f:+ f.write("\n".join(lines))diff --git a/src/graphify/graph_builder.py b/src/graphify/graph_builder.pynew file mode 100644index 000000000..95a078573--- /dev/null+++ b/src/graphify/graph_builder.py@@ -0,0 +1,24 @@+from __future__ import annotations+import networkx as nx+++def build_from_json(extraction: dict) -> nx.Graph:+ G = nx.Graph()+ for node in extraction.get("nodes", []):+ G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"})+ for edge in extraction.get("edges", []):+ G.add_edge(+ edge["source"],+ edge["target"],+ **{k: v for k, v in edge.items() if k not in ("source", "target")},+ )+ return G+++def build(extractions: list[dict]) -> nx.Graph:+ """Merge multiple extraction results into one graph."""+ G = nx.Graph()+ for ext in extractions:+ sub = build_from_json(ext)+ G.update(sub)+ return Gdiff --git a/src/graphify/models.py b/src/graphify/models.pynew file mode 100644index 000000000..36d2f3612--- /dev/null+++ b/src/graphify/models.py@@ -0,0 +1,45 @@+from __future__ import annotations+from dataclasses import dataclass, field+from enum import Enum+from typing import Optional+++class Confidence(str, Enum):+ EXTRACTED = "EXTRACTED"+ INFERRED = "INFERRED"+ AMBIGUOUS = "AMBIGUOUS"+++class FileType(str, Enum):+ CODE = "code"+ DOCUMENT = "document"+ PAPER = "paper"+++@dataclass+class GraphNode:+ id: str+ label: str+ file_type: FileType+ source_file: str+ source_location: Optional[str] = None+ community: Optional[int] = None+++@dataclass+class GraphEdge:+ source: str+ target: str+ relation: str+ confidence: Confidence+ source_file: str+ source_location: Optional[str] = None+ weight: float = 1.0+++@dataclass+class ExtractionResult:+ nodes: list[GraphNode] = field(default_factory=list)+ edges: list[GraphEdge] = field(default_factory=list)+ input_tokens: int = 0+ output_tokens: int = 0diff --git a/src/graphify/reporter.py b/src/graphify/reporter.pynew file mode 100644index 000000000..93a5cb895--- /dev/null+++ b/src/graphify/reporter.py@@ -0,0 +1,84 @@+from __future__ import annotations+from datetime import date+import networkx as nx+++def generate(+ G: nx.Graph,+ communities: dict[int, list[str]],+ cohesion_scores: dict[int, float],+ community_labels: dict[int, str],+ god_node_list: list[dict],+ surprise_list: list[dict],+ detection_result: dict,+ token_cost: dict,+ root: str,+) -> str:+ today = date.today().isoformat()++ confidences = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)]+ total = len(confidences) or 1+ ext_pct = round(confidences.count("EXTRACTED") / total * 100)+ inf_pct = round(confidences.count("INFERRED") / total * 100)+ amb_pct = round(confidences.count("AMBIGUOUS") / total * 100)++ lines = [+ f"# Graph Report — {root} ({today})",+ "",+ "## Corpus Check",+ ]+ if detection_result.get("warning"):+ lines.append(f"- {detection_result['warning']}")+ else:+ lines += [+ f"- {detection_result['total_files']} files · ~{detection_result['total_words']:,} words",+ "- Verdict: corpus is large enough that graph structure adds value.",+ ]++ lines += [+ "",+ "## Summary",+ f"- {G.number_of_nodes()} nodes · {G.number_of_edges()} edges · {len(communities)} communities detected",+ f"- Extraction: {ext_pct}% EXTRACTED · {inf_pct}% INFERRED · {amb_pct}% AMBIGUOUS",+ f"- Token cost: {token_cost.get('input', 0):,} input · {token_cost.get('output', 0):,} output",+ "",+ "## God Nodes (most connected — your core abstractions)",+ ]+ for i, node in enumerate(god_node_list, 1):+ lines.append(f"{i}. `{node['label']}` — {node['edges']} edges")++ lines += ["", "## Surprising Connections (you probably didn't know these)"]+ if surprise_list:+ for s in surprise_list:+ lines += [+ f"- `{s['source']}` ↔ `{s['target']}` [{s['confidence']}]",+ f" {s['source_files'][0]} ↔ {s['source_files'][1]}",+ ]+ else:+ lines.append("- None detected — all connections are within the same source files.")++ lines += ["", "## Communities"]+ for cid, nodes in communities.items():+ label = community_labels.get(cid, f"Community {cid}")+ score = cohesion_scores.get(cid, 0.0)+ node_labels = [G.nodes[n].get("label", n) for n in nodes[:8]]+ suffix = "..." if len(nodes) > 8 else ""+ lines += [+ "",+ f"### Community {cid} — \"{label}\"",+ f"Cohesion: {score}",+ f"Nodes ({len(nodes)}): {', '.join(node_labels)}{suffix}",+ ]++ ambiguous = [(u, v, d) for u, v, d in G.edges(data=True) if d.get("confidence") == "AMBIGUOUS"]+ if ambiguous:+ lines += ["", "## Ambiguous Edges — Review These"]+ for u, v, d in ambiguous:+ ul = G.nodes[u].get("label", u)+ vl = G.nodes[v].get("label", v)+ lines += [+ f"- `{ul}` → `{vl}` [AMBIGUOUS]",+ f" {d.get('source_file', '')} · relation: {d.get('relation', 'unknown')}",+ ]++ return "\n".join(lines)diff --git a/src/graphify/validator.py b/src/graphify/validator.pynew file mode 100644index 000000000..606ab3f5f--- /dev/null+++ b/src/graphify/validator.py@@ -0,0 +1,70 @@+from __future__ import annotations++VALID_FILE_TYPES = {"code", "document", "paper"}+VALID_CONFIDENCES = {"EXTRACTED", "INFERRED", "AMBIGUOUS"}+REQUIRED_NODE_FIELDS = {"id", "label", "file_type", "source_file"}+REQUIRED_EDGE_FIELDS = {"source", "target", "relation", "confidence", "source_file"}+++def validate_extraction(data: dict) -> list[str]:+ """+ Validate an extraction JSON dict against the graphify schema.+ Returns a list of error strings — empty list means valid.+ """+ if not isinstance(data, dict):+ return ["Extraction must be a JSON object"]++ errors: list[str] = []++ # Nodes+ if "nodes" not in data:+ errors.append("Missing required key 'nodes'")+ elif not isinstance(data["nodes"], list):+ errors.append("'nodes' must be a list")+ else:+ for i, node in enumerate(data["nodes"]):+ if not isinstance(node, dict):+ errors.append(f"Node {i} must be an object")+ continue+ for field in REQUIRED_NODE_FIELDS:+ if field not in node:+ errors.append(f"Node {i} (id={node.get('id', '?')!r}) missing required field '{field}'")+ if "file_type" in node and node["file_type"] not in VALID_FILE_TYPES:+ errors.append(+ f"Node {i} (id={node.get('id', '?')!r}) has invalid file_type "+ f"'{node['file_type']}' — must be one of {sorted(VALID_FILE_TYPES)}"+ )++ # Edges+ if "edges" not in data:+ errors.append("Missing required key 'edges'")+ elif not isinstance(data["edges"], list):+ errors.append("'edges' must be a list")+ else:+ node_ids = {n["id"] for n in data.get("nodes", []) if isinstance(n, dict) and "id" in n}+ for i, edge in enumerate(data["edges"]):+ if not isinstance(edge, dict):+ errors.append(f"Edge {i} must be an object")+ continue+ for field in REQUIRED_EDGE_FIELDS:+ if field not in edge:+ errors.append(f"Edge {i} missing required field '{field}'")+ if "confidence" in edge and edge["confidence"] not in VALID_CONFIDENCES:+ errors.append(+ f"Edge {i} has invalid confidence '{edge['confidence']}' "+ f"— must be one of {sorted(VALID_CONFIDENCES)}"+ )+ if "source" in edge and node_ids and edge["source"] not in node_ids:+ errors.append(f"Edge {i} source '{edge['source']}' does not match any node id")+ if "target" in edge and node_ids and edge["target"] not in node_ids:+ errors.append(f"Edge {i} target '{edge['target']}' does not match any node id")++ return errors+++def assert_valid(data: dict) -> None:+ """Raise ValueError with all errors if extraction is invalid."""+ errors = validate_extraction(data)+ if errors:+ msg = f"Extraction JSON has {len(errors)} error(s):\n" + "\n".join(f" • {e}" for e in errors)+ raise ValueError(msg)diff --git a/src/graphify/visualizer.py b/src/graphify/visualizer.pynew file mode 100644index 000000000..6942283dc--- /dev/null+++ b/src/graphify/visualizer.py@@ -0,0 +1,79 @@+from __future__ import annotations+from pathlib import Path+import networkx as nx+from pyvis.network import Network++COMMUNITY_COLORS = [+ "#4E79A7", "#F28E2B", "#E15759", "#76B7B2", "#59A14F",+ "#EDC948", "#B07AA1", "#FF9DA7", "#9C755F", "#BAB0AC",+]+MAX_NODES_FOR_VIZ = 5_000+++def generate_html(+ G: nx.Graph,+ communities: dict[int, list[str]],+ output_path: str,+ community_labels: dict[int, str] | None = None,+) -> None:+ if G.number_of_nodes() > MAX_NODES_FOR_VIZ:+ raise ValueError(+ f"Graph has {G.number_of_nodes()} nodes — too large for pyvis. "+ f"Use --no-viz or reduce input size."+ )++ node_community = {n: cid for cid, nodes in communities.items() for n in nodes}++ net = Network(height="800px", width="100%", bgcolor="#1a1a2e", font_color="white")+ net.barnes_hut()++ for node_id, data in G.nodes(data=True):+ cid = node_community.get(node_id, 0)+ color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)]+ net.add_node(+ node_id,+ label=data.get("label", node_id),+ color=color,+ title=(+ f"Source: {data.get('source_file', 'unknown')}\n"+ f"Type: {data.get('file_type', 'unknown')}\n"+ f"Community: {community_labels.get(cid, str(cid)) if community_labels else cid}"+ ),+ )++ for u, v, data in G.edges(data=True):+ confidence = data.get("confidence", "EXTRACTED")+ width = {"EXTRACTED": 2, "INFERRED": 1, "AMBIGUOUS": 1}.get(confidence, 1)+ net.add_edge(+ u, v,+ title=f"{data.get('relation', '')} [{confidence}]",+ width=width,+ dashes=(confidence != "EXTRACTED"),+ )++ net.save_graph(output_path)++ # Inject community legend into saved HTML+ if community_labels:+ legend_items = ""+ for cid in sorted(community_labels.keys()):+ color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)]+ label = community_labels[cid]+ n_nodes = len(communities.get(cid, []))+ legend_items += (+ f'<div style="margin:4px 0">'+ f'<span style="color:{color};font-size:18px">■</span> '+ f'<span style="font-size:13px">{label} ({n_nodes})</span>'+ f'</div>'+ )+ legend_html = (+ '<div style="position:fixed;top:10px;right:10px;background:#2a2a4e;'+ 'padding:12px 16px;border-radius:8px;font-family:sans-serif;color:white;'+ 'z-index:9999;min-width:180px;">'+ '<b style="font-size:14px">Communities</b><br>'+ + legend_items ++ '</div>'+ )+ content = Path(output_path).read_text()+ content = content.replace("</body>", legend_html + "\n</body>")+ Path(output_path).write_text(content)diff --git a/tests/__init__.py b/tests/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/fixtures/extraction.json b/tests/fixtures/extraction.jsonnew file mode 100644index 000000000..40afee0b0--- /dev/null+++ b/tests/fixtures/extraction.json@@ -0,0 +1,16 @@+{+ "nodes": [+ {"id": "n_transformer", "label": "Transformer", "file_type": "code", "source_file": "model.py", "source_location": "L1"},+ {"id": "n_attention", "label": "MultiHeadAttention", "file_type": "code", "source_file": "model.py", "source_location": "L10"},+ {"id": "n_layernorm", "label": "LayerNorm", "file_type": "code", "source_file": "model.py", "source_location": "L20"},+ {"id": "n_concept_attn","label": "attention mechanism", "file_type": "document", "source_file": "paper.md", "source_location": "§3.1"}+ ],+ "edges": [+ {"source": "n_transformer", "target": "n_attention", "relation": "contains", "confidence": "EXTRACTED", "source_file": "model.py", "weight": 1.0},+ {"source": "n_transformer", "target": "n_layernorm", "relation": "contains", "confidence": "EXTRACTED", "source_file": "model.py", "weight": 1.0},+ {"source": "n_attention", "target": "n_concept_attn", "relation": "implements", "confidence": "INFERRED", "source_file": "model.py", "weight": 0.8},+ {"source": "n_layernorm", "target": "n_concept_attn", "relation": "referenced", "confidence": "AMBIGUOUS", "source_file": "paper.md", "weight": 0.5}+ ],+ "input_tokens": 1200,+ "output_tokens": 340+}diff --git a/tests/fixtures/sample.md b/tests/fixtures/sample.mdnew file mode 100644index 000000000..76cb4b47a--- /dev/null+++ b/tests/fixtures/sample.md@@ -0,0 +1,5 @@+# Attention Is All You Need++The transformer architecture uses multi-head attention.+Layer normalization is applied before each sub-layer.+The feed-forward network consists of two linear transformations.diff --git a/tests/fixtures/sample.py b/tests/fixtures/sample.pynew file mode 100644index 000000000..9dd9f2a3e--- /dev/null+++ b/tests/fixtures/sample.py@@ -0,0 +1,6 @@+class Transformer:+ def __init__(self, d_model: int):+ self.d_model = d_model++ def forward(self, x):+ return xdiff --git a/tests/test_analyzer.py b/tests/test_analyzer.pynew file mode 100644index 000000000..2ccfb8e89--- /dev/null+++ b/tests/test_analyzer.py@@ -0,0 +1,116 @@+import json+import networkx as nx+from pathlib import Path+from graphify.graph_builder import build_from_json+from graphify.clusterer import cluster+from graphify.analyzer import god_nodes, surprising_connections, _is_concept_node++FIXTURES = Path(__file__).parent / "fixtures"+++def make_graph():+ return build_from_json(json.loads((FIXTURES / "extraction.json").read_text()))+++def test_god_nodes_returns_list():+ G = make_graph()+ result = god_nodes(G, top_n=3)+ assert isinstance(result, list)+ assert len(result) <= 3+++def test_god_nodes_sorted_by_degree():+ G = make_graph()+ result = god_nodes(G, top_n=10)+ degrees = [r["edges"] for r in result]+ assert degrees == sorted(degrees, reverse=True)+++def test_god_nodes_have_required_keys():+ G = make_graph()+ result = god_nodes(G, top_n=1)+ assert "id" in result[0]+ assert "label" in result[0]+ assert "edges" in result[0]+++def test_surprising_connections_cross_source_multi_file():+ """Multi-file graph: should find cross-file edges between real entities."""+ G = make_graph()+ communities = cluster(G)+ surprises = surprising_connections(G, communities)+ assert len(surprises) > 0+ for s in surprises:+ assert s["source_files"][0] != s["source_files"][1]+++def test_surprising_connections_excludes_concept_nodes():+ """Concept nodes (empty source_file) must not appear in surprises."""+ G = make_graph()+ # Add a concept node with empty source_file+ G.add_node("concept_x", label="Abstract Concept", file_type="document", source_file="")+ G.add_edge("n_transformer", "concept_x", relation="relates_to",+ confidence="INFERRED", source_file="", weight=0.5)+ communities = cluster(G)+ surprises = surprising_connections(G, communities)+ labels = [s["source"] for s in surprises] + [s["target"] for s in surprises]+ assert "Abstract Concept" not in labels+++def test_surprising_connections_single_file_uses_community_bridges():+ """Single-file graph: should return cross-community edges, not empty list."""+ G = nx.Graph()+ # Build a graph with 2 clear communities + 1 bridge edge+ for i in range(5):+ G.add_node(f"a{i}", label=f"A{i}", file_type="code", source_file="single.py",+ source_location=f"L{i}")+ for i in range(5):+ G.add_node(f"b{i}", label=f"B{i}", file_type="code", source_file="single.py",+ source_location=f"L{i+10}")+ # Dense intra-community edges+ for i in range(4):+ G.add_edge(f"a{i}", f"a{i+1}", relation="calls", confidence="EXTRACTED",+ source_file="single.py", weight=1.0)+ for i in range(4):+ G.add_edge(f"b{i}", f"b{i+1}", relation="calls", confidence="EXTRACTED",+ source_file="single.py", weight=1.0)+ # One cross-community bridge+ G.add_edge("a4", "b0", relation="references", confidence="INFERRED",+ source_file="single.py", weight=0.5)++ communities = cluster(G)+ surprises = surprising_connections(G, communities)+ # Should find at least the bridge edge+ assert len(surprises) > 0+++def test_surprising_connections_ambiguous_first():+ G = make_graph()+ communities = cluster(G)+ surprises = surprising_connections(G, communities)+ if len(surprises) >= 2:+ order = {"AMBIGUOUS": 0, "INFERRED": 1, "EXTRACTED": 2}+ confidences = [order[s["confidence"]] for s in surprises]+ assert confidences == sorted(confidences)+++def test_is_concept_node_empty_source():+ G = nx.Graph()+ G.add_node("c1", source_file="")+ assert _is_concept_node(G, "c1") is True+++def test_is_concept_node_real_file():+ G = nx.Graph()+ G.add_node("n1", source_file="model.py")+ assert _is_concept_node(G, "n1") is False+++def test_surprising_connections_have_required_keys():+ G = make_graph()+ communities = cluster(G)+ for s in surprising_connections(G, communities):+ assert "source" in s+ assert "target" in s+ assert "source_files" in s+ assert "confidence" in sdiff --git a/tests/test_ast_extractor.py b/tests/test_ast_extractor.pynew file mode 100644index 000000000..8b1c15393--- /dev/null+++ b/tests/test_ast_extractor.py@@ -0,0 +1,76 @@+from pathlib import Path+from graphify.ast_extractor import extract_python, extract, collect_files, _make_id++FIXTURES = Path(__file__).parent / "fixtures"+++def test_make_id_strips_dots_and_underscores():+ assert _make_id("_auth") == "auth"+ assert _make_id(".httpx._client") == "httpx_client"+++def test_make_id_consistent():+ """Same input always produces same output."""+ assert _make_id("foo", "Bar") == _make_id("foo", "Bar")+++def test_make_id_no_leading_trailing_underscores():+ result = _make_id("__init__")+ assert not result.startswith("_")+ assert not result.endswith("_")+++def test_extract_python_finds_class():+ result = extract_python(FIXTURES / "sample.py")+ labels = [n["label"] for n in result["nodes"]]+ assert "Transformer" in labels+++def test_extract_python_finds_methods():+ result = extract_python(FIXTURES / "sample.py")+ labels = [n["label"] for n in result["nodes"]]+ assert any("__init__" in l or "forward" in l for l in labels)+++def test_extract_python_no_dangling_edges():+ """All edge sources must reference a known node (targets may be external imports)."""+ result = extract_python(FIXTURES / "sample.py")+ node_ids = {n["id"] for n in result["nodes"]}+ for edge in result["edges"]:+ assert edge["source"] in node_ids, f"Dangling source: {edge['source']}"+++def test_extract_python_edges_are_extracted():+ result = extract_python(FIXTURES / "sample.py")+ for edge in result["edges"]:+ assert edge["confidence"] == "EXTRACTED"+++def test_extract_merges_multiple_files():+ files = list(FIXTURES.glob("*.py"))+ result = extract(files)+ assert len(result["nodes"]) > 0+ assert result["input_tokens"] == 0+++def test_collect_files_from_dir():+ files = collect_files(FIXTURES)+ assert all(f.suffix == ".py" for f in files)+ assert len(files) > 0+++def test_collect_files_skips_hidden():+ files = collect_files(FIXTURES)+ for f in files:+ assert not any(part.startswith(".") for part in f.parts)+++def test_no_dangling_edges_on_extract():+ """After merging multiple files, no internal edges should be dangling."""+ files = list(FIXTURES.glob("*.py"))+ result = extract(files)+ node_ids = {n["id"] for n in result["nodes"]}+ internal_relations = {"contains", "method", "inherits"}+ for edge in result["edges"]:+ if edge["relation"] in internal_relations:+ assert edge["source"] in node_ids, f"Dangling: {edge}"diff --git a/tests/test_clusterer.py b/tests/test_clusterer.pynew file mode 100644index 000000000..b5300bd75--- /dev/null+++ b/tests/test_clusterer.py@@ -0,0 +1,52 @@+import json+import networkx as nx+from pathlib import Path+from graphify.graph_builder import build_from_json+from graphify.clusterer import cluster, cohesion_score, score_all++FIXTURES = Path(__file__).parent / "fixtures"++def make_graph():+ return build_from_json(json.loads((FIXTURES / "extraction.json").read_text()))++def test_cluster_returns_dict():+ G = make_graph()+ communities = cluster(G)+ assert isinstance(communities, dict)++def test_cluster_covers_all_nodes():+ G = make_graph()+ communities = cluster(G)+ all_nodes = {n for nodes in communities.values() for n in nodes}+ assert all_nodes == set(G.nodes)++def test_cohesion_score_complete_graph():+ G = nx.complete_graph(4)+ G = nx.relabel_nodes(G, {i: str(i) for i in G.nodes})+ score = cohesion_score(G, list(G.nodes))+ assert score == 1.0++def test_cohesion_score_single_node():+ G = nx.Graph()+ G.add_node("a")+ score = cohesion_score(G, ["a"])+ assert score == 1.0++def test_cohesion_score_disconnected():+ G = nx.Graph()+ G.add_nodes_from(["a", "b", "c"])+ score = cohesion_score(G, ["a", "b", "c"])+ assert score == 0.0++def test_cohesion_score_range():+ G = make_graph()+ communities = cluster(G)+ for cid, nodes in communities.items():+ score = cohesion_score(G, nodes)+ assert 0.0 <= score <= 1.0++def test_score_all_keys_match_communities():+ G = make_graph()+ communities = cluster(G)+ scores = score_all(G, communities)+ assert set(scores.keys()) == set(communities.keys())diff --git a/tests/test_detector.py b/tests/test_detector.pynew file mode 100644index 000000000..fd407f290--- /dev/null+++ b/tests/test_detector.py@@ -0,0 +1,66 @@+from pathlib import Path+from graphify.detector import classify_file, count_words, detect, FileType, _looks_like_paper++FIXTURES = Path(__file__).parent / "fixtures"++def test_classify_python():+ assert classify_file(Path("foo.py")) == FileType.CODE++def test_classify_typescript():+ assert classify_file(Path("bar.ts")) == FileType.CODE++def test_classify_markdown():+ assert classify_file(Path("README.md")) == FileType.DOCUMENT++def test_classify_pdf():+ assert classify_file(Path("paper.pdf")) == FileType.PAPER++def test_classify_unknown_returns_none():+ assert classify_file(Path("image.png")) is None++def test_count_words_sample_md():+ words = count_words(FIXTURES / "sample.md")+ assert words > 5++def test_detect_finds_fixtures():+ result = detect(FIXTURES)+ assert result["total_files"] >= 2+ assert "code" in result["files"]+ assert "document" in result["files"]++def test_detect_warns_small_corpus():+ result = detect(FIXTURES)+ assert result["needs_graph"] is False+ assert result["warning"] is not None++def test_detect_skips_dotfiles():+ result = detect(FIXTURES)+ for files in result["files"].values():+ for f in files:+ assert "/." not in f+++def test_classify_md_paper_by_signals(tmp_path):+ """A .md file with enough paper signals should classify as PAPER."""+ paper = tmp_path / "paper.md"+ paper.write_text(+ "# Abstract\n\nWe propose a new method. See [1] and [23].\n"+ "This work was published in the Journal of AI. ArXiv preprint.\n"+ "See Equation 3 for details. \\cite{vaswani2017}.\n"+ )+ assert classify_file(paper) == FileType.PAPER+++def test_classify_md_doc_without_signals(tmp_path):+ """A plain .md file without paper signals should stay DOCUMENT."""+ doc = tmp_path / "notes.md"+ doc.write_text("# My Notes\n\nHere are some notes about the project.\n")+ assert classify_file(doc) == FileType.DOCUMENT+++def test_classify_attention_paper():+ """The real attention paper file should be classified as PAPER."""+ paper_path = Path("/home/safi/graphify_eval/papers/attention_is_all_you_need.md")+ if paper_path.exists():+ result = classify_file(paper_path)+ assert result == FileType.PAPERdiff --git a/tests/test_exporter.py b/tests/test_exporter.pynew file mode 100644index 000000000..65aec9e4b--- /dev/null+++ b/tests/test_exporter.py@@ -0,0 +1,54 @@+import json+import tempfile+from pathlib import Path+from graphify.graph_builder import build_from_json+from graphify.clusterer import cluster+from graphify.exporter import to_json, to_cypher++FIXTURES = Path(__file__).parent / "fixtures"++def make_graph():+ return build_from_json(json.loads((FIXTURES / "extraction.json").read_text()))++def test_to_json_creates_file():+ G = make_graph()+ communities = cluster(G)+ with tempfile.TemporaryDirectory() as tmp:+ out = Path(tmp) / "graph.json"+ to_json(G, communities, str(out))+ assert out.exists()++def test_to_json_valid_json():+ G = make_graph()+ communities = cluster(G)+ with tempfile.TemporaryDirectory() as tmp:+ out = Path(tmp) / "graph.json"+ to_json(G, communities, str(out))+ data = json.loads(out.read_text())+ assert "nodes" in data+ assert "links" in data++def test_to_json_nodes_have_community():+ G = make_graph()+ communities = cluster(G)+ with tempfile.TemporaryDirectory() as tmp:+ out = Path(tmp) / "graph.json"+ to_json(G, communities, str(out))+ data = json.loads(out.read_text())+ for node in data["nodes"]:+ assert "community" in node++def test_to_cypher_creates_file():+ G = make_graph()+ with tempfile.TemporaryDirectory() as tmp:+ out = Path(tmp) / "cypher.txt"+ to_cypher(G, str(out))+ assert out.exists()++def test_to_cypher_contains_merge_statements():+ G = make_graph()+ with tempfile.TemporaryDirectory() as tmp:+ out = Path(tmp) / "cypher.txt"+ to_cypher(G, str(out))+ content = out.read_text()+ assert "MERGE" in contentdiff --git a/tests/test_graph_builder.py b/tests/test_graph_builder.pynew file mode 100644index 000000000..28c9d6042--- /dev/null+++ b/tests/test_graph_builder.py@@ -0,0 +1,42 @@+import json+from pathlib import Path+from graphify.graph_builder import build_from_json, build++FIXTURES = Path(__file__).parent / "fixtures"++def load_extraction():+ return json.loads((FIXTURES / "extraction.json").read_text())++def test_build_from_json_node_count():+ G = build_from_json(load_extraction())+ assert G.number_of_nodes() == 4++def test_build_from_json_edge_count():+ G = build_from_json(load_extraction())+ assert G.number_of_edges() == 4++def test_nodes_have_label():+ G = build_from_json(load_extraction())+ assert G.nodes["n_transformer"]["label"] == "Transformer"++def test_edges_have_confidence():+ G = build_from_json(load_extraction())+ data = G.edges["n_attention", "n_concept_attn"]+ assert data["confidence"] == "INFERRED"++def test_ambiguous_edge_preserved():+ G = build_from_json(load_extraction())+ data = G.edges["n_layernorm", "n_concept_attn"]+ assert data["confidence"] == "AMBIGUOUS"++def test_build_merges_multiple_extractions():+ ext1 = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}],+ "edges": [], "input_tokens": 0, "output_tokens": 0}+ ext2 = {"nodes": [{"id": "n2", "label": "B", "file_type": "document", "source_file": "b.md"}],+ "edges": [{"source": "n1", "target": "n2", "relation": "references",+ "confidence": "INFERRED", "source_file": "b.md", "weight": 1.0}],+ "input_tokens": 0, "output_tokens": 0}+ G = build([ext1, ext2])+ assert G.number_of_nodes() == 2+ assert G.number_of_edges() == 1+diff --git a/tests/test_models.py b/tests/test_models.pynew file mode 100644index 000000000..98e1fc4c7--- /dev/null+++ b/tests/test_models.py@@ -0,0 +1,26 @@+from graphify.models import Confidence, FileType, GraphNode, GraphEdge, ExtractionResult++def test_confidence_values():+ assert Confidence.EXTRACTED.value == "EXTRACTED"+ assert Confidence.INFERRED.value == "INFERRED"+ assert Confidence.AMBIGUOUS.value == "AMBIGUOUS"++def test_graph_node_defaults():+ node = GraphNode(id="n1", label="MyClass", file_type=FileType.CODE, source_file="foo.py")+ assert node.community is None+ assert node.source_location is None++def test_graph_edge_defaults():+ edge = GraphEdge(source="n1", target="n2", relation="imports",+ confidence=Confidence.EXTRACTED, source_file="foo.py")+ assert edge.weight == 1.0++def test_extraction_result_accumulates():+ r = ExtractionResult()+ r.nodes.append(GraphNode(id="n1", label="X", file_type=FileType.CODE, source_file="a.py"))+ r.edges.append(GraphEdge(source="n1", target="n2", relation="calls",+ confidence=Confidence.INFERRED, source_file="a.py"))+ assert len(r.nodes) == 1+ assert len(r.edges) == 1+ r.input_tokens += 100+ assert r.input_tokens == 100diff --git a/tests/test_reporter.py b/tests/test_reporter.pynew file mode 100644index 000000000..c68e9139d--- /dev/null+++ b/tests/test_reporter.py@@ -0,0 +1,63 @@+import json+from pathlib import Path+from graphify.graph_builder import build_from_json+from graphify.clusterer import cluster, score_all+from graphify.analyzer import god_nodes, surprising_connections+from graphify.reporter import generate++FIXTURES = Path(__file__).parent / "fixtures"++def make_inputs():+ extraction = json.loads((FIXTURES / "extraction.json").read_text())+ G = build_from_json(extraction)+ communities = cluster(G)+ cohesion = score_all(G, communities)+ labels = {cid: f"Community {cid}" for cid in communities}+ gods = god_nodes(G)+ surprises = surprising_connections(G)+ detection = {"total_files": 4, "total_words": 62400, "needs_graph": True, "warning": None}+ tokens = {"input": extraction["input_tokens"], "output": extraction["output_tokens"]}+ return G, communities, cohesion, labels, gods, surprises, detection, tokens++def test_report_contains_header():+ G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs()+ report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project")+ assert "# Graph Report" in report++def test_report_contains_corpus_check():+ G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs()+ report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project")+ assert "## Corpus Check" in report++def test_report_contains_god_nodes():+ G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs()+ report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project")+ assert "## God Nodes" in report++def test_report_contains_surprising_connections():+ G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs()+ report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project")+ assert "## Surprising Connections" in report++def test_report_contains_communities():+ G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs()+ report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project")+ assert "## Communities" in report++def test_report_contains_ambiguous_section():+ G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs()+ report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project")+ assert "## Ambiguous Edges" in report++def test_report_shows_token_cost():+ G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs()+ report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project")+ assert "Token cost" in report+ assert "1,200" in report++def test_report_shows_raw_cohesion_scores():+ G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs()+ report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project")+ assert "Cohesion:" in report+ assert "✓" not in report+ assert "⚠" not in reportdiff --git a/tests/test_validator.py b/tests/test_validator.pynew file mode 100644index 000000000..555271e2e--- /dev/null+++ b/tests/test_validator.py@@ -0,0 +1,87 @@+import pytest+from graphify.validator import validate_extraction, assert_valid++VALID = {+ "nodes": [+ {"id": "n1", "label": "Foo", "file_type": "code", "source_file": "foo.py"},+ {"id": "n2", "label": "Bar", "file_type": "document", "source_file": "bar.md"},+ ],+ "edges": [+ {"source": "n1", "target": "n2", "relation": "references",+ "confidence": "EXTRACTED", "source_file": "foo.py", "weight": 1.0},+ ],+}++def test_valid_passes():+ assert validate_extraction(VALID) == []++def test_missing_nodes_key():+ errors = validate_extraction({"edges": []})+ assert any("nodes" in e for e in errors)++def test_missing_edges_key():+ errors = validate_extraction({"nodes": []})+ assert any("edges" in e for e in errors)++def test_not_a_dict():+ errors = validate_extraction([])+ assert len(errors) == 1++def test_invalid_file_type():+ data = {+ "nodes": [{"id": "n1", "label": "X", "file_type": "video", "source_file": "x.mp4"}],+ "edges": [],+ }+ errors = validate_extraction(data)+ assert any("file_type" in e for e in errors)++def test_invalid_confidence():+ data = {+ "nodes": [+ {"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"},+ {"id": "n2", "label": "B", "file_type": "code", "source_file": "b.py"},+ ],+ "edges": [+ {"source": "n1", "target": "n2", "relation": "calls",+ "confidence": "CERTAIN", "source_file": "a.py"},+ ],+ }+ errors = validate_extraction(data)+ assert any("confidence" in e for e in errors)++def test_dangling_edge_source():+ data = {+ "nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}],+ "edges": [+ {"source": "missing_id", "target": "n1", "relation": "calls",+ "confidence": "EXTRACTED", "source_file": "a.py"},+ ],+ }+ errors = validate_extraction(data)+ assert any("source" in e and "missing_id" in e for e in errors)++def test_dangling_edge_target():+ data = {+ "nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}],+ "edges": [+ {"source": "n1", "target": "ghost", "relation": "calls",+ "confidence": "EXTRACTED", "source_file": "a.py"},+ ],+ }+ errors = validate_extraction(data)+ assert any("target" in e and "ghost" in e for e in errors)++def test_missing_node_field():+ data = {+ "nodes": [{"id": "n1", "label": "A", "source_file": "a.py"}], # missing file_type+ "edges": [],+ }+ errors = validate_extraction(data)+ assert any("file_type" in e for e in errors)++def test_assert_valid_raises_on_errors():+ with pytest.raises(ValueError, match="error"):+ assert_valid({"nodes": [], "edges": [], "oops": True, **{"nodes": "bad"}})++def test_assert_valid_passes_silently():+ assert_valid(VALID) # should not raise