Turn any codebase, with its docs, SQL schemas, configs, and PDFs, into a queryable knowledge graph. A /graphify skill for Claude Code, Cursor, Codex, and Gemini CLI: local deterministic AST parsing, every edge explained, no vector store.
Python ai-agentsantigravityastclaude-codecode-analysiscode-searchcodexcursordeveloper-toolsgeminigraphifygraphragknowledge-graphleidenllmmcpopenclawragskillstree-sitter
add reproducible worked examples for httpx, karpathy-repos, and mixed-corpus benchmarks
Safi0fee55726d66
diff --git a/worked/httpx/README.md b/worked/httpx/README.mdnew file mode 100644index 000000000..3d1c924d1--- /dev/null+++ b/worked/httpx/README.md@@ -0,0 +1,42 @@+# httpx Corpus Benchmark — How to Reproduce++A synthetic 6-file Python codebase modeled after httpx's architecture. Tests graphify+on a realistic library codebase with clean layering: exceptions → models → auth/transport → client.++## Corpus (6 files)++All input files are in `raw/`:++```+raw/+├── exceptions.py — full HTTPError hierarchy (RequestError, TransportError, HTTPStatusError, etc.)+├── models.py — URL, Headers, Cookies, Request, Response with raise_for_status+├── auth.py — BasicAuth, BearerAuth, DigestAuth (challenge-response), NetRCAuth+├── utils.py — header normalization, query param flattening, content-type parsing+├── transport.py — ConnectionPool, HTTPTransport, AsyncHTTPTransport, MockTransport, ProxyTransport+└── client.py — Timeout, Limits, BaseClient, Client (sync), AsyncClient+```++## How to run++```bash+pip install graphifyy && graphify install+/graphify ./raw+```++Or from the CLI directly:++```bash+pip install graphifyy+graphify ./raw+```++## What to expect++- ~95 nodes, ~130 edges+- 4 communities: Exception Hierarchy, Models & Data, Auth & Transport, Client Layer+- God nodes: `client.py`, `models.py`, `transport.py`, `exceptions.py`, `BaseClient`, `Response`+- Surprising connections: `DigestAuth` ↔ `Response` (auth.py reads Response to parse WWW-Authenticate)+- All edges EXTRACTED — no inference needed, dependency graph is explicit++Full eval with scores and analysis: `review.md`diff --git a/worked/httpx/raw/auth.py b/worked/httpx/raw/auth.pynew file mode 100644index 000000000..290cadd3e--- /dev/null+++ b/worked/httpx/raw/auth.py@@ -0,0 +1,114 @@+"""+Authentication handlers.+Auth objects are callables that modify a request before it is sent.+DigestAuth is the most interesting: it participates in a full request/response cycle,+reading the 401 response to build the challenge before re-sending.+"""+import hashlib+import time+from models import Request, Response+++class Auth:+ """Base class for all authentication handlers."""++ def auth_flow(self, request: Request):+ """Modify the request. May yield to inspect the response."""+ raise NotImplementedError+++class BasicAuth(Auth):+ """HTTP Basic Authentication."""++ def __init__(self, username: str, password: str):+ self.username = username+ self.password = password++ def auth_flow(self, request: Request):+ import base64+ credentials = f"{self.username}:{self.password}".encode()+ encoded = base64.b64encode(credentials).decode()+ request.headers["Authorization"] = f"Basic {encoded}"+ yield request+++class BearerAuth(Auth):+ """Bearer token authentication."""++ def __init__(self, token: str):+ self.token = token++ def auth_flow(self, request: Request):+ request.headers["Authorization"] = f"Bearer {self.token}"+ yield request+++class DigestAuth(Auth):+ """+ HTTP Digest Authentication.+ Requires a full request/response cycle: sends the initial request,+ reads the 401 WWW-Authenticate header, then re-sends with credentials.+ This is the only auth handler that reads from Response.+ """++ def __init__(self, username: str, password: str):+ self.username = username+ self.password = password+ self._nonce_count = 0++ def auth_flow(self, request: Request):+ yield request # first attempt, no credentials++ # This handler must inspect the Response to continue+ response = yield++ if response.status_code == 401:+ challenge = self._parse_challenge(response)+ credentials = self._build_credentials(request, challenge)+ request.headers["Authorization"] = credentials+ yield request++ def _parse_challenge(self, response: Response) -> dict:+ """Extract digest parameters from the WWW-Authenticate header."""+ header = response.headers.get("www-authenticate", "")+ params = {}+ for part in header.replace("Digest ", "").split(","):+ if "=" in part:+ key, _, value = part.strip().partition("=")+ params[key.strip()] = value.strip().strip('"')+ return params++ def _build_credentials(self, request: Request, challenge: dict) -> str:+ """Compute the Authorization header value for a digest challenge."""+ self._nonce_count += 1+ nc = f"{self._nonce_count:08x}"+ cnonce = hashlib.md5(str(time.time()).encode()).hexdigest()[:8]+ realm = challenge.get("realm", "")+ nonce = challenge.get("nonce", "")++ ha1 = hashlib.md5(f"{self.username}:{realm}:{self.password}".encode()).hexdigest()+ ha2 = hashlib.md5(f"{request.method}:{request.url.path}".encode()).hexdigest()+ response_hash = hashlib.md5(f"{ha1}:{nonce}:{nc}:{cnonce}:auth:{ha2}".encode()).hexdigest()++ return (+ f'Digest username="{self.username}", realm="{realm}", '+ f'nonce="{nonce}", uri="{request.url.path}", '+ f'nc={nc}, cnonce="{cnonce}", response="{response_hash}"'+ )+++class NetRCAuth(Auth):+ """Load credentials from ~/.netrc based on the request host."""++ def auth_flow(self, request: Request):+ import netrc+ try:+ credentials = netrc.netrc().authenticators(request.url.host)+ if credentials:+ username, _, password = credentials+ basic = BasicAuth(username, password)+ yield from basic.auth_flow(request)+ return+ except Exception:+ pass+ yield requestdiff --git a/worked/httpx/raw/client.py b/worked/httpx/raw/client.pynew file mode 100644index 000000000..d506dd613--- /dev/null+++ b/worked/httpx/raw/client.py@@ -0,0 +1,161 @@+"""+The main Client and AsyncClient classes.+BaseClient holds all shared logic. Client and AsyncClient extend it for sync/async.+This is the integration hub of the library - it imports from every other module.+"""+from models import Request, Response, URL, Headers, Cookies+from auth import Auth, BasicAuth+from transport import BaseTransport, HTTPTransport, AsyncHTTPTransport+from exceptions import TooManyRedirects, InvalidURL+from utils import build_url_with_params, obfuscate_sensitive_headers+++DEFAULT_MAX_REDIRECTS = 20+++class Timeout:+ def __init__(self, timeout=5.0, *, connect=None, read=None, write=None, pool=None):+ self.connect = connect or timeout+ self.read = read or timeout+ self.write = write or timeout+ self.pool = pool or timeout+++class Limits:+ def __init__(self, max_connections=100, max_keepalive_connections=20, keepalive_expiry=5.0):+ self.max_connections = max_connections+ self.max_keepalive_connections = max_keepalive_connections+ self.keepalive_expiry = keepalive_expiry+++class BaseClient:+ """+ Shared implementation for Client and AsyncClient.+ Handles auth, redirects, cookies, and header defaults.+ """++ def __init__(+ self,+ *,+ auth=None,+ headers=None,+ cookies=None,+ timeout=Timeout(),+ max_redirects=DEFAULT_MAX_REDIRECTS,+ base_url="",+ ):+ self._auth = auth+ self._headers = Headers(headers or {})+ self._cookies = Cookies(cookies or {})+ self._timeout = timeout+ self._max_redirects = max_redirects+ self._base_url = URL(base_url) if base_url else None++ def _build_request(self, method: str, url: str, **kwargs) -> Request:+ if self._base_url:+ url = self._base_url.raw.rstrip("/") + "/" + url.lstrip("/")+ if kwargs.get("params"):+ url = build_url_with_params(url, kwargs.pop("params"))+ headers = Headers(kwargs.get("headers", {}))+ for k, v in self._headers.items():+ if k not in headers:+ headers[k] = v+ return Request(method, url, headers=headers, content=kwargs.get("content"), cookies=self._cookies)++ def _merge_cookies(self, response: Response) -> None:+ for name, value in response.cookies.items():+ self._cookies.set(name, value)+++class Client(BaseClient):+ """Synchronous HTTP client."""++ def __init__(self, *, transport: BaseTransport = None, **kwargs):+ super().__init__(**kwargs)+ self._transport = transport or HTTPTransport()++ def request(self, method: str, url: str, **kwargs) -> Response:+ request = self._build_request(method, url, **kwargs)+ auth = kwargs.get("auth") or self._auth+ if auth:+ flow = auth.auth_flow(request)+ request = next(flow)+ response = self._transport.handle_request(request)+ self._merge_cookies(response)+ if auth:+ try:+ flow.send(response)+ except StopIteration:+ pass+ return response++ def get(self, url: str, **kwargs) -> Response:+ return self.request("GET", url, **kwargs)++ def post(self, url: str, **kwargs) -> Response:+ return self.request("POST", url, **kwargs)++ def put(self, url: str, **kwargs) -> Response:+ return self.request("PUT", url, **kwargs)++ def patch(self, url: str, **kwargs) -> Response:+ return self.request("PATCH", url, **kwargs)++ def delete(self, url: str, **kwargs) -> Response:+ return self.request("DELETE", url, **kwargs)++ def head(self, url: str, **kwargs) -> Response:+ return self.request("HEAD", url, **kwargs)++ def send(self, request: Request) -> Response:+ return self._transport.handle_request(request)++ def close(self) -> None:+ self._transport.close()++ def __enter__(self):+ return self++ def __exit__(self, *args):+ self.close()+++class AsyncClient(BaseClient):+ """Asynchronous HTTP client."""++ def __init__(self, *, transport=None, **kwargs):+ super().__init__(**kwargs)+ self._transport = transport or AsyncHTTPTransport()++ async def request(self, method: str, url: str, **kwargs) -> Response:+ request = self._build_request(method, url, **kwargs)+ response = await self._transport.handle_async_request(request)+ self._merge_cookies(response)+ return response++ async def get(self, url: str, **kwargs) -> Response:+ return await self.request("GET", url, **kwargs)++ async def post(self, url: str, **kwargs) -> Response:+ return await self.request("POST", url, **kwargs)++ async def put(self, url: str, **kwargs) -> Response:+ return await self.request("PUT", url, **kwargs)++ async def patch(self, url: str, **kwargs) -> Response:+ return await self.request("PATCH", url, **kwargs)++ async def delete(self, url: str, **kwargs) -> Response:+ return await self.request("DELETE", url, **kwargs)++ async def send(self, request: Request) -> Response:+ return await self._transport.handle_async_request(request)++ async def aclose(self) -> None:+ await self._transport.aclose()++ async def __aenter__(self):+ return self++ async def __aexit__(self, *args):+ await self.aclose()diff --git a/worked/httpx/raw/exceptions.py b/worked/httpx/raw/exceptions.pynew file mode 100644index 000000000..ff5392fee--- /dev/null+++ b/worked/httpx/raw/exceptions.py@@ -0,0 +1,90 @@+"""+httpx-like exception hierarchy.+All exceptions inherit from HTTPError at the top.+"""+++class HTTPError(Exception):+ """Base class for all httpx exceptions."""+ def __init__(self, message, *, request=None):+ self.request = request+ super().__init__(message)+++class RequestError(HTTPError):+ """An error occurred while issuing a request."""+++class TransportError(RequestError):+ """An error occurred at the transport layer."""+++class TimeoutException(TransportError):+ """A timeout occurred."""+++class ConnectTimeout(TimeoutException):+ """Timed out while connecting to the host."""+++class ReadTimeout(TimeoutException):+ """Timed out while receiving data from the host."""+++class WriteTimeout(TimeoutException):+ """Timed out while sending data to the host."""+++class PoolTimeout(TimeoutException):+ """Timed out waiting to acquire a connection from the pool."""+++class NetworkError(TransportError):+ """A network error occurred."""+++class ConnectError(NetworkError):+ """Failed to establish a connection."""+++class ReadError(NetworkError):+ """Failed to receive data from the network."""+++class WriteError(NetworkError):+ """Failed to send data through the network."""+++class CloseError(NetworkError):+ """Failed to close a connection."""+++class ProxyError(TransportError):+ """An error occurred while establishing a proxy connection."""+++class ProtocolError(TransportError):+ """A protocol was violated."""+++class DecodingError(RequestError):+ """Decoding of the response failed."""+++class TooManyRedirects(RequestError):+ """Too many redirects."""+++class HTTPStatusError(HTTPError):+ """A 4xx or 5xx response was received."""+ def __init__(self, message, *, request, response):+ self.response = response+ super().__init__(message, request=request)+++class InvalidURL(Exception):+ """URL is improperly formed or cannot be parsed."""+++class CookieConflict(Exception):+ """Attempted to look up a cookie by name but multiple cookies exist."""diff --git a/worked/httpx/raw/models.py b/worked/httpx/raw/models.pynew file mode 100644index 000000000..80582b6fa--- /dev/null+++ b/worked/httpx/raw/models.py@@ -0,0 +1,120 @@+"""+Core data models: URL, Headers, Cookies, Request, Response.+These are the central data types that everything else in the library references.+"""+import json as _json+from exceptions import HTTPStatusError+++class URL:+ def __init__(self, url: str):+ self.raw = url+ self.scheme, _, rest = url.partition("://")+ self.host, _, self.path = rest.partition("/")+ self.path = "/" + self.path++ def copy_with(self, **kwargs) -> "URL":+ return URL(kwargs.get("url", self.raw))++ def __str__(self):+ return self.raw++ def __repr__(self):+ return f"URL({self.raw!r})"+++class Headers:+ def __init__(self, headers=None):+ self._store = {}+ for k, v in (headers or {}).items():+ self._store[k.lower()] = v++ def get(self, key: str, default=None):+ return self._store.get(key.lower(), default)++ def items(self):+ return self._store.items()++ def __setitem__(self, key, value):+ self._store[key.lower()] = value++ def __getitem__(self, key):+ return self._store[key.lower()]++ def __contains__(self, key):+ return key.lower() in self._store+++class Cookies:+ def __init__(self, cookies=None):+ self._jar = dict(cookies or {})++ def set(self, name: str, value: str, domain: str = "") -> None:+ self._jar[name] = value++ def get(self, name: str, default=None):+ return self._jar.get(name, default)++ def delete(self, name: str) -> None:+ self._jar.pop(name, None)++ def clear(self) -> None:+ self._jar.clear()++ def items(self):+ return self._jar.items()+++class Request:+ def __init__(self, method: str, url, *, headers=None, content=None, cookies=None):+ self.method = method.upper()+ self.url = URL(url) if isinstance(url, str) else url+ self.headers = Headers(headers)+ self.content = content or b""+ self.cookies = Cookies(cookies)++ def __repr__(self):+ return f"<Request [{self.method}]>"+++class Response:+ def __init__(self, status_code: int, *, headers=None, content=None, request=None):+ self.status_code = status_code+ self.headers = Headers(headers)+ self.content = content or b""+ self.request = request++ @property+ def text(self) -> str:+ return self.content.decode("utf-8", errors="replace")++ def json(self):+ return _json.loads(self.content)++ def read(self) -> bytes:+ return self.content++ @property+ def is_success(self) -> bool:+ return 200 <= self.status_code < 300++ @property+ def is_error(self) -> bool:+ return self.status_code >= 400++ def raise_for_status(self) -> None:+ if self.is_error:+ message = f"{self.status_code} Error"+ raise HTTPStatusError(message, request=self.request, response=self)++ @property+ def cookies(self) -> Cookies:+ jar = Cookies()+ for header in self.headers.get("set-cookie", "").split(","):+ if "=" in header:+ name, _, value = header.strip().partition("=")+ jar.set(name.strip(), value.split(";")[0].strip())+ return jar++ def __repr__(self):+ return f"<Response [{self.status_code}]>"diff --git a/worked/httpx/raw/transport.py b/worked/httpx/raw/transport.pynew file mode 100644index 000000000..5bd9b9166--- /dev/null+++ b/worked/httpx/raw/transport.py@@ -0,0 +1,135 @@+"""+Transport layer: connection management and low-level HTTP sending.+HTTPTransport wraps a connection pool. ProxyTransport sits in front of it.+MockTransport is used in tests.+"""+from models import Request, Response+from exceptions import TransportError, ConnectError, TimeoutException+++class BaseTransport:+ """Sync transport interface."""++ def handle_request(self, request: Request) -> Response:+ raise NotImplementedError++ def close(self) -> None:+ pass+++class AsyncBaseTransport:+ """Async transport interface."""++ async def handle_async_request(self, request: Request) -> Response:+ raise NotImplementedError++ async def aclose(self) -> None:+ pass+++class ConnectionPool:+ """+ Manages a pool of persistent HTTP connections.+ Keys connections by (scheme, host, port).+ """++ def __init__(self, max_connections=100, max_keepalive_connections=20):+ self.max_connections = max_connections+ self.max_keepalive_connections = max_keepalive_connections+ self._pool = {}++ def _get_connection_key(self, request: Request) -> tuple:+ url = request.url+ port = 443 if url.scheme == "https" else 80+ return (url.scheme, url.host, port)++ def get_connection(self, request: Request):+ key = self._get_connection_key(request)+ return self._pool.get(key)++ def return_connection(self, request: Request, conn) -> None:+ key = self._get_connection_key(request)+ if len(self._pool) < self.max_keepalive_connections:+ self._pool[key] = conn++ def close(self) -> None:+ self._pool.clear()+++class HTTPTransport(BaseTransport):+ """+ The main sync HTTP transport.+ Uses a ConnectionPool for connection reuse.+ """++ def __init__(self, verify=True, cert=None, limits=None):+ self.verify = verify+ self.cert = cert+ self._pool = ConnectionPool()++ def handle_request(self, request: Request) -> Response:+ conn = self._pool.get_connection(request)+ try:+ response = self._send(request, conn)+ self._pool.return_connection(request, conn)+ return response+ except TimeoutException:+ raise+ except Exception as exc:+ raise ConnectError(str(exc)) from exc++ def _send(self, request: Request, conn) -> Response:+ # Simplified: in real httpx this does the actual socket I/O+ return Response(200, headers={}, content=b"", request=request)++ def close(self) -> None:+ self._pool.close()+++class AsyncHTTPTransport(AsyncBaseTransport):+ """The async variant of HTTPTransport."""++ def __init__(self, verify=True, cert=None):+ self.verify = verify+ self.cert = cert++ async def handle_async_request(self, request: Request) -> Response:+ return Response(200, headers={}, content=b"", request=request)++ async def aclose(self) -> None:+ pass+++class MockTransport(BaseTransport):+ """+ A transport for testing that returns predefined responses.+ Pass a handler function that receives a Request and returns a Response.+ """++ def __init__(self, handler):+ self.handler = handler++ def handle_request(self, request: Request) -> Response:+ return self.handler(request)+++class ProxyTransport(BaseTransport):+ """+ Routes requests through an HTTP/HTTPS proxy.+ Wraps an inner transport and prepends proxy connection handling.+ """++ def __init__(self, proxy_url: str, *, inner: BaseTransport = None):+ self.proxy_url = proxy_url+ self._inner = inner or HTTPTransport()++ def handle_request(self, request: Request) -> Response:+ try:+ return self._inner.handle_request(request)+ except TransportError:+ raise+ except Exception as exc:+ raise TransportError(f"Proxy error: {exc}") from exc++ def close(self) -> None:+ self._inner.close()diff --git a/worked/httpx/raw/utils.py b/worked/httpx/raw/utils.pynew file mode 100644index 000000000..84ca4a3b8--- /dev/null+++ b/worked/httpx/raw/utils.py@@ -0,0 +1,85 @@+"""+Utility functions shared across the library.+Small helpers that don't belong in any one module.+"""+import re+from models import Cookies+++SENSITIVE_HEADERS = {"authorization", "cookie", "set-cookie", "proxy-authorization"}+++def primitive_value_to_str(value) -> str:+ """Convert a primitive value to its string representation."""+ if isinstance(value, bool):+ return "true" if value else "false"+ return str(value)+++def normalize_header_key(key: str) -> str:+ """Convert a header key to its canonical Title-Case form."""+ return "-".join(word.capitalize() for word in key.split("-"))+++def flatten_queryparams(params: dict) -> list:+ """+ Expand a params dict into a flat list of (key, value) pairs.+ List values become multiple pairs with the same key.+ """+ result = []+ for key, value in params.items():+ if isinstance(value, list):+ for item in value:+ result.append((key, primitive_value_to_str(item)))+ else:+ result.append((key, primitive_value_to_str(value)))+ return result+++def parse_content_type(content_type: str) -> tuple:+ """+ Parse a Content-Type header value.+ Returns (media_type, params_dict).+ Example: 'application/json; charset=utf-8' -> ('application/json', {'charset': 'utf-8'})+ """+ parts = [p.strip() for p in content_type.split(";")]+ media_type = parts[0]+ params = {}+ for part in parts[1:]:+ if "=" in part:+ key, _, value = part.partition("=")+ params[key.strip()] = value.strip()+ return media_type, params+++def obfuscate_sensitive_headers(headers: dict) -> dict:+ """Return a copy of headers with sensitive values replaced by [obfuscated]."""+ return {+ k: "[obfuscated]" if k.lower() in SENSITIVE_HEADERS else v+ for k, v in headers.items()+ }+++def unset_all_cookies(cookies: Cookies) -> None:+ """Clear all cookies from a cookie jar in place."""+ cookies.clear()+++def is_known_encoding(encoding: str) -> bool:+ """Check if a character encoding label is recognized by Python's codec system."""+ import codecs+ try:+ codecs.lookup(encoding)+ return True+ except LookupError:+ return False+++def build_url_with_params(base_url: str, params: dict) -> str:+ """Append query parameters to a URL string."""+ if not params:+ return base_url+ pairs = flatten_queryparams(params)+ query = "&".join(f"{k}={v}" for k, v in pairs)+ separator = "&" if "?" in base_url else "?"+ return f"{base_url}{separator}{query}"diff --git a/worked/karpathy-repos/README.md b/worked/karpathy-repos/README.mdnew file mode 100644index 000000000..849897284--- /dev/null+++ b/worked/karpathy-repos/README.md@@ -0,0 +1,63 @@+# Karpathy Repos Benchmark — How to Reproduce++This is the corpus that produced the 71.5x token reduction benchmark.++## Corpus (52 files)++### Code — clone these 3 repos++```bash+git clone https://github.com/karpathy/nanoGPT+git clone https://github.com/karpathy/minGPT+git clone https://github.com/karpathy/micrograd+```++### Papers — download these 5 PDFs++- Attention Is All You Need — https://arxiv.org/abs/1706.03762+- FlashAttention: Fast and Memory-Efficient Exact Attention — https://arxiv.org/abs/2205.14135+- FlashAttention-2 — https://arxiv.org/abs/2307.08691+- Neural Attention Residuals — https://arxiv.org/abs/2505.03840+- NeuralWalker: Graph Neural Networks with Walk-Based Attention — https://arxiv.org/abs/2502.02593++### Images — save these 4++- `gpt2_124M_loss.png` — nanoGPT training loss curve (in the nanoGPT repo)+- `gout.svg` — micrograd computation graph (in the micrograd repo)+- `moon_mlp.png` — MLP decision boundary (in the micrograd repo)+- Any screenshot or diagram from the Attention Is All You Need paper++## How to run++Put all files into a single folder called `raw/`:++```+raw/+├── nanoGPT/ (cloned repo)+├── minGPT/ (cloned repo)+├── micrograd/ (cloned repo)+├── attention.pdf+├── flashattention.pdf+├── flashattention2.pdf+├── attn_residuals.pdf+├── neuralwalker.pdf+├── gpt2_124M_loss.png+├── gout.svg+└── moon_mlp.png+```++Then in Claude Code:++```+pip install graphifyy && graphify install+/graphify ./raw+```++## What to expect++- ~285 nodes, ~340 edges, ~17 meaningful communities+- God nodes: `Value` (micrograd), `GPT` (nanoGPT), `Training Script`, `Layer`+- Surprising connections: nanoGPT Block and minGPT Block linked across repos, FlashAttention paper bridging into CausalSelfAttention in both repos+- Token reduction: 71.5x vs reading all 52 files cold++Full eval with scores and analysis: `review.md`diff --git a/worked/mixed-corpus/README.md b/worked/mixed-corpus/README.mdnew file mode 100644index 000000000..e62c32fef--- /dev/null+++ b/worked/mixed-corpus/README.md@@ -0,0 +1,45 @@+# Mixed Corpus Benchmark — How to Reproduce++A small but realistic mixed-input corpus: Python source files, a markdown paper with+arXiv citations, and one image. Tests graphify's ability to handle different file types+in a single run.++## Corpus (5 files)++All input files are in `raw/`:++```+raw/+├── analyze.py — graphify's graph analysis module (god_nodes, surprising_connections, etc.)+├── build.py — graphify's graph builder (build_from_json, networkx wrapper)+├── cluster.py — graphify's Leiden community detection (cluster, score_all)+├── attention_notes.md — Transformer paper notes (Vaswani et al., 2017), with arXiv citation+```++Note: the original benchmark included `attention_arabic.png` (an Arabic-language figure from the+Attention paper). PNG files are not stored in this repo. To reproduce with the image, save any+diagram or figure from the Attention Is All You Need paper as `raw/attention_arabic.png`.++## How to run++```bash+pip install graphifyy && graphify install+/graphify ./raw+```++Or from the CLI directly:++```bash+pip install graphifyy+graphify ./raw+```++## What to expect++- ~20 nodes, ~19 edges from AST alone (3 Python modules)+- 3 communities: Graph Analysis, Clustering & Scoring, Graph Building+- God nodes: `analyze.py`, `cluster.py`, `build.py`+- `attention_notes.md` classified as `paper` (arXiv heuristic fires on `1706.03762`)+- If you include the image: 1 extra node describing the figure content via vision++Full eval with scores and analysis: `review.md`diff --git a/worked/mixed-corpus/raw/analyze.py b/worked/mixed-corpus/raw/analyze.pynew file mode 100644index 000000000..cf5344960--- /dev/null+++ b/worked/mixed-corpus/raw/analyze.py@@ -0,0 +1,517 @@+"""Graph analysis: god nodes (most connected), surprising connections (cross-community), suggested questions."""+from __future__ import annotations+import networkx as nx+++def _node_community_map(communities: dict[int, list[str]]) -> dict[str, int]:+ """Invert communities dict: node_id -> community_id."""+ return {n: cid for cid, nodes in communities.items() for n in nodes}+++def _is_file_node(G: nx.Graph, node_id: str) -> bool:+ """+ Return True if this node is a file-level hub node (e.g. 'client', 'models')+ or an AST method stub (e.g. '.auth_flow()', '.__init__()').++ These are synthetic nodes created by the AST extractor and should be excluded+ from god nodes, surprising connections, and knowledge gap reporting.+ """+ label = G.nodes[node_id].get("label", "")+ if not label:+ return False+ # File-level hub: label is a filename with a code extension+ if label.split(".")[-1] in ("py", "ts", "js", "go", "rs", "java", "rb", "cpp", "c", "h"):+ return True+ # Method stub: AST extractor labels methods as '.method_name()'+ if label.startswith(".") and label.endswith("()"):+ return True+ # Module-level function stub: labeled 'function_name()' - only has a contains edge+ # These are real functions but structurally isolated by definition; not a gap worth flagging+ if label.endswith("()") and G.degree(node_id) <= 1:+ return True+ return False+++def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]:+ """Return the top_n most-connected real entities - the core abstractions.++ File-level hub nodes are excluded: they accumulate import/contains edges+ mechanically and don't represent meaningful architectural abstractions.+ """+ degree = dict(G.degree())+ sorted_nodes = sorted(degree.items(), key=lambda x: x[1], reverse=True)+ result = []+ for node_id, deg in sorted_nodes:+ if _is_file_node(G, node_id) or _is_concept_node(G, node_id):+ continue+ result.append({+ "id": node_id,+ "label": G.nodes[node_id].get("label", node_id),+ "edges": deg,+ })+ if len(result) >= top_n:+ break+ return result+++def surprising_connections(+ G: nx.Graph,+ communities: dict[int, list[str]] | None = None,+ top_n: int = 5,+) -> list[dict]:+ """+ Find connections that are genuinely surprising - not obvious from file structure.++ Strategy:+ - Multi-file corpora: cross-file edges between real entities (not concept nodes).+ Sorted AMBIGUOUS → INFERRED → EXTRACTED.+ - Single-file / single-source corpora: cross-community edges that bridge+ distant parts of the graph (betweenness centrality on edges).+ These reveal non-obvious structural couplings.++ Concept nodes (empty source_file, or injected semantic annotations) are excluded+ from surprising connections because they are intentional, not discovered.+ """+ # Identify unique source files (ignore empty/null source_file)+ source_files = {+ data.get("source_file", "")+ for _, data in G.nodes(data=True)+ if data.get("source_file", "")+ }+ is_multi_source = len(source_files) > 1++ if is_multi_source:+ return _cross_file_surprises(G, communities or {}, top_n)+ else:+ return _cross_community_surprises(G, communities or {}, top_n)+++def _is_concept_node(G: nx.Graph, node_id: str) -> bool:+ """+ Return True if this node is a manually-injected semantic concept node+ rather than a real entity found in source code.++ Signals:+ - Empty source_file+ - source_file doesn't look like a real file path (no extension)+ """+ data = G.nodes[node_id]+ source = data.get("source_file", "")+ if not source:+ return True+ # Has no file extension → probably a concept label, not a real file+ if "." not in source.split("/")[-1]:+ return True+ return False+++_CODE_EXTENSIONS = {"py", "ts", "tsx", "js", "go", "rs", "java", "rb", "cpp", "c", "h", "cs", "kt", "scala", "php"}+_DOC_EXTENSIONS = {"md", "txt", "rst"}+_PAPER_EXTENSIONS = {"pdf"}+_IMAGE_EXTENSIONS = {"png", "jpg", "jpeg", "webp", "gif", "svg"}+++def _file_category(path: str) -> str:+ ext = path.rsplit(".", 1)[-1].lower() if "." in path else ""+ if ext in _CODE_EXTENSIONS:+ return "code"+ if ext in _PAPER_EXTENSIONS:+ return "paper"+ if ext in _IMAGE_EXTENSIONS:+ return "image"+ return "doc"+++def _top_level_dir(path: str) -> str:+ """Return the first path component - used to detect cross-repo edges."""+ return path.split("/")[0] if "/" in path else path+++def _surprise_score(+ G: nx.Graph,+ u: str,+ v: str,+ data: dict,+ node_community: dict[str, int],+ u_source: str,+ v_source: str,+) -> tuple[int, list[str]]:+ """Score how surprising a cross-file edge is. Returns (score, reasons)."""+ score = 0+ reasons: list[str] = []++ # 1. Confidence weight - uncertain connections are more noteworthy+ conf = data.get("confidence", "EXTRACTED")+ conf_bonus = {"AMBIGUOUS": 3, "INFERRED": 2, "EXTRACTED": 1}.get(conf, 1)+ score += conf_bonus+ if conf in ("AMBIGUOUS", "INFERRED"):+ reasons.append(f"{conf.lower()} connection - not explicitly stated in source")++ # 2. Cross file-type bonus - code↔paper or code↔image is non-obvious+ cat_u = _file_category(u_source)+ cat_v = _file_category(v_source)+ if cat_u != cat_v:+ score += 2+ reasons.append(f"crosses file types ({cat_u} ↔ {cat_v})")++ # 3. Cross-repo bonus - different top-level directory+ if _top_level_dir(u_source) != _top_level_dir(v_source):+ score += 2+ reasons.append("connects across different repos/directories")++ # 4. Cross-community bonus - Leiden says these are structurally distant+ cid_u = node_community.get(u)+ cid_v = node_community.get(v)+ if cid_u is not None and cid_v is not None and cid_u != cid_v:+ score += 1+ reasons.append("bridges separate communities")++ # 5. Peripheral→hub: a low-degree node connecting to a high-degree one+ deg_u = G.degree(u)+ deg_v = G.degree(v)+ if min(deg_u, deg_v) <= 2 and max(deg_u, deg_v) >= 5:+ score += 1+ peripheral = G.nodes[u].get("label", u) if deg_u <= 2 else G.nodes[v].get("label", v)+ hub = G.nodes[v].get("label", v) if deg_u <= 2 else G.nodes[u].get("label", u)+ reasons.append(f"peripheral node `{peripheral}` unexpectedly reaches hub `{hub}`")++ return score, reasons+++def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: int) -> list[dict]:+ """+ Cross-file edges between real code/doc entities, ranked by a composite+ surprise score rather than confidence alone.++ Surprise score accounts for:+ - Confidence (AMBIGUOUS > INFERRED > EXTRACTED)+ - Cross file-type (code↔paper is more surprising than code↔code)+ - Cross-repo (different top-level directory)+ - Cross-community (Leiden says structurally distant)+ - Peripheral→hub (low-degree node reaching a god node)++ Each result includes a 'why' field explaining what makes it non-obvious.+ """+ node_community = _node_community_map(communities)+ candidates = []++ for u, v, data in G.edges(data=True):+ relation = data.get("relation", "")+ if relation in ("imports", "imports_from", "contains", "method"):+ continue+ if _is_concept_node(G, u) or _is_concept_node(G, v):+ continue+ if _is_file_node(G, u) or _is_file_node(G, v):+ continue++ u_source = G.nodes[u].get("source_file", "")+ v_source = G.nodes[v].get("source_file", "")++ if not u_source or not v_source or u_source == v_source:+ continue++ score, reasons = _surprise_score(G, u, v, data, node_community, u_source, v_source)+ src_id = data.get("_src", u)+ tgt_id = data.get("_tgt", v)+ candidates.append({+ "_score": score,+ "source": G.nodes[src_id].get("label", src_id),+ "target": G.nodes[tgt_id].get("label", tgt_id),+ "source_files": [+ G.nodes[src_id].get("source_file", ""),+ G.nodes[tgt_id].get("source_file", ""),+ ],+ "confidence": data.get("confidence", "EXTRACTED"),+ "relation": relation,+ "why": "; ".join(reasons) if reasons else "cross-file semantic connection",+ })++ candidates.sort(key=lambda x: x["_score"], reverse=True)+ for c in candidates:+ c.pop("_score")++ if candidates:+ return candidates[:top_n]++ return _cross_community_surprises(G, communities, top_n)+++def _cross_community_surprises(+ G: nx.Graph,+ communities: dict[int, list[str]],+ top_n: int,+) -> list[dict]:+ """+ For single-source corpora: find edges that bridge different communities.+ These are surprising because Leiden grouped everything else tightly -+ these edges cut across the natural structure.++ Falls back to high-betweenness edges if no community info is provided.+ """+ if not communities:+ # No community info - use edge betweenness centrality+ if G.number_of_edges() == 0:+ return []+ betweenness = nx.edge_betweenness_centrality(G)+ top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n]+ result = []+ for (u, v), score in top_edges:+ data = G.edges[u, v]+ result.append({+ "source": G.nodes[u].get("label", u),+ "target": G.nodes[v].get("label", v),+ "source_files": [+ G.nodes[u].get("source_file", ""),+ G.nodes[v].get("source_file", ""),+ ],+ "confidence": data.get("confidence", "EXTRACTED"),+ "relation": data.get("relation", ""),+ "note": f"Bridges graph structure (betweenness={score:.3f})",+ })+ return result++ # Build node → community map+ node_community = _node_community_map(communities)++ surprises = []+ for u, v, data in G.edges(data=True):+ cid_u = node_community.get(u)+ cid_v = node_community.get(v)+ if cid_u is None or cid_v is None or cid_u == cid_v:+ continue+ # Skip file hub nodes and plain structural edges+ if _is_file_node(G, u) or _is_file_node(G, v):+ continue+ relation = data.get("relation", "")+ if relation in ("imports", "imports_from", "contains", "method"):+ continue+ # This edge crosses community boundaries - interesting+ confidence = data.get("confidence", "EXTRACTED")+ src_id = data.get("_src", u)+ tgt_id = data.get("_tgt", v)+ surprises.append({+ "source": G.nodes[src_id].get("label", src_id),+ "target": G.nodes[tgt_id].get("label", tgt_id),+ "source_files": [+ G.nodes[src_id].get("source_file", ""),+ G.nodes[tgt_id].get("source_file", ""),+ ],+ "confidence": confidence,+ "relation": relation,+ "note": f"Bridges community {cid_u} → community {cid_v}",+ "_pair": tuple(sorted([cid_u, cid_v])),+ })++ # Sort: AMBIGUOUS first, then INFERRED, then EXTRACTED+ order = {"AMBIGUOUS": 0, "INFERRED": 1, "EXTRACTED": 2}+ surprises.sort(key=lambda x: order.get(x["confidence"], 3))++ # Deduplicate by community pair - one representative edge per (A→B) boundary.+ # Without this, a single high-betweenness god node dominates all results.+ seen_pairs: set[tuple] = set()+ deduped = []+ for s in surprises:+ pair = s.pop("_pair")+ if pair not in seen_pairs:+ seen_pairs.add(pair)+ deduped.append(s)+ return deduped[:top_n]+++def suggest_questions(+ G: nx.Graph,+ communities: dict[int, list[str]],+ community_labels: dict[int, str],+ top_n: int = 7,+) -> list[dict]:+ """+ Generate questions the graph is uniquely positioned to answer.+ Based on: AMBIGUOUS edges, bridge nodes, underexplored god nodes, isolated nodes.+ Each question has a 'type', 'question', and 'why' field.+ """+ questions = []+ node_community = _node_community_map(communities)++ # 1. AMBIGUOUS edges → unresolved relationship questions+ for u, v, data in G.edges(data=True):+ if data.get("confidence") == "AMBIGUOUS":+ ul = G.nodes[u].get("label", u)+ vl = G.nodes[v].get("label", v)+ relation = data.get("relation", "related to")+ questions.append({+ "type": "ambiguous_edge",+ "question": f"What is the exact relationship between `{ul}` and `{vl}`?",+ "why": f"Edge tagged AMBIGUOUS (relation: {relation}) - confidence is low.",+ })++ # 2. Bridge nodes (high betweenness) → cross-cutting concern questions+ if G.number_of_edges() > 0:+ betweenness = nx.betweenness_centrality(G)+ # Top bridge nodes that are NOT file-level hubs+ bridges = sorted(+ [(n, s) for n, s in betweenness.items()+ if not _is_file_node(G, n) and not _is_concept_node(G, n) and s > 0],+ key=lambda x: x[1],+ reverse=True,+ )[:3]+ for node_id, score in bridges:+ label = G.nodes[node_id].get("label", node_id)+ cid = node_community.get(node_id)+ comm_label = community_labels.get(cid, f"Community {cid}") if cid is not None else "unknown"+ neighbors = list(G.neighbors(node_id))+ neighbor_comms = {node_community.get(n) for n in neighbors if node_community.get(n) != cid}+ if neighbor_comms:+ other_labels = [community_labels.get(c, f"Community {c}") for c in neighbor_comms]+ questions.append({+ "type": "bridge_node",+ "question": f"Why does `{label}` connect `{comm_label}` to {', '.join(f'`{l}`' for l in other_labels)}?",+ "why": f"High betweenness centrality ({score:.3f}) - this node is a cross-community bridge.",+ })++ # 3. God nodes with many INFERRED edges → verification questions+ degree = dict(G.degree())+ top_nodes = sorted(+ [(n, d) for n, d in degree.items() if not _is_file_node(G, n)],+ key=lambda x: x[1],+ reverse=True,+ )[:5]+ for node_id, _ in top_nodes:+ inferred = [+ (u, v, d) for u, v, d in G.edges(node_id, data=True)+ if d.get("confidence") == "INFERRED"+ ]+ if len(inferred) >= 2:+ label = G.nodes[node_id].get("label", node_id)+ # Use _src/_tgt to get the correct direction; fall back to v (the other node)+ others = []+ for u, v, d in inferred[:2]:+ src_id = d.get("_src", u)+ tgt_id = d.get("_tgt", v)+ other_id = tgt_id if src_id == node_id else src_id+ others.append(G.nodes[other_id].get("label", other_id))+ questions.append({+ "type": "verify_inferred",+ "question": f"Are the {len(inferred)} inferred relationships involving `{label}` (e.g. with `{others[0]}` and `{others[1]}`) actually correct?",+ "why": f"`{label}` has {len(inferred)} INFERRED edges - model-reasoned connections that need verification.",+ })++ # 4. Isolated or weakly-connected nodes → exploration questions+ isolated = [+ n for n in G.nodes()+ if G.degree(n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n)+ ]+ if isolated:+ labels = [G.nodes[n].get("label", n) for n in isolated[:3]]+ questions.append({+ "type": "isolated_nodes",+ "question": f"What connects {', '.join(f'`{l}`' for l in labels)} to the rest of the system?",+ "why": f"{len(isolated)} weakly-connected nodes found - possible documentation gaps or missing edges.",+ })++ # 5. Low-cohesion communities → structural questions+ from .cluster import cohesion_score+ for cid, nodes in communities.items():+ score = cohesion_score(G, nodes)+ if score < 0.15 and len(nodes) >= 5:+ label = community_labels.get(cid, f"Community {cid}")+ questions.append({+ "type": "low_cohesion",+ "question": f"Should `{label}` be split into smaller, more focused modules?",+ "why": f"Cohesion score {score} - nodes in this community are weakly interconnected.",+ })++ if not questions:+ return [{+ "type": "no_signal",+ "question": None,+ "why": (+ "Not enough signal to generate questions. "+ "This usually means the corpus has no AMBIGUOUS edges, no bridge nodes, "+ "no INFERRED relationships, and all communities are tightly cohesive. "+ "Add more files or run with --mode deep to extract richer edges."+ ),+ }]++ return questions[:top_n]+++def graph_diff(G_old: nx.Graph, G_new: nx.Graph) -> dict:+ """Compare two graph snapshots and return what changed.++ Returns:+ {+ "new_nodes": [{"id": ..., "label": ...}],+ "removed_nodes": [{"id": ..., "label": ...}],+ "new_edges": [{"source": ..., "target": ..., "relation": ..., "confidence": ...}],+ "removed_edges": [...],+ "summary": "3 new nodes, 5 new edges, 1 node removed"+ }+ """+ old_nodes = set(G_old.nodes())+ new_nodes = set(G_new.nodes())++ added_node_ids = new_nodes - old_nodes+ removed_node_ids = old_nodes - new_nodes++ new_nodes_list = [+ {"id": n, "label": G_new.nodes[n].get("label", n)}+ for n in added_node_ids+ ]+ removed_nodes_list = [+ {"id": n, "label": G_old.nodes[n].get("label", n)}+ for n in removed_node_ids+ ]++ def edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple:+ return (u, v, data.get("relation", ""))++ old_edge_keys = {+ edge_key(G_old, u, v, d)+ for u, v, d in G_old.edges(data=True)+ }+ new_edge_keys = {+ edge_key(G_new, u, v, d)+ for u, v, d in G_new.edges(data=True)+ }++ added_edge_keys = new_edge_keys - old_edge_keys+ removed_edge_keys = old_edge_keys - new_edge_keys++ new_edges_list = []+ for u, v, d in G_new.edges(data=True):+ if edge_key(G_new, u, v, d) in added_edge_keys:+ new_edges_list.append({+ "source": u,+ "target": v,+ "relation": d.get("relation", ""),+ "confidence": d.get("confidence", ""),+ })++ removed_edges_list = []+ for u, v, d in G_old.edges(data=True):+ if edge_key(G_old, u, v, d) in removed_edge_keys:+ removed_edges_list.append({+ "source": u,+ "target": v,+ "relation": d.get("relation", ""),+ "confidence": d.get("confidence", ""),+ })++ parts = []+ if new_nodes_list:+ parts.append(f"{len(new_nodes_list)} new node{'s' if len(new_nodes_list) != 1 else ''}")+ if new_edges_list:+ parts.append(f"{len(new_edges_list)} new edge{'s' if len(new_edges_list) != 1 else ''}")+ if removed_nodes_list:+ parts.append(f"{len(removed_nodes_list)} node{'s' if len(removed_nodes_list) != 1 else ''} removed")+ if removed_edges_list:+ parts.append(f"{len(removed_edges_list)} edge{'s' if len(removed_edges_list) != 1 else ''} removed")+ summary = ", ".join(parts) if parts else "no changes"++ return {+ "new_nodes": new_nodes_list,+ "removed_nodes": removed_nodes_list,+ "new_edges": new_edges_list,+ "removed_edges": removed_edges_list,+ "summary": summary,+ }diff --git a/worked/mixed-corpus/raw/attention_notes.md b/worked/mixed-corpus/raw/attention_notes.mdnew file mode 100644index 000000000..6a60166f8--- /dev/null+++ b/worked/mixed-corpus/raw/attention_notes.md@@ -0,0 +1,76 @@+# Attention Mechanism Notes++Notes on the Transformer architecture from Vaswani et al., 2017.+arXiv: 1706.03762++## Abstract++The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder. The best performing models also connect the encoder and decoder through an attention mechanism. The Transformer is a model architecture eschewing recurrence and instead relying entirely on an attention mechanism to draw global dependencies between input and output.++## Multi-Head Attention++The model uses h=8 parallel attention heads. For each head, d_k = d_v = d_model/h = 64.++Scaled dot-product attention:++ Attention(Q, K, V) = softmax(QK^T / sqrt(d_k)) V++Multi-head attention runs h attention functions in parallel, then concatenates and projects:++ MultiHead(Q, K, V) = Concat(head_1, ..., head_h) W^O+ head_i = Attention(Q W_i^Q, K W_i^K, V W_i^V)++The scaling by sqrt(d_k) prevents the dot products from growing large in magnitude, which would push the softmax into regions with very small gradients.++## Architecture++The Transformer uses a stacked encoder-decoder structure.++Encoder: 6 identical layers, each with two sublayers:+1. Multi-head self-attention+2. Position-wise fully connected feed-forward network++Each sublayer uses a residual connection followed by layer normalization:+ output = LayerNorm(x + Sublayer(x))++Decoder: 6 identical layers, each with three sublayers:+1. Masked multi-head self-attention (prevents positions from attending to subsequent positions)+2. Multi-head attention over encoder output+3. Position-wise feed-forward network++d_model = 512 for all sublayers and embedding layers.+Feed-forward inner dimension = 2048.++## Positional Encoding++Since the model contains no recurrence and no convolution, positional encodings are added to the input embeddings to give the model information about the relative position of tokens:++ PE(pos, 2i) = sin(pos / 10000^(2i/d_model))+ PE(pos, 2i+1) = cos(pos / 10000^(2i/d_model))++This allows the model to easily learn to attend by relative positions.++## Why attention over recurrence++Three main advantages:+1. Total computational complexity per layer is lower for self-attention when sequence length is smaller than representation dimensionality+2. Computations that can be parallelized — recurrent layers require O(n) sequential operations+3. Path length between long-range dependencies is O(1) for self-attention vs O(n) for recurrence++## Results++WMT 2014 English-to-German: 28.4 BLEU, outperforming all previously published results by over 2 BLEU.+WMT 2014 English-to-French: 41.0 BLEU, new state of the art.+Training cost: 3.5 days on 8 P100 GPUs.++## Open questions++[1] Does the choice of h=8 heads generalize, or is it architecture-specific?+[2] The scaling factor sqrt(d_k) is justified empirically — is there a theoretical justification?+[3] How does learned positional encoding compare to sinusoidal at longer sequence lengths?++## References++[1] Vaswani, A., Shazeer, N., Parmar, N., et al. (2017). Attention Is All You Need. arXiv:1706.03762+[2] Ba, J., Kiros, J., Hinton, G. (2016). Layer Normalization. arXiv:1607.06450+[3] He, K., et al. (2016). Deep Residual Learning for Image Recognition. CVPR 2016.diff --git a/worked/mixed-corpus/raw/build.py b/worked/mixed-corpus/raw/build.pynew file mode 100644index 000000000..655820c04--- /dev/null+++ b/worked/mixed-corpus/raw/build.py@@ -0,0 +1,39 @@+# assemble node+edge dicts into a NetworkX graph, preserving edge direction+from __future__ import annotations+import sys+import networkx as nx+from .validate import validate_extraction+++def build_from_json(extraction: dict) -> nx.Graph:+ errors = validate_extraction(extraction)+ # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors.+ real_errors = [e for e in errors if "does not match any node id" not in e]+ if real_errors:+ print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr)+ G = nx.Graph()+ for node in extraction.get("nodes", []):+ G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"})+ node_set = set(G.nodes())+ for edge in extraction.get("edges", []):+ src, tgt = edge["source"], edge["target"]+ if src not in node_set or tgt not in node_set:+ continue # skip edges to external/stdlib nodes - expected, not an error+ attrs = {k: v for k, v in edge.items() if k not in ("source", "target")}+ # Preserve original edge direction - undirected graphs lose it otherwise,+ # causing display functions to show edges backwards.+ attrs["_src"] = src+ attrs["_tgt"] = tgt+ G.add_edge(src, tgt, **attrs)+ return G+++def build(extractions: list[dict]) -> nx.Graph:+ """Merge multiple extraction results into one graph."""+ combined: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0}+ for ext in extractions:+ combined["nodes"].extend(ext.get("nodes", []))+ combined["edges"].extend(ext.get("edges", []))+ combined["input_tokens"] += ext.get("input_tokens", 0)+ combined["output_tokens"] += ext.get("output_tokens", 0)+ return build_from_json(combined)diff --git a/worked/mixed-corpus/raw/cluster.py b/worked/mixed-corpus/raw/cluster.pynew file mode 100644index 000000000..b5c97b7c8--- /dev/null+++ b/worked/mixed-corpus/raw/cluster.py@@ -0,0 +1,104 @@+"""Leiden community detection on NetworkX graphs. Splits oversized communities. Returns cohesion scores."""+from __future__ import annotations+import networkx as nx+++def build_graph(nodes: list[dict], edges: list[dict]) -> nx.Graph:+ """Build a NetworkX graph from graphify node/edge dicts.++ Preserves original edge direction as _src/_tgt attributes so that+ display functions can show relationships in the correct direction,+ even though the graph is undirected for structural analysis.+ """+ G = nx.Graph()+ for n in nodes:+ G.add_node(n["id"], **{k: v for k, v in n.items() if k != "id"})+ for e in edges:+ attrs = {k: v for k, v in e.items() if k not in ("source", "target")}+ attrs["_src"] = e["source"]+ attrs["_tgt"] = e["target"]+ G.add_edge(e["source"], e["target"], **attrs)+ return G++_MAX_COMMUNITY_FRACTION = 0.25 # communities larger than 25% of graph get split+_MIN_SPLIT_SIZE = 10 # only split if community has at least this many nodes+++def cluster(G: nx.Graph) -> dict[int, list[str]]:+ """Run Leiden community detection. Returns {community_id: [node_ids]}.++ Community IDs are stable across runs: 0 = largest community after splitting.+ Oversized communities (> 25% of graph nodes, min 10) are split by running+ a second Leiden pass on the subgraph.+ """+ if G.number_of_nodes() == 0:+ return {}+ if G.number_of_edges() == 0:+ return {i: [n] for i, n in enumerate(sorted(G.nodes))}++ from graspologic.partition import leiden # lazy - avoids 15s numba JIT on import++ # Leiden warns and drops isolates - handle them separately+ isolates = [n for n in G.nodes() if G.degree(n) == 0]+ connected_nodes = [n for n in G.nodes() if G.degree(n) > 0]+ connected = G.subgraph(connected_nodes)++ raw: dict[int, list[str]] = {}+ if connected.number_of_nodes() > 0:+ partition: dict[str, int] = leiden(connected)+ for node, cid in partition.items():+ raw.setdefault(cid, []).append(node)++ # Each isolate becomes its own single-node community+ next_cid = max(raw.keys(), default=-1) + 1+ for node in isolates:+ raw[next_cid] = [node]+ next_cid += 1++ # Split oversized communities+ max_size = max(_MIN_SPLIT_SIZE, int(G.number_of_nodes() * _MAX_COMMUNITY_FRACTION))+ final_communities: list[list[str]] = []+ for nodes in raw.values():+ if len(nodes) > max_size:+ final_communities.extend(_split_community(G, nodes))+ else:+ final_communities.append(nodes)++ # Re-index by size descending for deterministic ordering+ final_communities.sort(key=len, reverse=True)+ return {i: sorted(nodes) for i, nodes in enumerate(final_communities)}+++def _split_community(G: nx.Graph, nodes: list[str]) -> list[list[str]]:+ """Run a second Leiden pass on a community subgraph to split it further."""+ subgraph = G.subgraph(nodes)+ if subgraph.number_of_edges() == 0:+ # No edges - split into individual nodes+ return [[n] for n in sorted(nodes)]+ try:+ from graspologic.partition import leiden+ sub_partition: dict[str, int] = leiden(subgraph)+ sub_communities: dict[int, list[str]] = {}+ for node, cid in sub_partition.items():+ sub_communities.setdefault(cid, []).append(node)+ if len(sub_communities) <= 1:+ # Leiden couldn't split it - return as-is+ return [sorted(nodes)]+ return [sorted(v) for v in sub_communities.values()]+ except Exception:+ return [sorted(nodes)]+++def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float:+ """Ratio of actual intra-community edges to maximum possible."""+ n = len(community_nodes)+ if n <= 1:+ return 1.0+ subgraph = G.subgraph(community_nodes)+ actual = subgraph.number_of_edges()+ possible = n * (n - 1) / 2+ return round(actual / possible, 2) if possible > 0 else 0.0+++def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]:+ return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()}