Skip to content

Turn any codebase, with its docs, SQL schemas, configs, and PDFs, into a queryable knowledge graph. A /graphify skill for Claude Code, Cursor, Codex, and Gemini CLI: local deterministic AST parsing, every edge explained, no vector store.

add reproducible worked examples for httpx, karpathy-repos, and mixed-corpus benchmarks

Safi0fee55726d66
diff --git a/worked/httpx/README.md b/worked/httpx/README.md
new file mode 100644
index 000000000..3d1c924d1
--- /dev/null
+++ b/worked/httpx/README.md
@@ -0,0 +1,42 @@
+# httpx Corpus Benchmark — How to Reproduce
+
+A synthetic 6-file Python codebase modeled after httpx's architecture. Tests graphify
+on a realistic library codebase with clean layering: exceptions → models → auth/transport → client.
+
+## Corpus (6 files)
+
+All input files are in `raw/`:
+
+```
+raw/
+├── exceptions.py — full HTTPError hierarchy (RequestError, TransportError, HTTPStatusError, etc.)
+├── models.py — URL, Headers, Cookies, Request, Response with raise_for_status
+├── auth.py — BasicAuth, BearerAuth, DigestAuth (challenge-response), NetRCAuth
+├── utils.py — header normalization, query param flattening, content-type parsing
+├── transport.py — ConnectionPool, HTTPTransport, AsyncHTTPTransport, MockTransport, ProxyTransport
+└── client.py — Timeout, Limits, BaseClient, Client (sync), AsyncClient
+```
+
+## How to run
+
+```bash
+pip install graphifyy && graphify install
+/graphify ./raw
+```
+
+Or from the CLI directly:
+
+```bash
+pip install graphifyy
+graphify ./raw
+```
+
+## What to expect
+
+- ~95 nodes, ~130 edges
+- 4 communities: Exception Hierarchy, Models & Data, Auth & Transport, Client Layer
+- God nodes: `client.py`, `models.py`, `transport.py`, `exceptions.py`, `BaseClient`, `Response`
+- Surprising connections: `DigestAuth` ↔ `Response` (auth.py reads Response to parse WWW-Authenticate)
+- All edges EXTRACTED — no inference needed, dependency graph is explicit
+
+Full eval with scores and analysis: `review.md`
diff --git a/worked/httpx/raw/auth.py b/worked/httpx/raw/auth.py
new file mode 100644
index 000000000..290cadd3e
--- /dev/null
+++ b/worked/httpx/raw/auth.py
@@ -0,0 +1,114 @@
+"""
+Authentication handlers.
+Auth objects are callables that modify a request before it is sent.
+DigestAuth is the most interesting: it participates in a full request/response cycle,
+reading the 401 response to build the challenge before re-sending.
+"""
+import hashlib
+import time
+from models import Request, Response
+
+
+class Auth:
+ """Base class for all authentication handlers."""
+
+ def auth_flow(self, request: Request):
+ """Modify the request. May yield to inspect the response."""
+ raise NotImplementedError
+
+
+class BasicAuth(Auth):
+ """HTTP Basic Authentication."""
+
+ def __init__(self, username: str, password: str):
+ self.username = username
+ self.password = password
+
+ def auth_flow(self, request: Request):
+ import base64
+ credentials = f"{self.username}:{self.password}".encode()
+ encoded = base64.b64encode(credentials).decode()
+ request.headers["Authorization"] = f"Basic {encoded}"
+ yield request
+
+
+class BearerAuth(Auth):
+ """Bearer token authentication."""
+
+ def __init__(self, token: str):
+ self.token = token
+
+ def auth_flow(self, request: Request):
+ request.headers["Authorization"] = f"Bearer {self.token}"
+ yield request
+
+
+class DigestAuth(Auth):
+ """
+ HTTP Digest Authentication.
+ Requires a full request/response cycle: sends the initial request,
+ reads the 401 WWW-Authenticate header, then re-sends with credentials.
+ This is the only auth handler that reads from Response.
+ """
+
+ def __init__(self, username: str, password: str):
+ self.username = username
+ self.password = password
+ self._nonce_count = 0
+
+ def auth_flow(self, request: Request):
+ yield request # first attempt, no credentials
+
+ # This handler must inspect the Response to continue
+ response = yield
+
+ if response.status_code == 401:
+ challenge = self._parse_challenge(response)
+ credentials = self._build_credentials(request, challenge)
+ request.headers["Authorization"] = credentials
+ yield request
+
+ def _parse_challenge(self, response: Response) -> dict:
+ """Extract digest parameters from the WWW-Authenticate header."""
+ header = response.headers.get("www-authenticate", "")
+ params = {}
+ for part in header.replace("Digest ", "").split(","):
+ if "=" in part:
+ key, _, value = part.strip().partition("=")
+ params[key.strip()] = value.strip().strip('"')
+ return params
+
+ def _build_credentials(self, request: Request, challenge: dict) -> str:
+ """Compute the Authorization header value for a digest challenge."""
+ self._nonce_count += 1
+ nc = f"{self._nonce_count:08x}"
+ cnonce = hashlib.md5(str(time.time()).encode()).hexdigest()[:8]
+ realm = challenge.get("realm", "")
+ nonce = challenge.get("nonce", "")
+
+ ha1 = hashlib.md5(f"{self.username}:{realm}:{self.password}".encode()).hexdigest()
+ ha2 = hashlib.md5(f"{request.method}:{request.url.path}".encode()).hexdigest()
+ response_hash = hashlib.md5(f"{ha1}:{nonce}:{nc}:{cnonce}:auth:{ha2}".encode()).hexdigest()
+
+ return (
+ f'Digest username="{self.username}", realm="{realm}", '
+ f'nonce="{nonce}", uri="{request.url.path}", '
+ f'nc={nc}, cnonce="{cnonce}", response="{response_hash}"'
+ )
+
+
+class NetRCAuth(Auth):
+ """Load credentials from ~/.netrc based on the request host."""
+
+ def auth_flow(self, request: Request):
+ import netrc
+ try:
+ credentials = netrc.netrc().authenticators(request.url.host)
+ if credentials:
+ username, _, password = credentials
+ basic = BasicAuth(username, password)
+ yield from basic.auth_flow(request)
+ return
+ except Exception:
+ pass
+ yield request
diff --git a/worked/httpx/raw/client.py b/worked/httpx/raw/client.py
new file mode 100644
index 000000000..d506dd613
--- /dev/null
+++ b/worked/httpx/raw/client.py
@@ -0,0 +1,161 @@
+"""
+The main Client and AsyncClient classes.
+BaseClient holds all shared logic. Client and AsyncClient extend it for sync/async.
+This is the integration hub of the library - it imports from every other module.
+"""
+from models import Request, Response, URL, Headers, Cookies
+from auth import Auth, BasicAuth
+from transport import BaseTransport, HTTPTransport, AsyncHTTPTransport
+from exceptions import TooManyRedirects, InvalidURL
+from utils import build_url_with_params, obfuscate_sensitive_headers
+
+
+DEFAULT_MAX_REDIRECTS = 20
+
+
+class Timeout:
+ def __init__(self, timeout=5.0, *, connect=None, read=None, write=None, pool=None):
+ self.connect = connect or timeout
+ self.read = read or timeout
+ self.write = write or timeout
+ self.pool = pool or timeout
+
+
+class Limits:
+ def __init__(self, max_connections=100, max_keepalive_connections=20, keepalive_expiry=5.0):
+ self.max_connections = max_connections
+ self.max_keepalive_connections = max_keepalive_connections
+ self.keepalive_expiry = keepalive_expiry
+
+
+class BaseClient:
+ """
+ Shared implementation for Client and AsyncClient.
+ Handles auth, redirects, cookies, and header defaults.
+ """
+
+ def __init__(
+ self,
+ *,
+ auth=None,
+ headers=None,
+ cookies=None,
+ timeout=Timeout(),
+ max_redirects=DEFAULT_MAX_REDIRECTS,
+ base_url="",
+ ):
+ self._auth = auth
+ self._headers = Headers(headers or {})
+ self._cookies = Cookies(cookies or {})
+ self._timeout = timeout
+ self._max_redirects = max_redirects
+ self._base_url = URL(base_url) if base_url else None
+
+ def _build_request(self, method: str, url: str, **kwargs) -> Request:
+ if self._base_url:
+ url = self._base_url.raw.rstrip("/") + "/" + url.lstrip("/")
+ if kwargs.get("params"):
+ url = build_url_with_params(url, kwargs.pop("params"))
+ headers = Headers(kwargs.get("headers", {}))
+ for k, v in self._headers.items():
+ if k not in headers:
+ headers[k] = v
+ return Request(method, url, headers=headers, content=kwargs.get("content"), cookies=self._cookies)
+
+ def _merge_cookies(self, response: Response) -> None:
+ for name, value in response.cookies.items():
+ self._cookies.set(name, value)
+
+
+class Client(BaseClient):
+ """Synchronous HTTP client."""
+
+ def __init__(self, *, transport: BaseTransport = None, **kwargs):
+ super().__init__(**kwargs)
+ self._transport = transport or HTTPTransport()
+
+ def request(self, method: str, url: str, **kwargs) -> Response:
+ request = self._build_request(method, url, **kwargs)
+ auth = kwargs.get("auth") or self._auth
+ if auth:
+ flow = auth.auth_flow(request)
+ request = next(flow)
+ response = self._transport.handle_request(request)
+ self._merge_cookies(response)
+ if auth:
+ try:
+ flow.send(response)
+ except StopIteration:
+ pass
+ return response
+
+ def get(self, url: str, **kwargs) -> Response:
+ return self.request("GET", url, **kwargs)
+
+ def post(self, url: str, **kwargs) -> Response:
+ return self.request("POST", url, **kwargs)
+
+ def put(self, url: str, **kwargs) -> Response:
+ return self.request("PUT", url, **kwargs)
+
+ def patch(self, url: str, **kwargs) -> Response:
+ return self.request("PATCH", url, **kwargs)
+
+ def delete(self, url: str, **kwargs) -> Response:
+ return self.request("DELETE", url, **kwargs)
+
+ def head(self, url: str, **kwargs) -> Response:
+ return self.request("HEAD", url, **kwargs)
+
+ def send(self, request: Request) -> Response:
+ return self._transport.handle_request(request)
+
+ def close(self) -> None:
+ self._transport.close()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.close()
+
+
+class AsyncClient(BaseClient):
+ """Asynchronous HTTP client."""
+
+ def __init__(self, *, transport=None, **kwargs):
+ super().__init__(**kwargs)
+ self._transport = transport or AsyncHTTPTransport()
+
+ async def request(self, method: str, url: str, **kwargs) -> Response:
+ request = self._build_request(method, url, **kwargs)
+ response = await self._transport.handle_async_request(request)
+ self._merge_cookies(response)
+ return response
+
+ async def get(self, url: str, **kwargs) -> Response:
+ return await self.request("GET", url, **kwargs)
+
+ async def post(self, url: str, **kwargs) -> Response:
+ return await self.request("POST", url, **kwargs)
+
+ async def put(self, url: str, **kwargs) -> Response:
+ return await self.request("PUT", url, **kwargs)
+
+ async def patch(self, url: str, **kwargs) -> Response:
+ return await self.request("PATCH", url, **kwargs)
+
+ async def delete(self, url: str, **kwargs) -> Response:
+ return await self.request("DELETE", url, **kwargs)
+
+ async def send(self, request: Request) -> Response:
+ return await self._transport.handle_async_request(request)
+
+ async def aclose(self) -> None:
+ await self._transport.aclose()
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *args):
+ await self.aclose()
diff --git a/worked/httpx/raw/exceptions.py b/worked/httpx/raw/exceptions.py
new file mode 100644
index 000000000..ff5392fee
--- /dev/null
+++ b/worked/httpx/raw/exceptions.py
@@ -0,0 +1,90 @@
+"""
+httpx-like exception hierarchy.
+All exceptions inherit from HTTPError at the top.
+"""
+
+
+class HTTPError(Exception):
+ """Base class for all httpx exceptions."""
+ def __init__(self, message, *, request=None):
+ self.request = request
+ super().__init__(message)
+
+
+class RequestError(HTTPError):
+ """An error occurred while issuing a request."""
+
+
+class TransportError(RequestError):
+ """An error occurred at the transport layer."""
+
+
+class TimeoutException(TransportError):
+ """A timeout occurred."""
+
+
+class ConnectTimeout(TimeoutException):
+ """Timed out while connecting to the host."""
+
+
+class ReadTimeout(TimeoutException):
+ """Timed out while receiving data from the host."""
+
+
+class WriteTimeout(TimeoutException):
+ """Timed out while sending data to the host."""
+
+
+class PoolTimeout(TimeoutException):
+ """Timed out waiting to acquire a connection from the pool."""
+
+
+class NetworkError(TransportError):
+ """A network error occurred."""
+
+
+class ConnectError(NetworkError):
+ """Failed to establish a connection."""
+
+
+class ReadError(NetworkError):
+ """Failed to receive data from the network."""
+
+
+class WriteError(NetworkError):
+ """Failed to send data through the network."""
+
+
+class CloseError(NetworkError):
+ """Failed to close a connection."""
+
+
+class ProxyError(TransportError):
+ """An error occurred while establishing a proxy connection."""
+
+
+class ProtocolError(TransportError):
+ """A protocol was violated."""
+
+
+class DecodingError(RequestError):
+ """Decoding of the response failed."""
+
+
+class TooManyRedirects(RequestError):
+ """Too many redirects."""
+
+
+class HTTPStatusError(HTTPError):
+ """A 4xx or 5xx response was received."""
+ def __init__(self, message, *, request, response):
+ self.response = response
+ super().__init__(message, request=request)
+
+
+class InvalidURL(Exception):
+ """URL is improperly formed or cannot be parsed."""
+
+
+class CookieConflict(Exception):
+ """Attempted to look up a cookie by name but multiple cookies exist."""
diff --git a/worked/httpx/raw/models.py b/worked/httpx/raw/models.py
new file mode 100644
index 000000000..80582b6fa
--- /dev/null
+++ b/worked/httpx/raw/models.py
@@ -0,0 +1,120 @@
+"""
+Core data models: URL, Headers, Cookies, Request, Response.
+These are the central data types that everything else in the library references.
+"""
+import json as _json
+from exceptions import HTTPStatusError
+
+
+class URL:
+ def __init__(self, url: str):
+ self.raw = url
+ self.scheme, _, rest = url.partition("://")
+ self.host, _, self.path = rest.partition("/")
+ self.path = "/" + self.path
+
+ def copy_with(self, **kwargs) -> "URL":
+ return URL(kwargs.get("url", self.raw))
+
+ def __str__(self):
+ return self.raw
+
+ def __repr__(self):
+ return f"URL({self.raw!r})"
+
+
+class Headers:
+ def __init__(self, headers=None):
+ self._store = {}
+ for k, v in (headers or {}).items():
+ self._store[k.lower()] = v
+
+ def get(self, key: str, default=None):
+ return self._store.get(key.lower(), default)
+
+ def items(self):
+ return self._store.items()
+
+ def __setitem__(self, key, value):
+ self._store[key.lower()] = value
+
+ def __getitem__(self, key):
+ return self._store[key.lower()]
+
+ def __contains__(self, key):
+ return key.lower() in self._store
+
+
+class Cookies:
+ def __init__(self, cookies=None):
+ self._jar = dict(cookies or {})
+
+ def set(self, name: str, value: str, domain: str = "") -> None:
+ self._jar[name] = value
+
+ def get(self, name: str, default=None):
+ return self._jar.get(name, default)
+
+ def delete(self, name: str) -> None:
+ self._jar.pop(name, None)
+
+ def clear(self) -> None:
+ self._jar.clear()
+
+ def items(self):
+ return self._jar.items()
+
+
+class Request:
+ def __init__(self, method: str, url, *, headers=None, content=None, cookies=None):
+ self.method = method.upper()
+ self.url = URL(url) if isinstance(url, str) else url
+ self.headers = Headers(headers)
+ self.content = content or b""
+ self.cookies = Cookies(cookies)
+
+ def __repr__(self):
+ return f"<Request [{self.method}]>"
+
+
+class Response:
+ def __init__(self, status_code: int, *, headers=None, content=None, request=None):
+ self.status_code = status_code
+ self.headers = Headers(headers)
+ self.content = content or b""
+ self.request = request
+
+ @property
+ def text(self) -> str:
+ return self.content.decode("utf-8", errors="replace")
+
+ def json(self):
+ return _json.loads(self.content)
+
+ def read(self) -> bytes:
+ return self.content
+
+ @property
+ def is_success(self) -> bool:
+ return 200 <= self.status_code < 300
+
+ @property
+ def is_error(self) -> bool:
+ return self.status_code >= 400
+
+ def raise_for_status(self) -> None:
+ if self.is_error:
+ message = f"{self.status_code} Error"
+ raise HTTPStatusError(message, request=self.request, response=self)
+
+ @property
+ def cookies(self) -> Cookies:
+ jar = Cookies()
+ for header in self.headers.get("set-cookie", "").split(","):
+ if "=" in header:
+ name, _, value = header.strip().partition("=")
+ jar.set(name.strip(), value.split(";")[0].strip())
+ return jar
+
+ def __repr__(self):
+ return f"<Response [{self.status_code}]>"
diff --git a/worked/httpx/raw/transport.py b/worked/httpx/raw/transport.py
new file mode 100644
index 000000000..5bd9b9166
--- /dev/null
+++ b/worked/httpx/raw/transport.py
@@ -0,0 +1,135 @@
+"""
+Transport layer: connection management and low-level HTTP sending.
+HTTPTransport wraps a connection pool. ProxyTransport sits in front of it.
+MockTransport is used in tests.
+"""
+from models import Request, Response
+from exceptions import TransportError, ConnectError, TimeoutException
+
+
+class BaseTransport:
+ """Sync transport interface."""
+
+ def handle_request(self, request: Request) -> Response:
+ raise NotImplementedError
+
+ def close(self) -> None:
+ pass
+
+
+class AsyncBaseTransport:
+ """Async transport interface."""
+
+ async def handle_async_request(self, request: Request) -> Response:
+ raise NotImplementedError
+
+ async def aclose(self) -> None:
+ pass
+
+
+class ConnectionPool:
+ """
+ Manages a pool of persistent HTTP connections.
+ Keys connections by (scheme, host, port).
+ """
+
+ def __init__(self, max_connections=100, max_keepalive_connections=20):
+ self.max_connections = max_connections
+ self.max_keepalive_connections = max_keepalive_connections
+ self._pool = {}
+
+ def _get_connection_key(self, request: Request) -> tuple:
+ url = request.url
+ port = 443 if url.scheme == "https" else 80
+ return (url.scheme, url.host, port)
+
+ def get_connection(self, request: Request):
+ key = self._get_connection_key(request)
+ return self._pool.get(key)
+
+ def return_connection(self, request: Request, conn) -> None:
+ key = self._get_connection_key(request)
+ if len(self._pool) < self.max_keepalive_connections:
+ self._pool[key] = conn
+
+ def close(self) -> None:
+ self._pool.clear()
+
+
+class HTTPTransport(BaseTransport):
+ """
+ The main sync HTTP transport.
+ Uses a ConnectionPool for connection reuse.
+ """
+
+ def __init__(self, verify=True, cert=None, limits=None):
+ self.verify = verify
+ self.cert = cert
+ self._pool = ConnectionPool()
+
+ def handle_request(self, request: Request) -> Response:
+ conn = self._pool.get_connection(request)
+ try:
+ response = self._send(request, conn)
+ self._pool.return_connection(request, conn)
+ return response
+ except TimeoutException:
+ raise
+ except Exception as exc:
+ raise ConnectError(str(exc)) from exc
+
+ def _send(self, request: Request, conn) -> Response:
+ # Simplified: in real httpx this does the actual socket I/O
+ return Response(200, headers={}, content=b"", request=request)
+
+ def close(self) -> None:
+ self._pool.close()
+
+
+class AsyncHTTPTransport(AsyncBaseTransport):
+ """The async variant of HTTPTransport."""
+
+ def __init__(self, verify=True, cert=None):
+ self.verify = verify
+ self.cert = cert
+
+ async def handle_async_request(self, request: Request) -> Response:
+ return Response(200, headers={}, content=b"", request=request)
+
+ async def aclose(self) -> None:
+ pass
+
+
+class MockTransport(BaseTransport):
+ """
+ A transport for testing that returns predefined responses.
+ Pass a handler function that receives a Request and returns a Response.
+ """
+
+ def __init__(self, handler):
+ self.handler = handler
+
+ def handle_request(self, request: Request) -> Response:
+ return self.handler(request)
+
+
+class ProxyTransport(BaseTransport):
+ """
+ Routes requests through an HTTP/HTTPS proxy.
+ Wraps an inner transport and prepends proxy connection handling.
+ """
+
+ def __init__(self, proxy_url: str, *, inner: BaseTransport = None):
+ self.proxy_url = proxy_url
+ self._inner = inner or HTTPTransport()
+
+ def handle_request(self, request: Request) -> Response:
+ try:
+ return self._inner.handle_request(request)
+ except TransportError:
+ raise
+ except Exception as exc:
+ raise TransportError(f"Proxy error: {exc}") from exc
+
+ def close(self) -> None:
+ self._inner.close()
diff --git a/worked/httpx/raw/utils.py b/worked/httpx/raw/utils.py
new file mode 100644
index 000000000..84ca4a3b8
--- /dev/null
+++ b/worked/httpx/raw/utils.py
@@ -0,0 +1,85 @@
+"""
+Utility functions shared across the library.
+Small helpers that don't belong in any one module.
+"""
+import re
+from models import Cookies
+
+
+SENSITIVE_HEADERS = {"authorization", "cookie", "set-cookie", "proxy-authorization"}
+
+
+def primitive_value_to_str(value) -> str:
+ """Convert a primitive value to its string representation."""
+ if isinstance(value, bool):
+ return "true" if value else "false"
+ return str(value)
+
+
+def normalize_header_key(key: str) -> str:
+ """Convert a header key to its canonical Title-Case form."""
+ return "-".join(word.capitalize() for word in key.split("-"))
+
+
+def flatten_queryparams(params: dict) -> list:
+ """
+ Expand a params dict into a flat list of (key, value) pairs.
+ List values become multiple pairs with the same key.
+ """
+ result = []
+ for key, value in params.items():
+ if isinstance(value, list):
+ for item in value:
+ result.append((key, primitive_value_to_str(item)))
+ else:
+ result.append((key, primitive_value_to_str(value)))
+ return result
+
+
+def parse_content_type(content_type: str) -> tuple:
+ """
+ Parse a Content-Type header value.
+ Returns (media_type, params_dict).
+ Example: 'application/json; charset=utf-8' -> ('application/json', {'charset': 'utf-8'})
+ """
+ parts = [p.strip() for p in content_type.split(";")]
+ media_type = parts[0]
+ params = {}
+ for part in parts[1:]:
+ if "=" in part:
+ key, _, value = part.partition("=")
+ params[key.strip()] = value.strip()
+ return media_type, params
+
+
+def obfuscate_sensitive_headers(headers: dict) -> dict:
+ """Return a copy of headers with sensitive values replaced by [obfuscated]."""
+ return {
+ k: "[obfuscated]" if k.lower() in SENSITIVE_HEADERS else v
+ for k, v in headers.items()
+ }
+
+
+def unset_all_cookies(cookies: Cookies) -> None:
+ """Clear all cookies from a cookie jar in place."""
+ cookies.clear()
+
+
+def is_known_encoding(encoding: str) -> bool:
+ """Check if a character encoding label is recognized by Python's codec system."""
+ import codecs
+ try:
+ codecs.lookup(encoding)
+ return True
+ except LookupError:
+ return False
+
+
+def build_url_with_params(base_url: str, params: dict) -> str:
+ """Append query parameters to a URL string."""
+ if not params:
+ return base_url
+ pairs = flatten_queryparams(params)
+ query = "&".join(f"{k}={v}" for k, v in pairs)
+ separator = "&" if "?" in base_url else "?"
+ return f"{base_url}{separator}{query}"
diff --git a/worked/karpathy-repos/README.md b/worked/karpathy-repos/README.md
new file mode 100644
index 000000000..849897284
--- /dev/null
+++ b/worked/karpathy-repos/README.md
@@ -0,0 +1,63 @@
+# Karpathy Repos Benchmark — How to Reproduce
+
+This is the corpus that produced the 71.5x token reduction benchmark.
+
+## Corpus (52 files)
+
+### Code — clone these 3 repos
+
+```bash
+git clone https://github.com/karpathy/nanoGPT
+git clone https://github.com/karpathy/minGPT
+git clone https://github.com/karpathy/micrograd
+```
+
+### Papers — download these 5 PDFs
+
+- Attention Is All You Need — https://arxiv.org/abs/1706.03762
+- FlashAttention: Fast and Memory-Efficient Exact Attention — https://arxiv.org/abs/2205.14135
+- FlashAttention-2 — https://arxiv.org/abs/2307.08691
+- Neural Attention Residuals — https://arxiv.org/abs/2505.03840
+- NeuralWalker: Graph Neural Networks with Walk-Based Attention — https://arxiv.org/abs/2502.02593
+
+### Images — save these 4
+
+- `gpt2_124M_loss.png` — nanoGPT training loss curve (in the nanoGPT repo)
+- `gout.svg` — micrograd computation graph (in the micrograd repo)
+- `moon_mlp.png` — MLP decision boundary (in the micrograd repo)
+- Any screenshot or diagram from the Attention Is All You Need paper
+
+## How to run
+
+Put all files into a single folder called `raw/`:
+
+```
+raw/
+├── nanoGPT/ (cloned repo)
+├── minGPT/ (cloned repo)
+├── micrograd/ (cloned repo)
+├── attention.pdf
+├── flashattention.pdf
+├── flashattention2.pdf
+├── attn_residuals.pdf
+├── neuralwalker.pdf
+├── gpt2_124M_loss.png
+├── gout.svg
+└── moon_mlp.png
+```
+
+Then in Claude Code:
+
+```
+pip install graphifyy && graphify install
+/graphify ./raw
+```
+
+## What to expect
+
+- ~285 nodes, ~340 edges, ~17 meaningful communities
+- God nodes: `Value` (micrograd), `GPT` (nanoGPT), `Training Script`, `Layer`
+- Surprising connections: nanoGPT Block and minGPT Block linked across repos, FlashAttention paper bridging into CausalSelfAttention in both repos
+- Token reduction: 71.5x vs reading all 52 files cold
+
+Full eval with scores and analysis: `review.md`
diff --git a/worked/mixed-corpus/README.md b/worked/mixed-corpus/README.md
new file mode 100644
index 000000000..e62c32fef
--- /dev/null
+++ b/worked/mixed-corpus/README.md
@@ -0,0 +1,45 @@
+# Mixed Corpus Benchmark — How to Reproduce
+
+A small but realistic mixed-input corpus: Python source files, a markdown paper with
+arXiv citations, and one image. Tests graphify's ability to handle different file types
+in a single run.
+
+## Corpus (5 files)
+
+All input files are in `raw/`:
+
+```
+raw/
+├── analyze.py — graphify's graph analysis module (god_nodes, surprising_connections, etc.)
+├── build.py — graphify's graph builder (build_from_json, networkx wrapper)
+├── cluster.py — graphify's Leiden community detection (cluster, score_all)
+├── attention_notes.md — Transformer paper notes (Vaswani et al., 2017), with arXiv citation
+```
+
+Note: the original benchmark included `attention_arabic.png` (an Arabic-language figure from the
+Attention paper). PNG files are not stored in this repo. To reproduce with the image, save any
+diagram or figure from the Attention Is All You Need paper as `raw/attention_arabic.png`.
+
+## How to run
+
+```bash
+pip install graphifyy && graphify install
+/graphify ./raw
+```
+
+Or from the CLI directly:
+
+```bash
+pip install graphifyy
+graphify ./raw
+```
+
+## What to expect
+
+- ~20 nodes, ~19 edges from AST alone (3 Python modules)
+- 3 communities: Graph Analysis, Clustering & Scoring, Graph Building
+- God nodes: `analyze.py`, `cluster.py`, `build.py`
+- `attention_notes.md` classified as `paper` (arXiv heuristic fires on `1706.03762`)
+- If you include the image: 1 extra node describing the figure content via vision
+
+Full eval with scores and analysis: `review.md`
diff --git a/worked/mixed-corpus/raw/analyze.py b/worked/mixed-corpus/raw/analyze.py
new file mode 100644
index 000000000..cf5344960
--- /dev/null
+++ b/worked/mixed-corpus/raw/analyze.py
@@ -0,0 +1,517 @@
+"""Graph analysis: god nodes (most connected), surprising connections (cross-community), suggested questions."""
+from __future__ import annotations
+import networkx as nx
+
+
+def _node_community_map(communities: dict[int, list[str]]) -> dict[str, int]:
+ """Invert communities dict: node_id -> community_id."""
+ return {n: cid for cid, nodes in communities.items() for n in nodes}
+
+
+def _is_file_node(G: nx.Graph, node_id: str) -> bool:
+ """
+ Return True if this node is a file-level hub node (e.g. 'client', 'models')
+ or an AST method stub (e.g. '.auth_flow()', '.__init__()').
+
+ These are synthetic nodes created by the AST extractor and should be excluded
+ from god nodes, surprising connections, and knowledge gap reporting.
+ """
+ label = G.nodes[node_id].get("label", "")
+ if not label:
+ return False
+ # File-level hub: label is a filename with a code extension
+ if label.split(".")[-1] in ("py", "ts", "js", "go", "rs", "java", "rb", "cpp", "c", "h"):
+ return True
+ # Method stub: AST extractor labels methods as '.method_name()'
+ if label.startswith(".") and label.endswith("()"):
+ return True
+ # Module-level function stub: labeled 'function_name()' - only has a contains edge
+ # These are real functions but structurally isolated by definition; not a gap worth flagging
+ if label.endswith("()") and G.degree(node_id) <= 1:
+ return True
+ return False
+
+
+def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]:
+ """Return the top_n most-connected real entities - the core abstractions.
+
+ File-level hub nodes are excluded: they accumulate import/contains edges
+ mechanically and don't represent meaningful architectural abstractions.
+ """
+ degree = dict(G.degree())
+ sorted_nodes = sorted(degree.items(), key=lambda x: x[1], reverse=True)
+ result = []
+ for node_id, deg in sorted_nodes:
+ if _is_file_node(G, node_id) or _is_concept_node(G, node_id):
+ continue
+ result.append({
+ "id": node_id,
+ "label": G.nodes[node_id].get("label", node_id),
+ "edges": deg,
+ })
+ if len(result) >= top_n:
+ break
+ return result
+
+
+def surprising_connections(
+ G: nx.Graph,
+ communities: dict[int, list[str]] | None = None,
+ top_n: int = 5,
+) -> list[dict]:
+ """
+ Find connections that are genuinely surprising - not obvious from file structure.
+
+ Strategy:
+ - Multi-file corpora: cross-file edges between real entities (not concept nodes).
+ Sorted AMBIGUOUS → INFERRED → EXTRACTED.
+ - Single-file / single-source corpora: cross-community edges that bridge
+ distant parts of the graph (betweenness centrality on edges).
+ These reveal non-obvious structural couplings.
+
+ Concept nodes (empty source_file, or injected semantic annotations) are excluded
+ from surprising connections because they are intentional, not discovered.
+ """
+ # Identify unique source files (ignore empty/null source_file)
+ source_files = {
+ data.get("source_file", "")
+ for _, data in G.nodes(data=True)
+ if data.get("source_file", "")
+ }
+ is_multi_source = len(source_files) > 1
+
+ if is_multi_source:
+ return _cross_file_surprises(G, communities or {}, top_n)
+ else:
+ return _cross_community_surprises(G, communities or {}, top_n)
+
+
+def _is_concept_node(G: nx.Graph, node_id: str) -> bool:
+ """
+ Return True if this node is a manually-injected semantic concept node
+ rather than a real entity found in source code.
+
+ Signals:
+ - Empty source_file
+ - source_file doesn't look like a real file path (no extension)
+ """
+ data = G.nodes[node_id]
+ source = data.get("source_file", "")
+ if not source:
+ return True
+ # Has no file extension → probably a concept label, not a real file
+ if "." not in source.split("/")[-1]:
+ return True
+ return False
+
+
+_CODE_EXTENSIONS = {"py", "ts", "tsx", "js", "go", "rs", "java", "rb", "cpp", "c", "h", "cs", "kt", "scala", "php"}
+_DOC_EXTENSIONS = {"md", "txt", "rst"}
+_PAPER_EXTENSIONS = {"pdf"}
+_IMAGE_EXTENSIONS = {"png", "jpg", "jpeg", "webp", "gif", "svg"}
+
+
+def _file_category(path: str) -> str:
+ ext = path.rsplit(".", 1)[-1].lower() if "." in path else ""
+ if ext in _CODE_EXTENSIONS:
+ return "code"
+ if ext in _PAPER_EXTENSIONS:
+ return "paper"
+ if ext in _IMAGE_EXTENSIONS:
+ return "image"
+ return "doc"
+
+
+def _top_level_dir(path: str) -> str:
+ """Return the first path component - used to detect cross-repo edges."""
+ return path.split("/")[0] if "/" in path else path
+
+
+def _surprise_score(
+ G: nx.Graph,
+ u: str,
+ v: str,
+ data: dict,
+ node_community: dict[str, int],
+ u_source: str,
+ v_source: str,
+) -> tuple[int, list[str]]:
+ """Score how surprising a cross-file edge is. Returns (score, reasons)."""
+ score = 0
+ reasons: list[str] = []
+
+ # 1. Confidence weight - uncertain connections are more noteworthy
+ conf = data.get("confidence", "EXTRACTED")
+ conf_bonus = {"AMBIGUOUS": 3, "INFERRED": 2, "EXTRACTED": 1}.get(conf, 1)
+ score += conf_bonus
+ if conf in ("AMBIGUOUS", "INFERRED"):
+ reasons.append(f"{conf.lower()} connection - not explicitly stated in source")
+
+ # 2. Cross file-type bonus - code↔paper or code↔image is non-obvious
+ cat_u = _file_category(u_source)
+ cat_v = _file_category(v_source)
+ if cat_u != cat_v:
+ score += 2
+ reasons.append(f"crosses file types ({cat_u} ↔ {cat_v})")
+
+ # 3. Cross-repo bonus - different top-level directory
+ if _top_level_dir(u_source) != _top_level_dir(v_source):
+ score += 2
+ reasons.append("connects across different repos/directories")
+
+ # 4. Cross-community bonus - Leiden says these are structurally distant
+ cid_u = node_community.get(u)
+ cid_v = node_community.get(v)
+ if cid_u is not None and cid_v is not None and cid_u != cid_v:
+ score += 1
+ reasons.append("bridges separate communities")
+
+ # 5. Peripheral→hub: a low-degree node connecting to a high-degree one
+ deg_u = G.degree(u)
+ deg_v = G.degree(v)
+ if min(deg_u, deg_v) <= 2 and max(deg_u, deg_v) >= 5:
+ score += 1
+ peripheral = G.nodes[u].get("label", u) if deg_u <= 2 else G.nodes[v].get("label", v)
+ hub = G.nodes[v].get("label", v) if deg_u <= 2 else G.nodes[u].get("label", u)
+ reasons.append(f"peripheral node `{peripheral}` unexpectedly reaches hub `{hub}`")
+
+ return score, reasons
+
+
+def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: int) -> list[dict]:
+ """
+ Cross-file edges between real code/doc entities, ranked by a composite
+ surprise score rather than confidence alone.
+
+ Surprise score accounts for:
+ - Confidence (AMBIGUOUS > INFERRED > EXTRACTED)
+ - Cross file-type (code↔paper is more surprising than code↔code)
+ - Cross-repo (different top-level directory)
+ - Cross-community (Leiden says structurally distant)
+ - Peripheral→hub (low-degree node reaching a god node)
+
+ Each result includes a 'why' field explaining what makes it non-obvious.
+ """
+ node_community = _node_community_map(communities)
+ candidates = []
+
+ for u, v, data in G.edges(data=True):
+ relation = data.get("relation", "")
+ if relation in ("imports", "imports_from", "contains", "method"):
+ continue
+ if _is_concept_node(G, u) or _is_concept_node(G, v):
+ continue
+ if _is_file_node(G, u) or _is_file_node(G, v):
+ continue
+
+ u_source = G.nodes[u].get("source_file", "")
+ v_source = G.nodes[v].get("source_file", "")
+
+ if not u_source or not v_source or u_source == v_source:
+ continue
+
+ score, reasons = _surprise_score(G, u, v, data, node_community, u_source, v_source)
+ src_id = data.get("_src", u)
+ tgt_id = data.get("_tgt", v)
+ candidates.append({
+ "_score": score,
+ "source": G.nodes[src_id].get("label", src_id),
+ "target": G.nodes[tgt_id].get("label", tgt_id),
+ "source_files": [
+ G.nodes[src_id].get("source_file", ""),
+ G.nodes[tgt_id].get("source_file", ""),
+ ],
+ "confidence": data.get("confidence", "EXTRACTED"),
+ "relation": relation,
+ "why": "; ".join(reasons) if reasons else "cross-file semantic connection",
+ })
+
+ candidates.sort(key=lambda x: x["_score"], reverse=True)
+ for c in candidates:
+ c.pop("_score")
+
+ if candidates:
+ return candidates[:top_n]
+
+ return _cross_community_surprises(G, communities, top_n)
+
+
+def _cross_community_surprises(
+ G: nx.Graph,
+ communities: dict[int, list[str]],
+ top_n: int,
+) -> list[dict]:
+ """
+ For single-source corpora: find edges that bridge different communities.
+ These are surprising because Leiden grouped everything else tightly -
+ these edges cut across the natural structure.
+
+ Falls back to high-betweenness edges if no community info is provided.
+ """
+ if not communities:
+ # No community info - use edge betweenness centrality
+ if G.number_of_edges() == 0:
+ return []
+ betweenness = nx.edge_betweenness_centrality(G)
+ top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n]
+ result = []
+ for (u, v), score in top_edges:
+ data = G.edges[u, v]
+ result.append({
+ "source": G.nodes[u].get("label", u),
+ "target": G.nodes[v].get("label", v),
+ "source_files": [
+ G.nodes[u].get("source_file", ""),
+ G.nodes[v].get("source_file", ""),
+ ],
+ "confidence": data.get("confidence", "EXTRACTED"),
+ "relation": data.get("relation", ""),
+ "note": f"Bridges graph structure (betweenness={score:.3f})",
+ })
+ return result
+
+ # Build node → community map
+ node_community = _node_community_map(communities)
+
+ surprises = []
+ for u, v, data in G.edges(data=True):
+ cid_u = node_community.get(u)
+ cid_v = node_community.get(v)
+ if cid_u is None or cid_v is None or cid_u == cid_v:
+ continue
+ # Skip file hub nodes and plain structural edges
+ if _is_file_node(G, u) or _is_file_node(G, v):
+ continue
+ relation = data.get("relation", "")
+ if relation in ("imports", "imports_from", "contains", "method"):
+ continue
+ # This edge crosses community boundaries - interesting
+ confidence = data.get("confidence", "EXTRACTED")
+ src_id = data.get("_src", u)
+ tgt_id = data.get("_tgt", v)
+ surprises.append({
+ "source": G.nodes[src_id].get("label", src_id),
+ "target": G.nodes[tgt_id].get("label", tgt_id),
+ "source_files": [
+ G.nodes[src_id].get("source_file", ""),
+ G.nodes[tgt_id].get("source_file", ""),
+ ],
+ "confidence": confidence,
+ "relation": relation,
+ "note": f"Bridges community {cid_u} → community {cid_v}",
+ "_pair": tuple(sorted([cid_u, cid_v])),
+ })
+
+ # Sort: AMBIGUOUS first, then INFERRED, then EXTRACTED
+ order = {"AMBIGUOUS": 0, "INFERRED": 1, "EXTRACTED": 2}
+ surprises.sort(key=lambda x: order.get(x["confidence"], 3))
+
+ # Deduplicate by community pair - one representative edge per (A→B) boundary.
+ # Without this, a single high-betweenness god node dominates all results.
+ seen_pairs: set[tuple] = set()
+ deduped = []
+ for s in surprises:
+ pair = s.pop("_pair")
+ if pair not in seen_pairs:
+ seen_pairs.add(pair)
+ deduped.append(s)
+ return deduped[:top_n]
+
+
+def suggest_questions(
+ G: nx.Graph,
+ communities: dict[int, list[str]],
+ community_labels: dict[int, str],
+ top_n: int = 7,
+) -> list[dict]:
+ """
+ Generate questions the graph is uniquely positioned to answer.
+ Based on: AMBIGUOUS edges, bridge nodes, underexplored god nodes, isolated nodes.
+ Each question has a 'type', 'question', and 'why' field.
+ """
+ questions = []
+ node_community = _node_community_map(communities)
+
+ # 1. AMBIGUOUS edges → unresolved relationship questions
+ for u, v, data in G.edges(data=True):
+ if data.get("confidence") == "AMBIGUOUS":
+ ul = G.nodes[u].get("label", u)
+ vl = G.nodes[v].get("label", v)
+ relation = data.get("relation", "related to")
+ questions.append({
+ "type": "ambiguous_edge",
+ "question": f"What is the exact relationship between `{ul}` and `{vl}`?",
+ "why": f"Edge tagged AMBIGUOUS (relation: {relation}) - confidence is low.",
+ })
+
+ # 2. Bridge nodes (high betweenness) → cross-cutting concern questions
+ if G.number_of_edges() > 0:
+ betweenness = nx.betweenness_centrality(G)
+ # Top bridge nodes that are NOT file-level hubs
+ bridges = sorted(
+ [(n, s) for n, s in betweenness.items()
+ if not _is_file_node(G, n) and not _is_concept_node(G, n) and s > 0],
+ key=lambda x: x[1],
+ reverse=True,
+ )[:3]
+ for node_id, score in bridges:
+ label = G.nodes[node_id].get("label", node_id)
+ cid = node_community.get(node_id)
+ comm_label = community_labels.get(cid, f"Community {cid}") if cid is not None else "unknown"
+ neighbors = list(G.neighbors(node_id))
+ neighbor_comms = {node_community.get(n) for n in neighbors if node_community.get(n) != cid}
+ if neighbor_comms:
+ other_labels = [community_labels.get(c, f"Community {c}") for c in neighbor_comms]
+ questions.append({
+ "type": "bridge_node",
+ "question": f"Why does `{label}` connect `{comm_label}` to {', '.join(f'`{l}`' for l in other_labels)}?",
+ "why": f"High betweenness centrality ({score:.3f}) - this node is a cross-community bridge.",
+ })
+
+ # 3. God nodes with many INFERRED edges → verification questions
+ degree = dict(G.degree())
+ top_nodes = sorted(
+ [(n, d) for n, d in degree.items() if not _is_file_node(G, n)],
+ key=lambda x: x[1],
+ reverse=True,
+ )[:5]
+ for node_id, _ in top_nodes:
+ inferred = [
+ (u, v, d) for u, v, d in G.edges(node_id, data=True)
+ if d.get("confidence") == "INFERRED"
+ ]
+ if len(inferred) >= 2:
+ label = G.nodes[node_id].get("label", node_id)
+ # Use _src/_tgt to get the correct direction; fall back to v (the other node)
+ others = []
+ for u, v, d in inferred[:2]:
+ src_id = d.get("_src", u)
+ tgt_id = d.get("_tgt", v)
+ other_id = tgt_id if src_id == node_id else src_id
+ others.append(G.nodes[other_id].get("label", other_id))
+ questions.append({
+ "type": "verify_inferred",
+ "question": f"Are the {len(inferred)} inferred relationships involving `{label}` (e.g. with `{others[0]}` and `{others[1]}`) actually correct?",
+ "why": f"`{label}` has {len(inferred)} INFERRED edges - model-reasoned connections that need verification.",
+ })
+
+ # 4. Isolated or weakly-connected nodes → exploration questions
+ isolated = [
+ n for n in G.nodes()
+ if G.degree(n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n)
+ ]
+ if isolated:
+ labels = [G.nodes[n].get("label", n) for n in isolated[:3]]
+ questions.append({
+ "type": "isolated_nodes",
+ "question": f"What connects {', '.join(f'`{l}`' for l in labels)} to the rest of the system?",
+ "why": f"{len(isolated)} weakly-connected nodes found - possible documentation gaps or missing edges.",
+ })
+
+ # 5. Low-cohesion communities → structural questions
+ from .cluster import cohesion_score
+ for cid, nodes in communities.items():
+ score = cohesion_score(G, nodes)
+ if score < 0.15 and len(nodes) >= 5:
+ label = community_labels.get(cid, f"Community {cid}")
+ questions.append({
+ "type": "low_cohesion",
+ "question": f"Should `{label}` be split into smaller, more focused modules?",
+ "why": f"Cohesion score {score} - nodes in this community are weakly interconnected.",
+ })
+
+ if not questions:
+ return [{
+ "type": "no_signal",
+ "question": None,
+ "why": (
+ "Not enough signal to generate questions. "
+ "This usually means the corpus has no AMBIGUOUS edges, no bridge nodes, "
+ "no INFERRED relationships, and all communities are tightly cohesive. "
+ "Add more files or run with --mode deep to extract richer edges."
+ ),
+ }]
+
+ return questions[:top_n]
+
+
+def graph_diff(G_old: nx.Graph, G_new: nx.Graph) -> dict:
+ """Compare two graph snapshots and return what changed.
+
+ Returns:
+ {
+ "new_nodes": [{"id": ..., "label": ...}],
+ "removed_nodes": [{"id": ..., "label": ...}],
+ "new_edges": [{"source": ..., "target": ..., "relation": ..., "confidence": ...}],
+ "removed_edges": [...],
+ "summary": "3 new nodes, 5 new edges, 1 node removed"
+ }
+ """
+ old_nodes = set(G_old.nodes())
+ new_nodes = set(G_new.nodes())
+
+ added_node_ids = new_nodes - old_nodes
+ removed_node_ids = old_nodes - new_nodes
+
+ new_nodes_list = [
+ {"id": n, "label": G_new.nodes[n].get("label", n)}
+ for n in added_node_ids
+ ]
+ removed_nodes_list = [
+ {"id": n, "label": G_old.nodes[n].get("label", n)}
+ for n in removed_node_ids
+ ]
+
+ def edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple:
+ return (u, v, data.get("relation", ""))
+
+ old_edge_keys = {
+ edge_key(G_old, u, v, d)
+ for u, v, d in G_old.edges(data=True)
+ }
+ new_edge_keys = {
+ edge_key(G_new, u, v, d)
+ for u, v, d in G_new.edges(data=True)
+ }
+
+ added_edge_keys = new_edge_keys - old_edge_keys
+ removed_edge_keys = old_edge_keys - new_edge_keys
+
+ new_edges_list = []
+ for u, v, d in G_new.edges(data=True):
+ if edge_key(G_new, u, v, d) in added_edge_keys:
+ new_edges_list.append({
+ "source": u,
+ "target": v,
+ "relation": d.get("relation", ""),
+ "confidence": d.get("confidence", ""),
+ })
+
+ removed_edges_list = []
+ for u, v, d in G_old.edges(data=True):
+ if edge_key(G_old, u, v, d) in removed_edge_keys:
+ removed_edges_list.append({
+ "source": u,
+ "target": v,
+ "relation": d.get("relation", ""),
+ "confidence": d.get("confidence", ""),
+ })
+
+ parts = []
+ if new_nodes_list:
+ parts.append(f"{len(new_nodes_list)} new node{'s' if len(new_nodes_list) != 1 else ''}")
+ if new_edges_list:
+ parts.append(f"{len(new_edges_list)} new edge{'s' if len(new_edges_list) != 1 else ''}")
+ if removed_nodes_list:
+ parts.append(f"{len(removed_nodes_list)} node{'s' if len(removed_nodes_list) != 1 else ''} removed")
+ if removed_edges_list:
+ parts.append(f"{len(removed_edges_list)} edge{'s' if len(removed_edges_list) != 1 else ''} removed")
+ summary = ", ".join(parts) if parts else "no changes"
+
+ return {
+ "new_nodes": new_nodes_list,
+ "removed_nodes": removed_nodes_list,
+ "new_edges": new_edges_list,
+ "removed_edges": removed_edges_list,
+ "summary": summary,
+ }
diff --git a/worked/mixed-corpus/raw/attention_notes.md b/worked/mixed-corpus/raw/attention_notes.md
new file mode 100644
index 000000000..6a60166f8
--- /dev/null
+++ b/worked/mixed-corpus/raw/attention_notes.md
@@ -0,0 +1,76 @@
+# Attention Mechanism Notes
+
+Notes on the Transformer architecture from Vaswani et al., 2017.
+arXiv: 1706.03762
+
+## Abstract
+
+The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder. The best performing models also connect the encoder and decoder through an attention mechanism. The Transformer is a model architecture eschewing recurrence and instead relying entirely on an attention mechanism to draw global dependencies between input and output.
+
+## Multi-Head Attention
+
+The model uses h=8 parallel attention heads. For each head, d_k = d_v = d_model/h = 64.
+
+Scaled dot-product attention:
+
+ Attention(Q, K, V) = softmax(QK^T / sqrt(d_k)) V
+
+Multi-head attention runs h attention functions in parallel, then concatenates and projects:
+
+ MultiHead(Q, K, V) = Concat(head_1, ..., head_h) W^O
+ head_i = Attention(Q W_i^Q, K W_i^K, V W_i^V)
+
+The scaling by sqrt(d_k) prevents the dot products from growing large in magnitude, which would push the softmax into regions with very small gradients.
+
+## Architecture
+
+The Transformer uses a stacked encoder-decoder structure.
+
+Encoder: 6 identical layers, each with two sublayers:
+1. Multi-head self-attention
+2. Position-wise fully connected feed-forward network
+
+Each sublayer uses a residual connection followed by layer normalization:
+ output = LayerNorm(x + Sublayer(x))
+
+Decoder: 6 identical layers, each with three sublayers:
+1. Masked multi-head self-attention (prevents positions from attending to subsequent positions)
+2. Multi-head attention over encoder output
+3. Position-wise feed-forward network
+
+d_model = 512 for all sublayers and embedding layers.
+Feed-forward inner dimension = 2048.
+
+## Positional Encoding
+
+Since the model contains no recurrence and no convolution, positional encodings are added to the input embeddings to give the model information about the relative position of tokens:
+
+ PE(pos, 2i) = sin(pos / 10000^(2i/d_model))
+ PE(pos, 2i+1) = cos(pos / 10000^(2i/d_model))
+
+This allows the model to easily learn to attend by relative positions.
+
+## Why attention over recurrence
+
+Three main advantages:
+1. Total computational complexity per layer is lower for self-attention when sequence length is smaller than representation dimensionality
+2. Computations that can be parallelized — recurrent layers require O(n) sequential operations
+3. Path length between long-range dependencies is O(1) for self-attention vs O(n) for recurrence
+
+## Results
+
+WMT 2014 English-to-German: 28.4 BLEU, outperforming all previously published results by over 2 BLEU.
+WMT 2014 English-to-French: 41.0 BLEU, new state of the art.
+Training cost: 3.5 days on 8 P100 GPUs.
+
+## Open questions
+
+[1] Does the choice of h=8 heads generalize, or is it architecture-specific?
+[2] The scaling factor sqrt(d_k) is justified empirically — is there a theoretical justification?
+[3] How does learned positional encoding compare to sinusoidal at longer sequence lengths?
+
+## References
+
+[1] Vaswani, A., Shazeer, N., Parmar, N., et al. (2017). Attention Is All You Need. arXiv:1706.03762
+[2] Ba, J., Kiros, J., Hinton, G. (2016). Layer Normalization. arXiv:1607.06450
+[3] He, K., et al. (2016). Deep Residual Learning for Image Recognition. CVPR 2016.
diff --git a/worked/mixed-corpus/raw/build.py b/worked/mixed-corpus/raw/build.py
new file mode 100644
index 000000000..655820c04
--- /dev/null
+++ b/worked/mixed-corpus/raw/build.py
@@ -0,0 +1,39 @@
+# assemble node+edge dicts into a NetworkX graph, preserving edge direction
+from __future__ import annotations
+import sys
+import networkx as nx
+from .validate import validate_extraction
+
+
+def build_from_json(extraction: dict) -> nx.Graph:
+ errors = validate_extraction(extraction)
+ # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors.
+ real_errors = [e for e in errors if "does not match any node id" not in e]
+ if real_errors:
+ print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr)
+ G = nx.Graph()
+ for node in extraction.get("nodes", []):
+ G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"})
+ node_set = set(G.nodes())
+ for edge in extraction.get("edges", []):
+ src, tgt = edge["source"], edge["target"]
+ if src not in node_set or tgt not in node_set:
+ continue # skip edges to external/stdlib nodes - expected, not an error
+ attrs = {k: v for k, v in edge.items() if k not in ("source", "target")}
+ # Preserve original edge direction - undirected graphs lose it otherwise,
+ # causing display functions to show edges backwards.
+ attrs["_src"] = src
+ attrs["_tgt"] = tgt
+ G.add_edge(src, tgt, **attrs)
+ return G
+
+
+def build(extractions: list[dict]) -> nx.Graph:
+ """Merge multiple extraction results into one graph."""
+ combined: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0}
+ for ext in extractions:
+ combined["nodes"].extend(ext.get("nodes", []))
+ combined["edges"].extend(ext.get("edges", []))
+ combined["input_tokens"] += ext.get("input_tokens", 0)
+ combined["output_tokens"] += ext.get("output_tokens", 0)
+ return build_from_json(combined)
diff --git a/worked/mixed-corpus/raw/cluster.py b/worked/mixed-corpus/raw/cluster.py
new file mode 100644
index 000000000..b5c97b7c8
--- /dev/null
+++ b/worked/mixed-corpus/raw/cluster.py
@@ -0,0 +1,104 @@
+"""Leiden community detection on NetworkX graphs. Splits oversized communities. Returns cohesion scores."""
+from __future__ import annotations
+import networkx as nx
+
+
+def build_graph(nodes: list[dict], edges: list[dict]) -> nx.Graph:
+ """Build a NetworkX graph from graphify node/edge dicts.
+
+ Preserves original edge direction as _src/_tgt attributes so that
+ display functions can show relationships in the correct direction,
+ even though the graph is undirected for structural analysis.
+ """
+ G = nx.Graph()
+ for n in nodes:
+ G.add_node(n["id"], **{k: v for k, v in n.items() if k != "id"})
+ for e in edges:
+ attrs = {k: v for k, v in e.items() if k not in ("source", "target")}
+ attrs["_src"] = e["source"]
+ attrs["_tgt"] = e["target"]
+ G.add_edge(e["source"], e["target"], **attrs)
+ return G
+
+_MAX_COMMUNITY_FRACTION = 0.25 # communities larger than 25% of graph get split
+_MIN_SPLIT_SIZE = 10 # only split if community has at least this many nodes
+
+
+def cluster(G: nx.Graph) -> dict[int, list[str]]:
+ """Run Leiden community detection. Returns {community_id: [node_ids]}.
+
+ Community IDs are stable across runs: 0 = largest community after splitting.
+ Oversized communities (> 25% of graph nodes, min 10) are split by running
+ a second Leiden pass on the subgraph.
+ """
+ if G.number_of_nodes() == 0:
+ return {}
+ if G.number_of_edges() == 0:
+ return {i: [n] for i, n in enumerate(sorted(G.nodes))}
+
+ from graspologic.partition import leiden # lazy - avoids 15s numba JIT on import
+
+ # Leiden warns and drops isolates - handle them separately
+ isolates = [n for n in G.nodes() if G.degree(n) == 0]
+ connected_nodes = [n for n in G.nodes() if G.degree(n) > 0]
+ connected = G.subgraph(connected_nodes)
+
+ raw: dict[int, list[str]] = {}
+ if connected.number_of_nodes() > 0:
+ partition: dict[str, int] = leiden(connected)
+ for node, cid in partition.items():
+ raw.setdefault(cid, []).append(node)
+
+ # Each isolate becomes its own single-node community
+ next_cid = max(raw.keys(), default=-1) + 1
+ for node in isolates:
+ raw[next_cid] = [node]
+ next_cid += 1
+
+ # Split oversized communities
+ max_size = max(_MIN_SPLIT_SIZE, int(G.number_of_nodes() * _MAX_COMMUNITY_FRACTION))
+ final_communities: list[list[str]] = []
+ for nodes in raw.values():
+ if len(nodes) > max_size:
+ final_communities.extend(_split_community(G, nodes))
+ else:
+ final_communities.append(nodes)
+
+ # Re-index by size descending for deterministic ordering
+ final_communities.sort(key=len, reverse=True)
+ return {i: sorted(nodes) for i, nodes in enumerate(final_communities)}
+
+
+def _split_community(G: nx.Graph, nodes: list[str]) -> list[list[str]]:
+ """Run a second Leiden pass on a community subgraph to split it further."""
+ subgraph = G.subgraph(nodes)
+ if subgraph.number_of_edges() == 0:
+ # No edges - split into individual nodes
+ return [[n] for n in sorted(nodes)]
+ try:
+ from graspologic.partition import leiden
+ sub_partition: dict[str, int] = leiden(subgraph)
+ sub_communities: dict[int, list[str]] = {}
+ for node, cid in sub_partition.items():
+ sub_communities.setdefault(cid, []).append(node)
+ if len(sub_communities) <= 1:
+ # Leiden couldn't split it - return as-is
+ return [sorted(nodes)]
+ return [sorted(v) for v in sub_communities.values()]
+ except Exception:
+ return [sorted(nodes)]
+
+
+def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float:
+ """Ratio of actual intra-community edges to maximum possible."""
+ n = len(community_nodes)
+ if n <= 1:
+ return 1.0
+ subgraph = G.subgraph(community_nodes)
+ actual = subgraph.number_of_edges()
+ possible = n * (n - 1) / 2
+ return round(actual / possible, 2) if possible > 0 else 0.0
+
+
+def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]:
+ return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()}