Code graph and memory for your coding agents, in one Rust program. For Claude Code, Codex, Cursor and any MCP client.
Rust agent-memoryai-agentsclaude-codecode-analysiscode-graphcode-intelligencecode-searchcodexcoding-agentscontext-engineeringcursordeveloper-toolsgraphragknowledge-graphlocal-firstmcpmcp-servermodel-context-protocolrusttree-sitter
#!/usr/bin/env python3
"""C4c: the billed cache meter. One 1M-token kubernetes pack, then Muse Spark and Muse Wonder follow-ups.
usage:
c4c.py prepare --work WORK --graphicle BIN --tokenizer TOKENIZER.json --texts TEXTS [--budget 1015808]
checks kubernetes out at its snapshot S (WORK/c4c/src/kubernetes, one shallow fetch), builds its index with
BIN (WORK/c4c/data/kubernetes, timed), then the pack of each of the first 20 kubernetes questions:
`graphicle pack "<issue>" --budget B --layout cache --prefix-share 0.9 --format xml|json --tokenizer ...`.
Every pack must have the same prefix_sha256; writes WORK/c4c/packs/<id>.xml and WORK/c4c/packs.json.
c4c.py run --work WORK --out OUT [--profile meta|mock] [--base-url URL] [--key-file FILE] [--spend-cap auto]
[--spend-log FILE] [--caps caps.json] [--prices prices.json] [--tokenizer FILE] [--skip-count]
the session: muse-spark-1.3 on questions 1-20, then muse-wonder-1.0 on questions 1-10, one call at a time
in question order, each model's first call alone, all under one prompt_cache_key with 24-hour retention.
Before the first billed call, POST /responses/input_tokens counts call 1's prompt; over 1,040,384 tokens
the run stops and asks for packs built at 983,040 (prepare --budget 983040). If Spark's second call is
served less than half from the cache, the session stops after Spark's third call. Writes OUT/calls.jsonl
(tokens as Meta reports them, the cached share and the dollars those usage fields bill at prices.json),
OUT/summary.json (the first call against the follow-ups, per model) and OUT/raw/.
The questions are the whole-repository benchmark's kubernetes questions (a development repository), in file
order; their answers are scored as a secondary. Every call goes through the spend log and caps of
../whole-repo/run.py.
"""
import argparse
import fcntl
import hashlib
import json
import math
import os
import statistics
import subprocess
import sys
import time
HERE = os.path.dirname(os.path.abspath(__file__))
WR = os.path.join(HERE, "..", "whole-repo")
sys.path.insert(0, WR)
from pack_arm import pack_command # noqa: E402
from run import (CACHE_RETENTION, PROMPT, PROFILES, CapReached, Client, Spend, logged_total, now_iso, # noqa: E402
parse_files, text_of, usage_of, user_message)
from score import normalize_answer, score_answer # noqa: E402
from tokens import Counter # noqa: E402
N_SPARK, N_WONDER = 20, 10
SPARK, WONDER = "muse-spark-1.3", "muse-wonder-1.0"
BUDGET, SMALLER_BUDGET, SHARE = 1015808, 983040, 0.9
WINDOW, OUT_CAP = 1048576, 8192
OUT_TOKENS_ESTIMATE = 4000
QUESTIONS = os.path.join(WR, "questions", "questions-kubernetes.jsonl")
def questions(texts, path=QUESTIONS):
qs = [json.loads(l) for l in open(path) if l.strip()]
tx = {t["id"]: t["text"] for t in map(json.loads, open(os.path.join(texts, "texts-kubernetes.jsonl")))}
for q in qs:
if hashlib.sha256(tx[q["id"]].encode("utf-8")).hexdigest() != q["text_sha256"]:
raise SystemExit(f"{q['id']}: the text does not match its text_sha256")
return [(q, tx[q["id"]]) for q in qs[:N_SPARK]]
def cmd_prepare(a):
work = os.path.abspath(a.work)
snap = json.load(open(os.path.join(WR, "snapshots.json")))["kubernetes"]["sha"]
src = os.path.abspath(a.src or os.path.join(work, "c4c", "src", "kubernetes"))
data = os.path.abspath(a.data or os.path.join(work, "c4c", "data", "kubernetes"))
if a.data and os.path.isdir(a.data) and os.listdir(a.data):
# An index the same binary built already (node_setup.sh g1): recorded, not rebuilt.
os.makedirs(os.path.join(work, "c4c"), exist_ok=True)
with open(os.path.join(work, "c4c", "index.json"), "w") as f:
json.dump({"graphicle": subprocess.run([a.graphicle, "--version"], capture_output=True,
text=True).stdout.strip(), "data": data, "reused": True}, f)
if not os.path.exists(os.path.join(src, ".git")):
os.makedirs(src, exist_ok=True)
for cmd in (["git", "init", "-q"], ["git", "remote", "add", "origin", "https://github.com/kubernetes/kubernetes.git"],
["git", "fetch", "-q", "--depth", "1", "origin", snap], ["git", "checkout", "-q", "--detach", "FETCH_HEAD"]):
subprocess.run(cmd, cwd=src, check=True)
head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=src, capture_output=True, text=True).stdout.strip()
if head != snap:
raise SystemExit(f"kubernetes is at {head}, not {snap}")
version = subprocess.run([a.graphicle, "--version"], capture_output=True, text=True).stdout.strip()
index_info = os.path.join(work, "c4c", "index.json")
if not os.path.exists(index_info):
os.makedirs(data, exist_ok=True)
t0 = time.time()
r = subprocess.run([a.graphicle, "sync", src, "--data-dir", data, "--json"], capture_output=True, text=True)
if r.returncode != 0:
raise SystemExit("graphicle sync: " + r.stderr[-300:])
with open(index_info, "w") as f:
json.dump({"graphicle": version, "seconds": round(time.time() - t0, 1)}, f)
packs_dir = os.path.join(work, "c4c", "packs")
os.makedirs(packs_dir, exist_ok=True)
counter = Counter(a.tokenizer)
manifest = {"graphicle": version, "S": snap, "budget": a.budget, "layout": "cache", "prefix_share": SHARE,
"counter": counter.name, "packs": []}
for q, text in questions(a.texts, a.questions):
pj = None
for fmt in ("json", "xml"):
r = subprocess.run(pack_command(a.graphicle, src, data, text, a.budget, a.tokenizer, fmt, "cache", SHARE),
capture_output=True, text=True, timeout=3600)
if r.returncode != 0:
raise SystemExit(f"{q['id']}: graphicle pack: {r.stderr[-300:]}")
if fmt == "json":
pj = json.loads(r.stdout)
else:
with open(os.path.join(packs_dir, q["id"] + ".xml"), "w", encoding="utf-8") as f:
f.write(r.stdout)
manifest["packs"].append({"id": q["id"], "tokens": pj.get("tokens"), "prefix_sha256": pj.get("prefix_sha256"),
"prefix_tokens": pj.get("prefix_tokens"),
"xml_sha256": hashlib.sha256(r.stdout.encode("utf-8")).hexdigest(),
"xml_tokens": counter.count(r.stdout)})
print(f"{q['id']}: {manifest['packs'][-1]['tokens']} tokens, prefix {str(pj.get('prefix_sha256'))[:12]}", flush=True)
prefixes = {p["prefix_sha256"] for p in manifest["packs"]}
manifest["one_prefix"] = len(prefixes) == 1 and None not in prefixes
with open(os.path.join(work, "c4c", "packs.json"), "w") as f:
json.dump(manifest, f, indent=1)
if not manifest["one_prefix"]:
raise SystemExit(f"the packs have {len(prefixes)} prefixes: the session would not share one; stopped")
print(f"packs ready: one prefix {prefixes.pop()[:12]}, budget {a.budget}")
def summarize(calls, models=(SPARK, WONDER)):
out = {}
for model in models:
cs = [c for c in calls if c["model"] == model and not c.get("error")]
if not cs:
continue
first, rest = cs[0], cs[1:]
out[model] = {"calls": len(cs), "first": {k: first[k] for k in ("input_tokens", "cached_tokens", "cost_usd",
"cached_share", "seconds")},
"total_usd": sum(c["cost_usd"] for c in cs), "hit5": sum(c.get("hit5", 0) for c in cs)}
if rest:
costs = [c["cost_usd"] for c in rest]
shares = [c["cached_share"] for c in rest]
out[model]["follow_ups"] = {"n": len(rest), "cost_mean": statistics.mean(costs),
"cost_median": statistics.median(costs), "cost_min": min(costs),
"cost_max": max(costs), "cached_share_mean": statistics.mean(shares),
"cached_share_min": min(shares)}
return out
def cmd_run(a):
work = os.path.abspath(a.work)
prof = PROFILES[a.profile]
base_url = a.base_url or prof["base_url"]
if not base_url:
raise SystemExit(f"--base-url is needed with --profile {a.profile}")
manifest = json.load(open(os.path.join(work, "c4c", "packs.json")))
if not manifest.get("one_prefix"):
raise SystemExit("the packs do not share one prefix")
prices = json.load(open(a.prices))
models = [(SPARK if a.profile != "mock" else "mock", N_SPARK), (WONDER if a.profile != "mock" else "mock-wonder", N_WONDER)]
for m, _n in models:
if m not in prices["models"]:
raise SystemExit(f"no price for {m}")
provider = prices["models"][models[0][0]].get("provider", a.profile)
caps = json.load(open(a.caps))["usd"] if os.path.exists(a.caps) else {}
global_cap = caps.get(provider)
key = open(a.key_file).read().strip() if a.key_file else os.environ.get(prof["key_env"], "")
if not key:
raise SystemExit(f"no key: set {prof['key_env']} or pass --key-file")
spend_log = os.path.abspath(a.spend_log or os.path.join(work, "spend", "spend-log.jsonl"))
os.makedirs(os.path.join(a.out, "raw"), exist_ok=True)
os.makedirs(os.path.dirname(spend_log), exist_ok=True)
lock = open(spend_log + f".{provider}.lock", "w")
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
sys.exit(f"REFUSED: another run for {provider} holds {lock.name}")
counter = Counter(a.tokenizer)
qs = questions(a.texts, a.questions)
packs = {p["id"]: p for p in manifest["packs"]}
prefix12 = manifest["packs"][0]["prefix_sha256"][:12]
cache_key = f"c4c:kubernetes@{manifest['S'][:12]}:pack-cache@1M:{prefix12}"
def prompt_of(q, text):
body = open(os.path.join(work, "c4c", "packs", q["id"] + ".xml"), encoding="utf-8").read()
if hashlib.sha256(body.encode("utf-8")).hexdigest() != packs[q["id"]]["xml_sha256"]:
raise SystemExit(f"{q['id']}: the pack file changed since prepare")
return PROMPT.format(context=body + "\n\n", repo=q["repo"], sha=q["S"], issue=text)
def cost(price, tin, tcached, tout):
return ((tin - tcached) * price["input"] + tcached * price.get("cached_input", price["input"])
+ tout * price["output"]) / 1e6
# The estimate: each model's first call uncached, the others with the 90% prefix from the cache.
est = 0.0
for m, n in models:
p = prices["models"][m]
for i, (q, text) in enumerate(qs[:n]):
tin = counter.count(prompt_of(q, text)) if a.tokenizer else packs[q["id"]]["tokens"] + 1000
shared = packs[q["id"]].get("prefix_tokens") or int(SHARE * packs[q["id"]]["tokens"])
est += cost(p, tin, 0 if i == 0 else min(tin, shared), OUT_TOKENS_ESTIMATE)
calls_path = os.path.join(a.out, "calls.jsonl")
done = [json.loads(l) for l in open(calls_path)] if os.path.exists(calls_path) else []
logged = logged_total(spend_log, provider)
print(f"c4c: estimate ${est:.2f}; {provider} log total ${logged:.2f}"
+ (f", cap ${global_cap:.2f}" if global_cap is not None else ""), flush=True)
if not done and global_cap is not None and logged + est > global_cap:
sys.exit(f"REFUSED: the {provider} log total ${logged:.2f} plus the estimate ${est:.2f} would pass the cap")
spent_before = json.load(open(os.path.join(a.out, "spend.json")))["spent_usd"] \
if os.path.exists(os.path.join(a.out, "spend.json")) else 0.0
cap = spent_before + math.ceil(1.5 * est) if str(a.spend_cap).lower() == "auto" else float(a.spend_cap)
if global_cap is not None:
cap = min(cap, spent_before + global_cap - logged)
spend = Spend(os.path.join(a.out, "spend.json"), cap, prices["models"][models[0][0]], spend_log, provider, global_cap)
client = Client(base_url, key, "responses")
run_meta = {"session": "c4c", "cache_key": cache_key, "budget": manifest["budget"], "graphicle": manifest["graphicle"],
"S": manifest["S"], "models": [m for m, _n in models], "estimate_usd": est, "spend_cap_usd": cap,
"global_cap_usd": global_cap, "spend_log": spend_log, "started": now_iso()}
if not done and not a.skip_count:
q, text = qs[0]
t0 = time.time()
r = client.post("/responses/input_tokens", {"model": models[0][0], "input": [user_message(prompt_of(q, text))]})
n = r.get("input_tokens")
# Into the ledger too; Meta's pages do not say whether counting is billed, so it is logged at $0.
spend.settle(0.0, 0.0, {"time": now_iso(), "provider": provider, "model": models[0][0], "api": "input_tokens",
"arm": "c4c window check", "budget": "1M", "run": os.path.abspath(a.out),
"cache_key": None, "repo": "kubernetes", "question": q["id"], "turn": 0,
"benchmark": "c4c", "status": "ok", "input_tokens": n or 0, "cached_tokens": 0,
"output_tokens": 0, "reasoning_tokens": 0, "seconds": round(time.time() - t0, 3),
"cost_usd": 0.0})
run_meta["window_check"] = {"input_tokens": n, "limit": WINDOW - OUT_CAP}
print(f"window check: call 1 counts {n} input tokens (limit {WINDOW - OUT_CAP})", flush=True)
if n is not None and n > WINDOW - OUT_CAP:
with open(os.path.join(a.out, "run.json"), "w") as f:
json.dump(run_meta, f, indent=1)
sys.exit(f"STOP: call 1 would pass the window; build the packs at {SMALLER_BUDGET} "
f"(c4c.py prepare --budget {SMALLER_BUDGET}) and run again")
with open(os.path.join(a.out, "run.json"), "w") as f:
json.dump(run_meta, f, indent=1)
have = {(c["model"], c["id"]) for c in done if not c.get("error")}
stopped = None
for m, n in models:
price = prices["models"][m]
for i, (q, text) in enumerate(qs[:n], 1):
if (m, q["id"]) in have:
continue
spark_calls = [c for c in done if c["model"] == models[0][0] and not c.get("error")]
if m == models[0][0] and i > 3 and len(spark_calls) >= 2 and \
spark_calls[1]["cached_tokens"] < 0.5 * spark_calls[1]["input_tokens"]:
stopped = "Spark's second call was served less than half from the cache: stopped after its third call"
break
prompt = prompt_of(q, text)
spend.price = price
row = {"n": i, "model": m, "id": q["id"], "cache_key": cache_key, "pack_tokens": packs[q["id"]]["tokens"],
"error": None}
body = {"model": m, "input": [user_message(prompt)], "store": False, "reasoning": {"effort": "medium"},
"max_output_tokens": OUT_CAP, "prompt_cache_key": cache_key,
"prompt_cache_retention": CACHE_RETENTION}
rec = {"time": now_iso(), "provider": provider, "model": m, "api": "responses", "arm": "c4c pack-cache@1M",
"budget": "1M", "run": os.path.abspath(a.out), "cache_key": cache_key, "repo": "kubernetes",
"question": q["id"], "turn": 1, "benchmark": "c4c"}
try:
worst = spend.reserve(counter.count(prompt), OUT_CAP)
except CapReached as e:
stopped = f"cap: {e}"
break
t0 = time.time()
try:
resp = client.post("/responses", body)
except Exception as e:
spend.settle(worst, 0.0, dict(rec, status="error", error=str(e)[:200], input_tokens=0, cached_tokens=0,
output_tokens=0, reasoning_tokens=0, seconds=round(time.time() - t0, 3),
cost_usd=0.0))
row["error"] = str(e)[:300]
with open(calls_path, "a") as f:
f.write(json.dumps(row, sort_keys=True) + "\n")
done.append(row)
continue
tin, tcached, tout, treason = usage_of(resp, "responses")
c = cost(price, tin, tcached, tout)
secs = round(time.time() - t0, 3)
spend.settle(worst, c, dict(rec, status="ok", input_tokens=tin, cached_tokens=tcached, output_tokens=tout,
reasoning_tokens=treason, seconds=secs, cost_usd=c))
answer = text_of(resp, "responses")
files = parse_files(answer)
row.update(score_answer(normalize_answer(files) if files is not None else [], q["gold"]))
row.update({"input_tokens": tin, "cached_tokens": tcached, "output_tokens": tout, "reasoning_tokens": treason,
"seconds": secs, "cost_usd": c, "cached_share": (tcached / tin) if tin else 0.0,
"answer_parsed": files is not None, "usage": resp.get("usage")})
with open(os.path.join(a.out, "raw", f"{m}-{q['id']}.json"), "w") as f:
json.dump({"response": resp, "answer": answer}, f)
with open(calls_path, "a") as f:
f.write(json.dumps(row, sort_keys=True) + "\n")
done.append(row)
print(f"{m} {i:2d} {q['id']}: in {tin:,} cached {tcached:,} ({row['cached_share']:.1%}) out {tout:,} "
f"${c:.4f}", flush=True)
if stopped:
break
summary = summarize(done, [m for m, _n in models])
summary["stopped"] = stopped
with open(os.path.join(a.out, "summary.json"), "w") as f:
json.dump(summary, f, indent=1)
run_meta.update({"finished": now_iso(), "spend_usd": spend.spent, "stopped": stopped})
with open(os.path.join(a.out, "run.json"), "w") as f:
json.dump(run_meta, f, indent=1)
print(json.dumps(summary, indent=1)[:3000])
def cmd_check(a):
"""The first question's pack built again by another binary, compared with the session's."""
work = os.path.abspath(a.work)
manifest = json.load(open(os.path.join(work, "c4c", "packs.json")))
q, text = questions(a.texts, a.questions)[0]
want = next(p for p in manifest["packs"] if p["id"] == q["id"])
out = {"id": q["id"], "session_graphicle": manifest["graphicle"], "session_xml_sha256": want["xml_sha256"],
"session_prefix_sha256": want["prefix_sha256"], "budget": manifest["budget"],
"check_binary": os.path.abspath(a.graphicle)}
for fmt in ("json", "xml"):
r = subprocess.run(pack_command(a.graphicle, os.path.abspath(a.src), os.path.abspath(a.data), text,
manifest["budget"], a.tokenizer, fmt, "cache", SHARE),
capture_output=True, text=True, timeout=3600)
if r.returncode != 0:
raise SystemExit(f"graphicle pack: {r.stderr[-300:]}")
if fmt == "json":
out["check_prefix_sha256"] = json.loads(r.stdout).get("prefix_sha256")
else:
out["check_xml_sha256"] = hashlib.sha256(r.stdout.encode("utf-8")).hexdigest()
out["same"] = out["check_xml_sha256"] == want["xml_sha256"] and out["check_prefix_sha256"] == want["prefix_sha256"]
with open(os.path.join(work, "c4c", "check-" + a.name + ".json"), "w") as f:
json.dump(out, f, indent=1)
print(json.dumps(out, indent=1))
def main():
ap = argparse.ArgumentParser()
sub = ap.add_subparsers(dest="cmd", required=True)
c = sub.add_parser("check")
c.add_argument("--work", required=True)
c.add_argument("--graphicle", required=True)
c.add_argument("--tokenizer", required=True)
c.add_argument("--texts", required=True)
c.add_argument("--questions", default=QUESTIONS)
c.add_argument("--src", required=True)
c.add_argument("--data", required=True, help="an index of --src built by --graphicle")
c.add_argument("--name", required=True, help="names the result file WORK/c4c/check-NAME.json")
p = sub.add_parser("prepare")
p.add_argument("--work", required=True)
p.add_argument("--graphicle", required=True)
p.add_argument("--tokenizer", required=True)
p.add_argument("--texts", required=True)
p.add_argument("--budget", type=int, default=BUDGET)
p.add_argument("--questions", default=QUESTIONS)
p.add_argument("--src", help="a kubernetes checkout at S to use (default WORK/c4c/src/kubernetes, fetched)")
p.add_argument("--data", help="its index, built by the same binary (default WORK/c4c/data/kubernetes)")
r = sub.add_parser("run")
r.add_argument("--work", required=True)
r.add_argument("--out", required=True)
r.add_argument("--texts", required=True)
r.add_argument("--questions", default=QUESTIONS)
r.add_argument("--profile", default="meta", choices=["meta", "mock"])
r.add_argument("--base-url")
r.add_argument("--key-file")
r.add_argument("--spend-cap", default="auto")
r.add_argument("--spend-log")
r.add_argument("--caps", default=os.path.join(WR, "caps.json"))
r.add_argument("--prices", default=os.path.join(WR, "prices.json"))
r.add_argument("--tokenizer")
r.add_argument("--skip-count", action="store_true", help="do not count call 1 with /responses/input_tokens")
a = ap.parse_args()
{"prepare": cmd_prepare, "run": cmd_run, "check": cmd_check}[a.cmd](a)
if __name__ == "__main__":
main()