Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
14.0 KB
Raw Blame
#!/usr/bin/env python3
"""Paired live cache-affinity probe. Runs only when explicitly invoked.

Both binaries must contain identical changes except cache affinity. This runner
records immutable copies/hashes; it cannot establish their source equivalence.
Each arm warms a primary checkout then measures a fresh linked checkout. Arms
have separate repositories, counterbalanced execution order, identical prompts,
and isolated credentials/HOME. A fresh routing key is not proof of a cold server
cache, especially for providers that ignore the routing key.
"""
import argparse
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import shutil
import signal
import statistics
import subprocess
import sys
import time

HELPERS = Path(__file__).resolve().parents[1] / 'graff-evals' / 'measurement.py'
spec = importlib.util.spec_from_file_location('measurement', HELPERS)
measurement = importlib.util.module_from_spec(spec)
spec.loader.exec_module(measurement)
ANSWER = 'CACHE_OK'


def digest(path):
    h = hashlib.sha256()
    with open(path, 'rb') as source:
        for chunk in iter(lambda: source.read(1024 * 1024), b''):
            h.update(chunk)
    return h.hexdigest()


def write_json(path, data):
    path.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
    path.chmod(0o600)


def git(args, env):
    p = subprocess.run(['git', *args], env=env, capture_output=True, text=True, timeout=30)
    if p.returncode:
        raise RuntimeError('Git fixture setup failed: ' + p.stderr[-2000:])


def fixture(root, env):
    primary, linked = root / 'primary', root / 'linked'
    git(['init', '-q', str(primary)], env)
    git(['-C', str(primary), '-c', 'user.name=Cache fixture', '-c',
         '[email protected]', 'commit', '-q', '--allow-empty', '-m', 'fixture'], env)
    git(['-C', str(primary), 'worktree', 'add', '-q', '--detach', str(linked)], env)
    return primary, linked


def prompt_for(repeat):
    # >2,000 whitespace words plus numbers: well above common cache thresholds.
    # The task and bytes are identical across the two arms. These records are
    # synthetic data, not instructions; no workspace or account material enters.
    records = '\n'.join(f'Record {i:04d}: amber cedar quartz willow; quantity {i % 97}; status unchanged.'
                        for i in range(320))
    return (f'Synthetic cache routing fixture, paired repetition {repeat}.\n'
            'Read the reference records below. Do not use tools or inspect files.\n'
            'Reply with exactly CACHE_OK and nothing else. No explanation or formatting.\n'
            '<reference-data>\n' + records + '\n</reference-data>\n'
            'The required complete answer is exactly CACHE_OK.\n')


def parse_trace(cwd):
    rows, malformed = [], 0
    for path in sorted((cwd / '.graff' / 'traces').glob('*.jsonl')):
        for line in path.read_text(errors='replace').splitlines():
            try:
                row = json.loads(line)
            except ValueError:
                malformed += 1
                continue
            if isinstance(row, dict):
                rows.append(row)
    return rows, malformed


def validate(stdout, stderr, rows, model, code, timed_out):
    usage = measurement.graff_usage(stderr)
    final_lines = [measurement.ANSI.sub('', line).strip() for line in stderr.splitlines()
                   if measurement.FOOTER.fullmatch(measurement.ANSI.sub('', line).strip())]
    writes_known = bool(final_lines and measurement.FOOTER.fullmatch(final_lines[-1]).group(4) is not None)
    if usage and not writes_known:
        usage['writes'] = None
    issues = []
    if code != 0 or timed_out:
        issues.append('process_failed_or_timed_out')
    if measurement.ANSI.sub('', stdout).strip() != ANSWER:
        issues.append('answer_not_exact')
    api = [r for r in rows if r.get('ev') == 'api']
    roots = [r for r in api if r.get('agent') == 'main']
    identities = sorted({(r['provider'], r['model']) for r in rows
                         if isinstance(r.get('provider'), str) and isinstance(r.get('model'), str)})
    if not roots or any(r.get('model') != model for r in roots):
        issues.append('missing_or_wrong_root_model_trace')
    if not identities or any(provider != 'codegraff' or actual != model for provider, actual in identities):
        issues.append('missing_or_wrong_provider_model_trace')
    normalized = None
    if not usage:
        issues.append('missing_usage')
    elif not 0 < usage.get('calls', 0) <= 2 or usage.get('in', 0) <= 0 or usage.get('out', 0) <= 0:
        issues.append('invalid_or_empty_usage')
    elif usage['cached'] + (usage['writes'] if writes_known else 0) > usage['in']:
        issues.append('cached_and_written_tokens_exceed_input')
    elif len(api) != usage['calls']:
        issues.append('trace_call_count_does_not_match_usage')
    else:
        # Input is inclusive of reads and writes: never add cached tokens again.
        normalized = {'calls': usage['calls'], 'input_tokens': usage['in'],
                      'output_tokens': usage['out'], 'cached_input_tokens': usage['cached'],
                      'cache_write_tokens': usage['writes'],
                      'ordinary_input_tokens': usage['in'] - usage['cached'] - usage['writes'] if writes_known else None,
                      'cache_read_fraction': usage['cached'] / usage['in'],
                      'cost_usd': usage.get('cost_usd'), 'cost_kind': usage.get('cost_kind', 'unknown')}
        if usage.get('sub_calls') or usage.get('unpriced_calls'):
            normalized['cost_usd'] = None
    return {'valid': not issues, 'issues': issues, 'usage': normalized,
            'raw_final_usage': final_lines[-1] if final_lines else None,
            'unknown_usage_fields': [] if writes_known else ['cache_write_tokens', 'ordinary_input_tokens'],
            'parsed_final_usage': usage or None, 'observed_provider_models': identities,
            'observed_root_models': sorted({r.get('model', '') for r in roots}),
            'trace_api_calls': len(api)}


def run(binary, cwd, artifacts, model, prompt, timeout, source_env):
    artifacts.mkdir(mode=0o700)
    env = measurement.provider_environment('codegraff', str(artifacts), source_env, model)
    env['PWD'] = str(cwd)
    env['GRAFF_LEARNING_PRIVACY'] = 'local'
    with binary.open('rb') as executable:
        shebang = executable.readline(256)
    # Windows cannot launch Python scripts through CreateProcess by shebang.
    launcher = [sys.executable] if os.name == 'nt' and (binary.suffix.lower() == '.py' or
                (shebang.startswith(b'#!') and b'python' in shebang.lower())) else []
    argv = [*launcher, str(binary), '--no-local-tools', '--yolo', '--max-model-calls', '2',
            '--no-telemetry', '-p', prompt]
    expected_hash = digest(binary)
    started = time.monotonic_ns()
    proc = subprocess.Popen(argv, cwd=cwd, env=env, stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE, text=True, start_new_session=True)
    timed_out = False
    try:
        stdout, stderr = proc.communicate(timeout=timeout)
    except subprocess.TimeoutExpired:
        timed_out = True
        if os.name == 'nt':
            proc.terminate()
        else:
            os.killpg(proc.pid, signal.SIGTERM)
        try:
            stdout, stderr = proc.communicate(timeout=5)
        except subprocess.TimeoutExpired:
            if os.name == 'nt':
                proc.kill()
            else:
                os.killpg(proc.pid, signal.SIGKILL)
            stdout, stderr = proc.communicate()
    wall_ns = time.monotonic_ns() - started
    measurement.private_logs(str(artifacts), stdout, stderr)
    rows, malformed = parse_trace(cwd)
    write_json(artifacts / 'trace-records.json', rows)
    result = validate(stdout, stderr, rows, model, proc.returncode, timed_out)
    if digest(binary) != expected_hash:
        result['valid'] = False
        result['issues'].append('binary_changed_during_run')
    if malformed:
        result['valid'] = False
        result['issues'].append('malformed_trace_records')
    result.update(wall_seconds=wall_ns / 1e9, binary_sha256=expected_hash,
                  prompt_sha256=hashlib.sha256(prompt.encode()).hexdigest(),
                  exit_code=proc.returncode, timed_out=timed_out,
                  requested_provider='codegraff', requested_model=model,
                  command_without_prompt=argv[:-1], artifacts=str(artifacts))
    write_json(artifacts / 'result.json', result)
    return result


def aggregate(pairs, models):
    output = {}
    for model in models:
        selected = [p for p in pairs if p['model'] == model]
        valid = [p for p in selected if all(p[arm][phase]['valid'] for arm in ('before', 'after')
                                          for phase in ('warm', 'linked'))]
        report = {'pairs': len(selected), 'valid_pairs': len(valid), 'metrics': {}}
        for metric in ('wall_seconds', 'input_tokens', 'output_tokens', 'cached_input_tokens',
                       'ordinary_input_tokens', 'cache_write_tokens', 'cache_read_fraction', 'cost_usd'):
            deltas, left, right = [], [], []
            for pair in valid:
                b, a = pair['before']['linked'], pair['after']['linked']
                bval = b[metric] if metric == 'wall_seconds' else b['usage'].get(metric)
                aval = a[metric] if metric == 'wall_seconds' else a['usage'].get(metric)
                if bval is not None and aval is not None:
                    left.append(bval)
                    right.append(aval)
                    deltas.append(aval - bval)
            report['metrics'][metric] = {'known_pairs': len(deltas),
                'before_median': statistics.median(left) if left else None,
                'after_median': statistics.median(right) if right else None,
                'paired_delta_median': statistics.median(deltas) if deltas else None}
        output[model] = report
    return output


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument('--before', required=True, type=Path)
    ap.add_argument('--after', required=True, type=Path)
    ap.add_argument('--model', required=True, action='append')
    ap.add_argument('--out', required=True, type=Path, help='New private artifact directory; must not exist')
    ap.add_argument('--repeats', type=int, default=3)
    ap.add_argument('--timeout', type=float, default=180)
    args = ap.parse_args()
    if args.repeats < 1 or args.timeout <= 0:
        ap.error('repeats and timeout must be positive')
    if len(set(args.model)) != len(args.model) or any(not m or '\n' in m or '\r' in m for m in args.model):
        ap.error('models must be unique nonempty single-line names')
    if not os.environ.get('CODEGRAFF_API_KEY'):
        ap.error('CODEGRAFF_API_KEY must be explicitly supplied')
    # Umask precedes all subprocesses, fixtures, HOME creation and trace output.
    os.umask(0o077)
    root = args.out.resolve()
    root.mkdir(mode=0o700, parents=False, exist_ok=False)
    binaries = root / 'binaries'
    binaries.mkdir(mode=0o700)
    copies, hashes = {}, {}
    for arm in ('before', 'after'):
        source = getattr(args, arm).resolve(strict=True)
        if not source.is_file() or not os.access(source, os.X_OK):
            ap.error(f'{arm} must be an executable file')
        before_hash = digest(source)
        dest = binaries / f'{arm}{source.suffix}'
        shutil.copyfile(source, dest)
        dest.chmod(0o500)
        if digest(dest) != before_hash or digest(source) != before_hash:
            raise RuntimeError('input binary changed while being captured')
        copies[arm], hashes[arm] = dest, before_hash
    if hashes['before'] == hashes['after']:
        ap.error('before/after binaries are identical; no treatment difference')
    write_json(root / 'manifest.json', {'binary_sha256': hashes, 'models': args.model,
               'repeats': args.repeats, 'source_equivalence': 'caller must establish cache-only difference',
               'design': 'separate real repositories per arm/repetition; primary warm then fresh linked',
               'caveat': 'fresh routing keys cannot force server cache coldness; automatic cache may ignore keys',
               'script_sha256': digest(Path(__file__).resolve()), 'measurement_sha256': digest(HELPERS)})
    env = {k: os.environ[k] for k in ('PATH', 'SYSTEMROOT') if k in os.environ}
    env.update(HOME=str(root), GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull)
    pairs = []
    for repeat in range(args.repeats):
        prompt = prompt_for(repeat)
        for model_index, model in enumerate(args.model):
            pair_root = root / f'pair-{repeat:02d}-{model_index:02d}'
            pair_root.mkdir(mode=0o700)
            (pair_root / 'prompt.txt').write_text(prompt)
            order = ('before', 'after') if (repeat + model_index) % 2 == 0 else ('after', 'before')
            pair = {'model': model, 'repeat': repeat, 'order': order}
            for arm in order:
                arm_root = pair_root / arm
                arm_root.mkdir(mode=0o700)
                primary, linked = fixture(arm_root, env)
                receipts = arm_root / 'receipts'
                receipts.mkdir(mode=0o700)
                pair[arm] = {}
                for phase, cwd in (('warm', primary), ('linked', linked)):
                    pair[arm][phase] = run(copies[arm], cwd, receipts / phase, model,
                                           prompt, args.timeout, os.environ)
                    print(json.dumps({'model': model, 'repeat': repeat, 'arm': arm, 'phase': phase,
                                      'valid': pair[arm][phase]['valid'], 'issues': pair[arm][phase]['issues']}), flush=True)
            pairs.append(pair)
            write_json(root / 'pairs.json', pairs)
            write_json(root / 'summary.json', aggregate(pairs, args.model))
    print(json.dumps({'summary': aggregate(pairs, args.model), 'artifacts': str(root)}, indent=2))
    if not all(p[arm][phase]['valid'] for p in pairs for arm in ('before', 'after') for phase in ('warm', 'linked')):
        raise SystemExit(1)


if __name__ == '__main__':
    main()