Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

4.9 KB
Raw Blame
"""Account for catalog, complete SDK tasks and separately recorded model trials."""
import argparse
import hashlib
import json
from pathlib import Path

import tiktoken


def compact(value):
    return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)


def resource(case):
    path = case.with_name(case.name + ".log")
    scope = next(json.loads(line) for line in reversed(path.read_text().splitlines())
                 if line.startswith('{"resource_scope":'))
    observation = scope["payload_resource_observation"]
    assert scope["abort"] is None and scope["payload_resource_observation_verified"]
    assert observation["payload_cleanup_verified"] and observation["payload_exit_code"] == 0
    assert all(observation["resource_events"]["memory.events"][key] == 0
               for key in ["high", "max", "oom", "oom_kill"])
    return {"wall_seconds": scope["elapsed_seconds"],
            "cpu_seconds": observation["cpu_stat"]["usage_usec"] / 1e6,
            "kernel_family_peak_bytes_including_page_cache": observation["memory_peak_bytes"],
            "resource_stop": scope["abort"], "cleanup_verified": True,
            "final_whole_scope_counters": scope["resource_events_final"],
            "guard_sources_sha256": scope["payload_observer_sources_sha256"],
            "log_sha256": hashlib.sha256(path.read_bytes()).hexdigest()}


def sdk(case, count):
    report = json.loads((case / "report.json").read_text())
    assert report["status"] == "passed" and report["cleanup_verified"]
    catalog = json.loads((case / "catalog.json").read_text())
    calls = json.loads((case / "transcript.json").read_text())
    inputs = [{key: tool[key] for key in ["name", "description", "inputSchema"]}
              for tool in catalog["tools"]]
    arguments = sum(count(compact({"name": call["name"], "arguments": call["arguments"]}))
                    for call in calls)
    text = sum(count(block["text"]) for call in calls for block in call["reply"]["content"]
               if block["type"] == "text")
    return {"case": case.name, "kind": "SDK", "profile": report["profile"],
            "artifact_sha256": report["artifact_sha256"],
            "tool_count": len(catalog["tools"]), "calls": report["calls"],
            "parent_evidence_id": report["parent_evidence_id"],
            "parent_semantic_sha256": report["parent_semantic_sha256"],
            "token_estimates": {"input_catalog_once": count(compact(inputs)),
                                "arguments": arguments, "text_replies": text,
                                "whole_task": count(compact(inputs)) + arguments + text,
                                "raw_catalog_including_output_schemas": count(compact(catalog))},
            "actual_model_usage": None, **resource(case),
            "receipt_sha256": {name: hashlib.sha256((case / name).read_bytes()).hexdigest()
                               for name in ["report.json", "catalog.json", "transcript.json"]}}


def model(case):
    summary = json.loads((case / "summary.json").read_text())
    assert summary["exit_code"] == 0 and not summary["timeout"]
    assert summary["selected_artifact_bytes_unchanged"]
    assert json.loads((case / "cleanup.json").read_text())["profile_removed"]
    return {"case": case.name, "kind": "model", "client": summary["codex_version"],
            "requested_model": summary["requested_model"], "reasoning": summary["reasoning"],
            "model_identity_scope": summary["model_identity_scope"],
            "actual_model_usage": summary["usage"],
            "artifact_sha256": summary["selected_artifact_sha256_before"],
            "calls": summary["completed_mcp_calls"],
            "factual_correctness": "requires independent oracle and answer review",
            **resource(case),
            "receipt_sha256": {name: hashlib.sha256((case / name).read_bytes()).hexdigest()
                               for name in ["summary.json", "answer.txt", "events.jsonl",
                                            "trial-settings.json", "context-receipt.json"]}}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("cases", type=Path, nargs="+")
    args = parser.parse_args()
    encoding = tiktoken.get_encoding("cl100k_base")
    count = lambda value: len(encoding.encode(value, disallowed_special=()))
    result = {"tokenizer": f"tiktoken {tiktoken.__version__}", "encoding": encoding.name,
              "estimate_scope": "One complete name/description/input-schema catalog, every RPC argument and text reply; raw output-schema transport counted separately. Model usage is separate and is not inferred from estimates.",
              "records": [sdk(case, count) if (case / "report.json").exists() else model(case)
                          for case in args.cases]}
    args.output.write_text(json.dumps(result, indent=2) + "\n")


if __name__ == "__main__":
    main()