Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(evaluation): require explicit REA server attribution

morluto443c7aff77c8
diff --git a/src/evaluation/CodexAgentEval.ts b/src/evaluation/CodexAgentEval.ts
index 55ec8333e..744524067 100644
--- a/src/evaluation/CodexAgentEval.ts
+++ b/src/evaluation/CodexAgentEval.ts
@@ -205,7 +205,7 @@ export const evaluateCodexEvents = (
);
const reaCalls = state.calls.filter(
- ({ server }) => server === null || server.toLowerCase() === "rea",
+ ({ server }) => server?.toLowerCase() === "rea",
);
const signatures = new Map<string, number>();
let targetRevision = 0;
diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.ts
index 6b46d9b1d..910fd7ae6 100644
--- a/tests/evaluation/model-evals/codexAgentEval.test.ts
+++ b/tests/evaluation/model-evals/codexAgentEval.test.ts
@@ -218,7 +218,23 @@ describe("Codex completed tool call parsing", () => {
},
);
- it("preserves legacy completed calls with no server field", () => {
+ it.each([
+ {
+ label: "missing server identity",
+ identity: {},
+ routed: false,
+ },
+ {
+ label: "another server",
+ identity: { server: "other" },
+ routed: false,
+ },
+ {
+ label: "verified REA alias",
+ identity: { server_name: "rea" },
+ routed: true,
+ },
+ ])("routes only $label calls", ({ identity, routed }) => {
const metrics = evaluateCodexEvents(
[
{
@@ -228,16 +244,20 @@ describe("Codex completed tool call parsing", () => {
status: "completed",
tool: "inspect_managed_artifact",
arguments: {},
+ ...identity,
},
},
],
"inspect_managed_artifact",
+ { requiredToolSubsequence: ["inspect_managed_artifact"] },
);
expect(metrics).toMatchObject({
- naturalUse: true,
- correctFirstTool: true,
- reaCalls: [{ server: null }],
+ naturalUse: routed,
+ correctFirstTool: routed,
+ firstTool: routed ? "inspect_managed_artifact" : null,
+ requiredToolSubsequenceMet: routed,
+ reaCalls: routed ? [{ server: "rea" }] : [],
});
});
});