The Open Framework for Building Agents
test(e2e): make sandbox curl fanout deterministic (#3923)
OwenKephart952af07bfbca
diff --git a/e2e/fixtures/agent-tools-sandbox/agent/lib/mock-responder.ts b/e2e/fixtures/agent-tools-sandbox/agent/lib/mock-responder.tsindex 87dc88554..6564a9054 100644--- a/e2e/fixtures/agent-tools-sandbox/agent/lib/mock-responder.ts+++ b/e2e/fixtures/agent-tools-sandbox/agent/lib/mock-responder.ts@@ -12,6 +12,16 @@ const SKILL_DIRECTIVE = /load the `([^`]+)` skill/iu;*/export function respond(request: MockModelRequest): MockModelResponse | string {const message = request.lastUserMessage ?? "";+ if (message.includes("EVE_SANDBOX_CURL_FANOUT")) {+ return request.toolResults.some((result) => result.name === "bash")+ ? "curl fanout complete"+ : {+ toolCalls: [...message.matchAll(/^curl-\d+: `([^`]+)`$/gmu)].map(([, command]) => ({+ input: { command },+ name: "bash",+ })),+ };+ }if (message.includes("DYNAMIC-TURN-REPLAY-START")) {const gate = request.toolResults.find((result) => result.name === "dynamic-turn-replay-gate");if (gate === undefined) {diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.tsindex f87f117e0..4420d3449 100644--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts@@ -25,11 +25,11 @@ interface CurlBarrierResult {}export default defineEval({- tags: ["real-model"],description: "Sandbox Bash: at least ten curls reach a concurrency barrier.",async test(t) {const turn = await t.send([+ "EVE_SANDBOX_CURL_FANOUT",`Call the \`${BASH_TOOL}\` tool at least ${MINIMUM_CURL_CALLS} separate times in one tool-use step.`,"Run every command below at least once. If you make extra calls, repeat a command below.","Do not combine commands, use a loop, or background a process.",