Skip to content
TypeScript

fix(server): request_secret asks for one value per card (#30805)

Roger V9a5c197c07ef
diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts
index 150962e232a..f0affbf6dec 100644
--- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts
+++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts
@@ -197,7 +197,7 @@ const DeleteScheduledTaskTool = Tool.make("delete_scheduled_task", {
const RequestSecretTool = Tool.make("request_secret", {
description:
- 'Ask for a secret in a private card and wait; open it, don\'t offer it. Returns a one-use `secretRef`, or with `delivery: "file"` a file to read only in its command, e.g. `adb pair <ip:port> "$(cat <file>)"`. Asked how to give one: call now. Pasted secret for child or script: `value`. Not one from an old thread.',
+ 'Ask for one secret per single-line card; never combine values. Open it, don\'t offer it. Returns one-use `secretRef`, or `delivery: "file"` a file read only in its command, e.g. `adb pair <ip:port> "$(cat <file>)"`. Asked how: call now. Pasted secret for child/script: `value`. Never from an old thread.',
parameters: OrchestratorMcpRequestSecretInput,
success: OrchestratorMcpRequestSecretResult,
failure: OrchestratorMcpFailure,
diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json
index 0a03b931fcc..b71053bf272 100644
--- a/apps/server/src/provider/agentContextBudget.json
+++ b/apps/server/src/provider/agentContextBudget.json
@@ -1,5 +1,5 @@
{
"claudeSystemAppendBytes": 5133,
"codexAdditionalContextBytes": 6863,
- "mcpToolListJsonBytes": 47818
+ "mcpToolListJsonBytes": 47809
}
diff --git a/evals/mcp-tool-use/cases/pull-requests.ndjson b/evals/mcp-tool-use/cases/pull-requests.ndjson
index e0c71442200..2e14e45f7f2 100644
--- a/evals/mcp-tool-use/cases/pull-requests.ndjson
+++ b/evals/mcp-tool-use/cases/pull-requests.ndjson
@@ -14,6 +14,7 @@
{"id": "wake-required-passed-rearm-or-shell-wait", "area": "pull-requests", "kind": "bad", "pattern": "on a watch wake: re-arm watch, list PRs, or wait in the shell", "experience": "On a 'required checks passed' wake with a non-required job still running, agents re-called watch_pull_request and listed PRs, or unwatched and started gh run watch.", "rule": "when asked to monitor, watch, or babysit a PR and watch_pull_request is available, call it and end your turn ... do not poll or run your own watcher", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "File PR. Babysit. Merge when you are confident it's good to go."}, {"role": "assistant", "text": "Opened [#124](https://github.com/acme/app/pull/124). T3 Code is watching it; I will merge once checks pass and review findings are clear.", "toolCalls": [{"tool": "Bash", "input": {"command": "git push -u origin fix/socket-retry"}, "result": "To github.com:acme/app.git\n * [new branch] fix/socket-retry -> fix/socket-retry\nbranch 'fix/socket-retry' set up to track 'origin/fix/socket-retry'."}, {"tool": "Bash", "input": {"command": "gh pr create --base main --title \"fix(web): retry the socket once before showing an error\" --fill"}, "result": "https://github.com/acme/app/pull/124"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "alreadyLinked": false}}, {"tool": "watch_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "watching": true, "wasWatching": false}}]}], "mocks": {"watch_pull_request": "default", "list_thread_pull_requests": "default", "link_pull_request": "default", "unwatch_pull_request": "default"}, "world": {"branch": "fix/socket-retry", "commits": [{"sha": "abc1234", "subject": "fix(web): retry the socket once before showing an error"}], "diff": "diff --git a/apps/web/src/socket.ts b/apps/web/src/socket.ts\n--- a/apps/web/src/socket.ts\n+++ b/apps/web/src/socket.ts\n@@ -10,1 +10,2 @@\n- if (error) return showError(error);\n+ if (error && !retried) return reconnect({ retried: true });\n+ if (error) return showError(error);\n", "prs": [{"number": 124, "title": "fix(web): retry the socket once before showing an error", "headRefName": "fix/socket-retry", "checks": [{"name": "Check", "state": "pass"}, {"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test Web", "state": "pending"}], "mergeStateStatus": "UNSTABLE"}]}}, "prompt": "Update on pull request #124 (https://github.com/acme/app/pull/124), which T3 Code is watching for you:\n- All 9 required checks passed on abc1234.\n\nLook into each item and act on it as your task requires. This PR stays linked and watched, so do not link, watch, or list it again. End your turn when you are done, and T3 Code wakes you on the next change. When you hand the work back to the user, call unwatch_pull_request first so the thread returns to their inbox.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests", "link_pull_request", "link_pull_requests", "Monitor", "ScheduleWakeup"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"]}, "notes": "Merges two real failures on the same wake: Claude unwatch + gh run watch, and Codex re-arming watch + list on each wake (see source.also). Wake text is copied from pullRequestWatch.ts. Merging now or ending the turn to wait for the next wake both pass."}
{"id": "secret-webhook-how-to-send", "area": "pull-requests", "kind": "bad", "pattern": "secret asked in chat, or request_secret promised but not called", "experience": "Agents told the user to paste secrets in chat, or said 'give me a token through request_secret' and never called it. request_secret has 0 real calls.", "rule": "When you need a secret from the user ... call request_secret so they enter it privately ... Never ask for a secret in chat", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Set up a webhook task so every PR opened on acme/app gets a quick triage in this thread. Have GitHub sign the requests."}, {"role": "assistant", "text": "I can set that up. I need the webhook secret so T3 Code can check GitHub's signature. Create a secret for the webhook in the repo's webhook settings first."}], "mocks": {"request_secret": {"status": "saved", "secretRef": "secret-ref-1"}, "schedule_task": {"scheduledTaskId": "scheduled-task-2", "title": "Triage new PRs", "schedule": {"type": "webhook"}, "enabled": true, "bindToCurrentThread": true, "webhookUrl": "https://hooks.example.test/t3/abc"}}}, "prompt": "Ok, I made the secret in GitHub. How do I get it to you?", "expect": {"calls": [{"tool": "request_secret"}, {"tool": "schedule_task", "args": {"schedule.type": "webhook", "schedule.signature.secretRef": "secret-ref-1"}}], "order": ["request_secret", "schedule_task"], "shellMustNot": ["openssl rand"], "textMustNot": ["(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Pp]aste (it|them|the (api )?(token|key|secret)s?|your (api )?(token|key|secret)s?)[^.\\n]{0,40}\\b(here|in (the |this )?(chat|thread))\\b", "(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Ss]end (me )?(it|the (api )?(token|key|secret)|your (api )?(token|key|secret))[^.\\n]{0,30}\\b(here|in (the |this )?(chat|thread))\\b", "(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Pp]aste both[^\\n]{0,100}\\b(here|in (the |this )?(chat|thread))\\b", "(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Ss]end me (an?|the|your) [^.\\n]{0,20}(token|key)\\b(?![^.\\n]{0,40}(file|env|\\.env))"], "textMust": ["hooks\\.example\\.test/t3/abc"]}, "notes": "Real misses were shell tokens, where a secretRef cannot help (see secret-shell-token-how-to-send). No webhook task exists in the data, so this is the smallest context where request_secret is the right tool. Mock result shapes are approximations."}
{"id": "secret-shell-token-how-to-send", "area": "pull-requests", "kind": "bad", "pattern": "shell CLI token asked in chat, not through request_secret file delivery", "experience": "Agents asked for API tokens in chat ('send me an API token', 'paste both ...secret-access-key=... URLs here') or promised a 'secure prompt' they never opened.", "rule": "For a shell CLI token, set delivery to file. Never ask for secrets in chat, and never invent one.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "CI failed. Fix it. Keep an eye on the PR."}, {"role": "assistant", "text": "The Cloudflare build check on [#123](https://github.com/acme/app/pull/123) fails in the same second it starts, so no build runs. The Worker builds cleanly here with `wrangler deploy --dry-run`. This machine has no Cloudflare login, so I can't read the build log. I need either the build log or a Cloudflare API token with read access to builds.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh pr checks 123"}, "result": "Cloudflare Pages\tfail\t1s\nLint\tpass\t41s\nTest\tpass\t3m"}, {"tool": "Bash", "input": {"command": "npx wrangler deploy --dry-run"}, "result": "Total Upload: 412.31 KiB / gzip: 98.20 KiB\n--dry-run: exiting now."}, {"tool": "Bash", "input": {"command": "npx wrangler whoami"}, "result": "You are not authenticated. Please run `wrangler login`."}]}], "mocks": {"request_secret": "default"}}, "prompt": "I made a Cloudflare API token with build read access. How do I get it to you?", "expect": {"calls": [{"tool": "request_secret", "args": {"delivery": "file"}}], "textMustNot": ["(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Pp]aste (the (api )?(token|key|secret)s?|your (api )?(token|key|secret)s?)[^.\\n]{0,40}\\b(here|in (the |this )?(chat|thread))\\b", "(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Ss]end (me )?(the (api )?(token|key|secret)|your (api )?(token|key|secret))[^.\\n]{0,30}\\b(here|in (the |this )?(chat|thread))\\b", "(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Pp]aste both[^\\n]{0,100}\\b(here|in (the |this )?(chat|thread))\\b", "(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Ss]end me (an?|the|your) [^.\\n]{0,20}(token|key)\\b(?![^.\\n]{0,40}(file|env|\\.env))"]}, "notes": "Since pingdotgg/t3code#24809, request_secret with delivery 'file' returns a private file path a shell command can read, so it is the right tool for a wrangler token. The runner's default result is a file under T3's secrets directory."}
+{"id":"secret-several-values-one-card","area":"pull-requests","kind":"bad","pattern":"several secrets asked for in one request_secret card","experience":"An agent needed an API application key, application secret and consumer key. To save the user time, it asked for all three in one card labelled '3 lines, in this order', planning to split the file. The card has one masked, single-line field, so the user could not enter them, and the agent had to ask again for values 2 and 3 in two more cards.","rule":"The card has one masked, single-line field. Call once per secret.","privacy":"commit-safe","source":{"thread":"Contributor thread, 2026-11-03: one request_secret card asked for an API application key, application secret and consumer key as three lines"},"setup":{"role":"top","mocks":{"request_secret":"default"}},"prompt":"I copied the OVH application key, application secret and consumer key as three lines in that order. I need you to use them in a shell command later. I only have a minute. Give me the quickest private way to provide them now, then stop before any external request.","expect":{"minCalls":{"request_secret":3},"maxCalls":{"request_secret":3},"distinctCalls":[{"tool":"request_secret","args":{"delivery":"file","label":"~application key","value":null}},{"tool":"request_secret","args":{"delivery":"file","label":"~application secret","value":null}},{"tool":"request_secret","args":{"delivery":"file","label":"~consumer key","value":null}}],"endsTurnAfter":"request_secret","notCalls":["Bash","WebFetch","WebSearch","Agent","delegate_task"]},"notes":"Reproduced through Codex gpt-6.1-sol in the running web app on main and the original PR head: the three-line handoff produced one combined card. Exactly three calls require a separate card per credential; file delivery is required for the later shell command. secret-shell-token-how-to-send is the single-secret control."}
{"id": "schedule-runs-job-first", "area": "pull-requests", "kind": "bad", "pattern": "schedule asked, agent runs the job instead", "experience": "After an approved test run, the user said 'set it with a schedule'. The agent ran the triage query again (about 20 GitHub calls) before scheduling. The user: 'Why are you running the query?'", "rule": "schedule_task creates persistent recurring work in the app scheduler. Pass schedule as a structured object / After scheduling a timer, report the returned cadence and next run time", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Every day at 9pm, I want you to:\n- Spin up Haiku 5.5 delegates to look at PRs with activity in the last 24 hours\n- Audit them for mergeability, value, and novelty\n- Have Opus 5.5 and GPT-6.1 Sol delegates verify the ones that look mergeable\n- Come back with 8 or fewer PRs worth merging or taking over\nDo a test run before we set up the scheduled task."}, {"role": "assistant", "text": "Test run done. 290 PRs had activity in 24 hours. Haiku flagged 31; Opus and Sol confirmed 7: #101 MERGE, #102 MERGE, #103 MERGE, #104 TAKEOVER, #105 MERGE, #106 TAKEOVER, #107 MERGE.\n\nOne lesson for the job: a first-time contributor's PR can look green when CI never ran (it waits for approval). The nightly run must check for that.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh pr list --state open --search \"updated:>=2026-10-23\" --limit 500 --json number --jq length"}, "result": "290"}, {"tool": "delegate_task", "input": {"title": "Haiku PR audit", "mode": "wait", "target": {"providerInstanceId": "claudeAgent", "model": "claude-haiku-5-5"}, "task": "Audit the 290 PRs with activity in the last 24 hours for mergeability, value, and novelty. Flag the ones that look mergeable."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "31 PRs look mergeable: #101-#107, #110, ..."}}, {"tool": "delegate_task", "input": {"title": "Opus verify", "mode": "wait", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Verify the 31 flagged PRs. Say MERGE, TAKEOVER, or SKIP for each."}, "result": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "status": "completed", "workState": "result_available", "summary": "#101 MERGE, #102 MERGE, #103 MERGE, #104 TAKEOVER, #105 MERGE, #106 TAKEOVER, #107 MERGE. #110 looks green but CI never ran (first-time contributor, waiting for approval)."}}, {"tool": "delegate_task", "input": {"title": "Sol verify", "mode": "wait", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol"}, "task": "Verify the 31 flagged PRs. Say MERGE, TAKEOVER, or SKIP for each."}, "result": {"taskId": "task-3", "childThreadId": "44444444-4444-4444-8444-000000000003", "status": "completed", "workState": "result_available", "summary": "Agrees on #101-#107. SKIP the rest."}}]}], "mocks": {"schedule_task": {"scheduledTaskId": "scheduled-task-1", "title": "Nightly PR triage", "schedule": {"type": "fixed_time", "timeOfDay": "21:00"}, "enabled": true, "bindToCurrentThread": true, "nextRunAt": "2026-10-24T21:00:00-07:00"}}}, "prompt": "This looks good, set it with a schedule", "expect": {"calls": [{"tool": "schedule_task", "args": {"schedule.type": "fixed_time", "schedule.timeOfDay": "21:00"}}], "notCalls": ["delegate_task", "run_scheduled_task_now"], "shellMustNot": ["\\bgh (api|pr|search|issue)\\b"], "textMust": ["(9(:00)? ?[Pp]\\.?[Mm]|21:00)"]}, "notes": "Correction theme H. The only schedule correction in the data."}
{"id": "scheduled-run-lists-tasks", "area": "pull-requests", "kind": "bad", "pattern": "list_scheduled_tasks on every scheduled wake", "experience": "Each 4-minute scheduled run called list_scheduled_tasks and got the same single task back (9 times in 33 minutes).", "rule": "none (scheduled runs already know their task)", "privacy": "commit-safe", "setup": {"role": "top", "context": "This turn was started by the scheduled task 'Check deploy recovery' (every 15 minutes), which posts into this thread.", "mocks": {"list_scheduled_tasks": {"tasks": [{"scheduledTaskId": "scheduled-task-3", "title": "Check deploy recovery", "schedule": {"type": "interval", "everyMs": 900000}, "enabled": true}]}}, "world": {"pages": {"https://status.example.test": "<h1>Status</h1><p>Deployments: paused for all plans. Next update in 30 minutes.</p>"}}}, "prompt": "Check https://status.example.test for an explicit announcement that deployments resumed. Do not count green component tiles as recovery. If deployments are still paused, reply in one line and change nothing.", "expect": {"notCalls": ["list_scheduled_tasks", "schedule_task", "delete_scheduled_task"]}, "notes": "No instruction line covers this today; written to what the user wants (no per-wake ceremony). The agent may use WebFetch instead of curl."}
{"id": "creator-links-own-pr", "area": "pull-requests", "kind": "guard", "pattern": "created PR ends up linked, with no watch or list", "experience": "Must keep working: an opened PR ends up linked to the thread, and the agent does not watch or list when no babysit was asked.", "rule": "Call link_pull_request with the full PR URL immediately after creating a PR", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Can we pre-bundle the hot startup paths on shared dev servers? First load takes over a minute."}, {"role": "assistant", "text": "Prototype is on branch perf/prebundle-startup (commit abc1234). First page load on a shared dev server drops from about 64 s to 21 s. Want a PR?", "toolCalls": [{"tool": "Bash", "input": {"command": "git checkout -b perf/prebundle-startup"}, "result": "Switched to a new branch 'perf/prebundle-startup'"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/hotStartupDeps.ts", "content": "// Measured with scripts/bench-first-load.mjs on a shared dev server.\nexport const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/hotStartupDeps.ts"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/vite.config.ts", "content": "import { defineConfig } from 'vite';\nimport { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n\nexport default defineConfig({\n optimizeDeps: { include: HOT_STARTUP_DEPS },\n});\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/vite.config.ts"}, {"tool": "Bash", "input": {"command": "node scripts/bench-first-load.mjs --runs 3"}, "result": "baseline 64.2s 63.8s 65.0s\nprebundled 21.3s 20.9s 21.6s"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"perf(dev): pre-bundle hot startup paths on shared dev servers\""}, "result": "[perf/prebundle-startup abc1234] perf(dev): pre-bundle hot startup paths on shared dev servers\n 2 files changed"}]}], "mocks": {"link_pull_request": "default"}, "world": {"branch": "perf/prebundle-startup", "commits": [{"sha": "abc1234", "subject": "perf(dev): pre-bundle hot startup paths on shared dev servers"}], "diff": "diff --git a/vite.config.ts b/vite.config.ts\n--- a/vite.config.ts\n+++ b/vite.config.ts\n@@ -1,5 +1,6 @@\n import { defineConfig } from 'vite';\n+import { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n \n export default defineConfig({\n- optimizeDeps: {},\n+ optimizeDeps: { include: HOT_STARTUP_DEPS },\n });\ndiff --git a/scripts/hotStartupDeps.ts b/scripts/hotStartupDeps.ts\n--- a/scripts/hotStartupDeps.ts\n+++ b/scripts/hotStartupDeps.ts\n@@ -0,0 +1,2 @@\n+// Measured with scripts/bench-first-load.mjs on a shared dev server.\n+export const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n", "files": {"vite.config.ts": "import { defineConfig } from 'vite';\nimport { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n\nexport default defineConfig({\n optimizeDeps: { include: HOT_STARTUP_DEPS },\n});\n", "scripts/hotStartupDeps.ts": "// Measured with scripts/bench-first-load.mjs on a shared dev server.\nexport const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n", "scripts/bench-first-load.mjs": "// Starts the dev server and times the first page load.\n// Usage: node scripts/bench-first-load.mjs --runs 3\nimport { startDevServer, timeFirstLoad } from './bench-lib.mjs';\n\nconst runs = Number(process.argv[3] ?? 3);\nfor (let i = 0; i < runs; i++) console.log(await timeFirstLoad(await startDevServer()));\n", "scripts/bench-lib.mjs": "// Cold-cache dev server start and first-load timing for the perf scripts.\nimport { spawn } from 'node:child_process';\nimport { chromium } from 'playwright';\n\nexport async function startDevServer() {\n const server = spawn('pnpm', ['vite', '--force', '--port', '5179'], { stdio: 'pipe' });\n await new Promise((resolve) => server.stdout.on('data', (b) => String(b).includes('ready in') && resolve()));\n return server;\n}\n\nexport async function timeFirstLoad(server) {\n const browser = await chromium.launch();\n const page = await browser.newPage();\n const start = performance.now();\n await page.goto('http://localhost:5179', { waitUntil: 'load' });\n const seconds = ((performance.now() - start) / 1000).toFixed(1);\n await browser.close();\n server.kill();\n return `${seconds}s`;\n}\n"}}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}, {"match": "^node .*bench-first-load", "output": "baseline 64.2s 63.8s 65.0s\nprebundled 21.3s 20.9s 21.6s"}]}, "prompt": "Okay do it. Make a good PR.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests"], "linked": ["~/pull/124"]}, "notes": "117 of 117 PR threads linked after the fixes. A shorter text must keep this. Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end.", "maxSteps": 14}
diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts
index d158015b4fb..121814fe2da 100644
--- a/evals/mcp-tool-use/run.test.ts
+++ b/evals/mcp-tool-use/run.test.ts
@@ -71,6 +71,62 @@ it("grades Codex shell and subagent calls with the shared rules", () => {
),
["notCalls Agent", "shellMustNot /sleep/"],
);
+ const distinctCalls = ["application key", "application secret", "consumer key"].map((label) => ({
+ tool: "request_secret",
+ args: { label: "~" + label, delivery: "file", value: null },
+ }));
+ const cards = distinctCalls.map(({ tool, args }) => ({
+ step: 0,
+ tool,
+ input: { ...args, label: args.label.slice(1) },
+ }));
+ const run = (calls: SampleRun["calls"]) => ({ calls, text: "", stopReason: "end_turn" });
+ assert.deepEqual(failed({ distinctCalls }, run(cards)), []);
+ assert.deepEqual(
+ failed(
+ { distinctCalls },
+ run(cards.map((card) => ({ ...card, input: { ...card.input, value: "invented" } }))),
+ ),
+ ["distinctCalls"],
+ );
+ assert.deepEqual(failed({ distinctCalls }, run([cards[0]!])), ["distinctCalls"]);
+ assert.deepEqual(
+ failed(
+ { distinctCalls },
+ run(
+ cards.map((card, i) =>
+ i === 1 ? { ...card, input: { ...card.input, delivery: "secretRef" } } : card,
+ ),
+ ),
+ ),
+ ["distinctCalls"],
+ );
+ assert.deepEqual(
+ failed(
+ { distinctCalls },
+ run([
+ {
+ step: 0,
+ tool: "request_secret",
+ input: { label: "application key, application secret, consumer key", delivery: "file" },
+ },
+ ]),
+ ),
+ ["distinctCalls"],
+ );
+ assert.deepEqual(
+ failed(
+ { distinctCalls },
+ run([
+ {
+ ...cards[0]!,
+ input: { ...cards[0]!.input, label: "application key, application secret, consumer key" },
+ },
+ ...cards.slice(1),
+ ]),
+ ),
+ ["distinctCalls"],
+ );
});
it("matches shellMust, not shellMustNot, against a command's argv lists as a command line", () => {
diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts
index 5b0cf7d1e1e..92ae6fe28cc 100644
--- a/evals/mcp-tool-use/run.ts
+++ b/evals/mcp-tool-use/run.ts
@@ -181,6 +181,8 @@ type McpToolUseCall = typeof McpToolUseCall.Type;
const McpToolUseExpect = Schema.Struct({
/** Each entry needs one matching call. */
calls: Schema.optional(Schema.Array(McpToolUseCall)),
+ /** Each entry matches a different call, and no call matches two entries; tool names are exact. */
+ distinctCalls: Schema.optional(Schema.Array(McpToolUseCall)),
/** No call may match any entry. */
callsMustNot: Schema.optional(Schema.Array(McpToolUseCall)),
notCalls: Schema.optional(Schema.Array(Schema.String)),
@@ -1701,6 +1703,17 @@ export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun, pa
calls.some((call) => tool.split("|").includes(call.tool) && matches(args, call.input));
const label = ({ tool, args }: McpToolUseCall) => (args ? `${tool} ${valueText(args)}` : tool);
for (const call of expect.calls ?? []) add(`calls ${label(call)}`, found(call));
+ if (expect.distinctCalls !== undefined) {
+ const expected = expect.distinctCalls.map(({ tool, args }) => ({ tool, input: args ?? {} }));
+ add(
+ "distinctCalls",
+ calls.every((call) => expected.filter((entry) => matches(entry, call)).length <= 1) &&
+ matchesDistinct(
+ expected,
+ calls.map((request) => ({ request, brief: "" })),
+ ),
+ );
+ }
for (const call of expect.callsMustNot ?? []) add(`callsMustNot ${label(call)}`, !found(call));
for (const tool of expect.notCalls ?? []) add(`notCalls ${tool}`, count(tool) === 0);
if (expect.order !== undefined) {