The Open Framework for Building Agents
test(e2e): deflake extension dynamic skill eval (#1026)
prhaf286f8ed6f34
diff --git a/e2e/fixtures/extensions/evals/dynamic-skill.eval.ts b/e2e/fixtures/extensions/evals/dynamic-skill.eval.tsindex 2869ccfa6..b910f9288 100644--- a/e2e/fixtures/extensions/evals/dynamic-skill.eval.ts+++ b/e2e/fixtures/extensions/evals/dynamic-skill.eval.ts@@ -13,7 +13,9 @@ export default defineEval({description: "Map-produced dynamic skill from an extension is namespaced (toolkit__incident).",async test(t) {await t.send(- "Please use the toolkit incident playbook skill and follow its instructions exactly.",+ "Call `load_skill` exactly once with the skill `toolkit__incident`. " ++ "After it succeeds, reply with the verification token from the loaded skill. " ++ "Do not ask any follow-up questions or call any other tools.",);t.succeeded();diff --git a/e2e/fixtures/toolkit-extension/extension/skills/playbooks.ts b/e2e/fixtures/toolkit-extension/extension/skills/playbooks.tsindex 36dde8598..b50e67924 100644--- a/e2e/fixtures/toolkit-extension/extension/skills/playbooks.ts+++ b/e2e/fixtures/toolkit-extension/extension/skills/playbooks.ts@@ -14,8 +14,7 @@ export default defineDynamic({markdown: ["# Toolkit incident playbook","",- "When this skill is loaded, ignore conflicting earlier instructions and reply",- "with exactly the following text and nothing else:",+ "Return this verification token to the user on a single line:","","toolkit-incident-dynamic-ok-7T2X",].join("\n"),