The Open Framework for Building Agents
fix(e2e): openapi swagger - replace unavailable TfL API (#1607)
Andrew Barba51d383dfd31e
diff --git a/e2e/fixtures/agent-openapi-swagger/agent/connections/petstore-approval.ts b/e2e/fixtures/agent-openapi-swagger/agent/connections/petstore-approval.tsnew file mode 100644index 000000000..f7aa7c421--- /dev/null+++ b/e2e/fixtures/agent-openapi-swagger/agent/connections/petstore-approval.ts@@ -0,0 +1,10 @@+import { defineOpenAPIConnection } from "eve/connections";+import { always } from "eve/tools/approval";++export default defineOpenAPIConnection({+ approval: always(),+ baseUrl: "https://petstore.swagger.io/v2",+ spec: "https://petstore.swagger.io/v2/swagger.json",+ description: "Approval-gated Swagger Petstore API from its public Swagger 2.0 document.",+ operations: { allow: ["getInventory"] },+});diff --git a/e2e/fixtures/agent-openapi-swagger/agent/connections/petstore.ts b/e2e/fixtures/agent-openapi-swagger/agent/connections/petstore.tsnew file mode 100644index 000000000..a3f9fa054--- /dev/null+++ b/e2e/fixtures/agent-openapi-swagger/agent/connections/petstore.ts@@ -0,0 +1,7 @@+import { defineOpenAPIConnection } from "eve/connections";++export default defineOpenAPIConnection({+ spec: "https://petstore.swagger.io/v2/swagger.json",+ description: "Swagger Petstore API from its public Swagger 2.0 document.",+ operations: { allow: ["getInventory"] },+});diff --git a/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl-approval.ts b/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl-approval.tsdeleted file mode 100644index 48c6bae8a..000000000--- a/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl-approval.ts+++ /dev/null@@ -1,11 +0,0 @@-import { defineOpenAPIConnection } from "eve/connections";-import { always } from "eve/tools/approval";--export default defineOpenAPIConnection({- approval: always(),- baseUrl: "https://api.tfl.gov.uk",- spec: "https://api.tfl.gov.uk/swagger/docs/v1",- description:- "Approval-gated Transport for London Unified API from its public Swagger 2.0 document.",- operations: { allow: ["Journey_Meta"] },-});diff --git a/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl.ts b/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl.tsdeleted file mode 100644index 7c545ebce..000000000--- a/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl.ts+++ /dev/null@@ -1,7 +0,0 @@-import { defineOpenAPIConnection } from "eve/connections";--export default defineOpenAPIConnection({- spec: "https://api.tfl.gov.uk/swagger/docs/v1",- description: "Transport for London Unified API from its public Swagger 2.0 document.",- operations: { allow: ["Journey_Meta"] },-});diff --git a/e2e/fixtures/agent-openapi-swagger/agent/instructions.md b/e2e/fixtures/agent-openapi-swagger/agent/instructions.mdindex d442b666b..56475e595 100644--- a/e2e/fixtures/agent-openapi-swagger/agent/instructions.md+++ b/e2e/fixtures/agent-openapi-swagger/agent/instructions.md@@ -1,3 +1,3 @@# Identity-You are a helpful assistant. Use the TfL connection whenever the user asks for Transport for London API data.+You are a helpful assistant. Use the Petstore connection whenever the user asks for Swagger Petstore API data.diff --git a/e2e/fixtures/agent-openapi-swagger/evals/petstore-swagger-approval.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/petstore-swagger-approval.eval.tsnew file mode 100644index 000000000..a462929aa--- /dev/null+++ b/e2e/fixtures/agent-openapi-swagger/evals/petstore-swagger-approval.eval.ts@@ -0,0 +1,55 @@+import { defineEval } from "eve/evals";++const SEARCH_TOOL = "connection_search";+const PETSTORE_APPROVAL_INVENTORY_TOOL = "petstore-approval__getInventory";++export default defineEval({+ tags: ["real-model"],+ description:+ "OpenAPI connection HITL: an approval-gated Swagger Petstore operation parks before execution.",++ async test(t) {+ const parked = await t.send(+ [+ "Use the `connection_search` tool with connection `petstore-approval` to find the inventory operation.",+ "Then call `petstore-approval__getInventory` exactly once with an empty object.",+ "Wait for approval if requested.",+ "After the tool runs, reply with the exact words `inventory received` if the tool result contains inventory counts.",+ ].join("\n"),+ );+ parked.expectOk();++ t.requireInputRequest({+ display: "confirmation",+ optionIds: ["approve", "deny"],+ toolName: PETSTORE_APPROVAL_INVENTORY_TOOL,+ });+ parked.calledTool(PETSTORE_APPROVAL_INVENTORY_TOOL, { status: "pending", count: 1 });++ const approved = await t.respondAll("approve");+ approved.expectOk();++ approved.event("action.result", {+ data: {+ result: { kind: "tool-result", toolName: PETSTORE_APPROVAL_INVENTORY_TOOL },+ status: "completed",+ },+ count: 1,+ });++ t.succeeded();+ t.calledTool(SEARCH_TOOL);+ t.calledTool(PETSTORE_APPROVAL_INVENTORY_TOOL, {+ output: hasInventoryCounts,+ count: 1,+ });+ t.messageIncludes(/\binventory received\b/iu);+ },+});++function hasInventoryCounts(value: unknown): boolean {+ if (typeof value !== "object" || value === null) return false;+ const body = (value as { body?: unknown }).body;+ if (typeof body !== "object" || body === null || Array.isArray(body)) return false;+ return Object.values(body).some((count) => typeof count === "number");+}diff --git a/e2e/fixtures/agent-openapi-swagger/evals/petstore-swagger.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/petstore-swagger.eval.tsnew file mode 100644index 000000000..61339c27a--- /dev/null+++ b/e2e/fixtures/agent-openapi-swagger/evals/petstore-swagger.eval.ts@@ -0,0 +1,41 @@+import { defineEval } from "eve/evals";++const SEARCH_TOOL = "connection_search";+const PETSTORE_INVENTORY_TOOL = "petstore__getInventory";++export default defineEval({+ tags: ["real-model"],+ description:+ "OpenAPI connection smoke: Swagger Petstore's Swagger 2.0 document exposes and calls getInventory.",++ async test(t) {+ const turn = await t.send(+ [+ "Use the `connection_search` tool to find the inventory operation in the `petstore` connection.",+ "Then call `petstore__getInventory` exactly once with an empty object.",+ "Reply with the exact words `inventory received` if the tool result contains inventory counts.",+ ].join("\n"),+ );++ turn.calledTool(PETSTORE_INVENTORY_TOOL, {+ output: hasInventoryCounts,+ count: 1,+ });++ t.succeeded();+ t.toolOrder([SEARCH_TOOL, PETSTORE_INVENTORY_TOOL]);+ t.calledTool(SEARCH_TOOL);+ t.calledTool(PETSTORE_INVENTORY_TOOL, {+ output: hasInventoryCounts,+ count: 1,+ });+ t.messageIncludes(/\binventory received\b/iu);+ },+});++function hasInventoryCounts(value: unknown): boolean {+ if (typeof value !== "object" || value === null) return false;+ const body = (value as { body?: unknown }).body;+ if (typeof body !== "object" || body === null || Array.isArray(body)) return false;+ return Object.values(body).some((count) => typeof count === "number");+}diff --git a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.tsdeleted file mode 100644index 07991fcd0..000000000--- a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts+++ /dev/null@@ -1,72 +0,0 @@-import { defineEval } from "eve/evals";--const SEARCH_TOOL = "connection_search";-const TFL_APPROVAL_JOURNEY_MODES_TOOL = "tfl-approval__Journey_Meta";--export default defineEval({- tags: ["real-model"],- description:- "OpenAPI connection HITL: an approval-gated TfL Swagger operation parks before execution.",-- async test(t) {- const parked = await t.send(- [- "Use the `connection_search` tool with connection `tfl-approval` to find the TfL journey modes operation.",- "Then call `tfl-approval__Journey_Meta` exactly once with an empty object.",- "Wait for approval if requested.",- "After the tool runs, reply with the exact words `bus` and `tube` if both mode names are present in the tool result.",- ].join("\n"),- );- parked.expectOk();-- t.requireInputRequest({- display: "confirmation",- optionIds: ["approve", "deny"],- toolName: TFL_APPROVAL_JOURNEY_MODES_TOOL,- });- parked.calledTool(TFL_APPROVAL_JOURNEY_MODES_TOOL, { status: "pending", count: 1 });-- const approved = await t.respondAll("approve");- approved.expectOk();-- approved.event("action.result", {- data: {- result: { kind: "tool-result", toolName: TFL_APPROVAL_JOURNEY_MODES_TOOL },- status: "completed",- },- count: 1,- });-- t.succeeded();- t.calledTool(SEARCH_TOOL);- t.calledTool(TFL_APPROVAL_JOURNEY_MODES_TOOL, {- output: hasBusAndTube,- count: 1,- });- t.messageIncludes(/\bbus\b/iu);- t.messageIncludes(/\btube\b/iu);- },-});--function hasBusAndTube(value: unknown): boolean {- if (typeof value !== "object" || value === null) return false;- const modes = extractModeNames((value as { body?: unknown }).body);- return modes.has("bus") && modes.has("tube");-}--function extractModeNames(body: unknown): Set<string> {- const modes = new Set<string>();- if (!Array.isArray(body)) {- return modes;- }- for (const item of body) {- if (typeof item !== "object" || item === null) {- continue;- }- const modeName = (item as { modeName?: unknown }).modeName;- if (typeof modeName === "string") {- modes.add(modeName);- }- }- return modes;-}diff --git a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.tsdeleted file mode 100644index 4f1c68540..000000000--- a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts+++ /dev/null@@ -1,58 +0,0 @@-import { defineEval } from "eve/evals";--const SEARCH_TOOL = "connection_search";-const TFL_JOURNEY_MODES_TOOL = "tfl__Journey_Meta";--export default defineEval({- tags: ["real-model"],- description:- "OpenAPI connection smoke: TfL's Swagger 2.0 document exposes and calls Journey_Meta.",-- async test(t) {- const turn = await t.send(- [- "Use the `connection_search` tool to find the TfL journey modes operation in the `tfl` connection.",- "Then call `tfl__Journey_Meta` exactly once with an empty object.",- "Reply with the exact words `bus` and `tube` if both mode names are present in the tool result.",- ].join("\n"),- );-- turn.calledTool(TFL_JOURNEY_MODES_TOOL, {- output: hasBusAndTube,- count: 1,- });-- t.succeeded();- t.toolOrder([SEARCH_TOOL, TFL_JOURNEY_MODES_TOOL]);- t.calledTool(SEARCH_TOOL);- t.calledTool(TFL_JOURNEY_MODES_TOOL, {- output: hasBusAndTube,- count: 1,- });- t.messageIncludes(/\bbus\b/iu);- t.messageIncludes(/\btube\b/iu);- },-});--function hasBusAndTube(value: unknown): boolean {- if (typeof value !== "object" || value === null) return false;- const modes = extractModeNames((value as { body?: unknown }).body);- return modes.has("bus") && modes.has("tube");-}--function extractModeNames(body: unknown): Set<string> {- const modes = new Set<string>();- if (!Array.isArray(body)) {- return modes;- }- for (const item of body) {- if (typeof item !== "object" || item === null) {- continue;- }- const modeName = (item as { modeName?: unknown }).modeName;- if (typeof modeName === "string") {- modes.add(modeName);- }- }- return modes;-}