Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(e2e): openapi swagger - replace unavailable TfL API (#1607)

Andrew Barba51d383dfd31e
diff --git a/e2e/fixtures/agent-openapi-swagger/agent/connections/petstore-approval.ts b/e2e/fixtures/agent-openapi-swagger/agent/connections/petstore-approval.ts
new file mode 100644
index 000000000..f7aa7c421
--- /dev/null
+++ b/e2e/fixtures/agent-openapi-swagger/agent/connections/petstore-approval.ts
@@ -0,0 +1,10 @@
+import { defineOpenAPIConnection } from "eve/connections";
+import { always } from "eve/tools/approval";
+
+export default defineOpenAPIConnection({
+ approval: always(),
+ baseUrl: "https://petstore.swagger.io/v2",
+ spec: "https://petstore.swagger.io/v2/swagger.json",
+ description: "Approval-gated Swagger Petstore API from its public Swagger 2.0 document.",
+ operations: { allow: ["getInventory"] },
+});
diff --git a/e2e/fixtures/agent-openapi-swagger/agent/connections/petstore.ts b/e2e/fixtures/agent-openapi-swagger/agent/connections/petstore.ts
new file mode 100644
index 000000000..a3f9fa054
--- /dev/null
+++ b/e2e/fixtures/agent-openapi-swagger/agent/connections/petstore.ts
@@ -0,0 +1,7 @@
+import { defineOpenAPIConnection } from "eve/connections";
+
+export default defineOpenAPIConnection({
+ spec: "https://petstore.swagger.io/v2/swagger.json",
+ description: "Swagger Petstore API from its public Swagger 2.0 document.",
+ operations: { allow: ["getInventory"] },
+});
diff --git a/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl-approval.ts b/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl-approval.ts
deleted file mode 100644
index 48c6bae8a..000000000
--- a/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl-approval.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-import { defineOpenAPIConnection } from "eve/connections";
-import { always } from "eve/tools/approval";
-
-export default defineOpenAPIConnection({
- approval: always(),
- baseUrl: "https://api.tfl.gov.uk",
- spec: "https://api.tfl.gov.uk/swagger/docs/v1",
- description:
- "Approval-gated Transport for London Unified API from its public Swagger 2.0 document.",
- operations: { allow: ["Journey_Meta"] },
-});
diff --git a/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl.ts b/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl.ts
deleted file mode 100644
index 7c545ebce..000000000
--- a/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import { defineOpenAPIConnection } from "eve/connections";
-
-export default defineOpenAPIConnection({
- spec: "https://api.tfl.gov.uk/swagger/docs/v1",
- description: "Transport for London Unified API from its public Swagger 2.0 document.",
- operations: { allow: ["Journey_Meta"] },
-});
diff --git a/e2e/fixtures/agent-openapi-swagger/agent/instructions.md b/e2e/fixtures/agent-openapi-swagger/agent/instructions.md
index d442b666b..56475e595 100644
--- a/e2e/fixtures/agent-openapi-swagger/agent/instructions.md
+++ b/e2e/fixtures/agent-openapi-swagger/agent/instructions.md
@@ -1,3 +1,3 @@
# Identity
-You are a helpful assistant. Use the TfL connection whenever the user asks for Transport for London API data.
+You are a helpful assistant. Use the Petstore connection whenever the user asks for Swagger Petstore API data.
diff --git a/e2e/fixtures/agent-openapi-swagger/evals/petstore-swagger-approval.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/petstore-swagger-approval.eval.ts
new file mode 100644
index 000000000..a462929aa
--- /dev/null
+++ b/e2e/fixtures/agent-openapi-swagger/evals/petstore-swagger-approval.eval.ts
@@ -0,0 +1,55 @@
+import { defineEval } from "eve/evals";
+
+const SEARCH_TOOL = "connection_search";
+const PETSTORE_APPROVAL_INVENTORY_TOOL = "petstore-approval__getInventory";
+
+export default defineEval({
+ tags: ["real-model"],
+ description:
+ "OpenAPI connection HITL: an approval-gated Swagger Petstore operation parks before execution.",
+
+ async test(t) {
+ const parked = await t.send(
+ [
+ "Use the `connection_search` tool with connection `petstore-approval` to find the inventory operation.",
+ "Then call `petstore-approval__getInventory` exactly once with an empty object.",
+ "Wait for approval if requested.",
+ "After the tool runs, reply with the exact words `inventory received` if the tool result contains inventory counts.",
+ ].join("\n"),
+ );
+ parked.expectOk();
+
+ t.requireInputRequest({
+ display: "confirmation",
+ optionIds: ["approve", "deny"],
+ toolName: PETSTORE_APPROVAL_INVENTORY_TOOL,
+ });
+ parked.calledTool(PETSTORE_APPROVAL_INVENTORY_TOOL, { status: "pending", count: 1 });
+
+ const approved = await t.respondAll("approve");
+ approved.expectOk();
+
+ approved.event("action.result", {
+ data: {
+ result: { kind: "tool-result", toolName: PETSTORE_APPROVAL_INVENTORY_TOOL },
+ status: "completed",
+ },
+ count: 1,
+ });
+
+ t.succeeded();
+ t.calledTool(SEARCH_TOOL);
+ t.calledTool(PETSTORE_APPROVAL_INVENTORY_TOOL, {
+ output: hasInventoryCounts,
+ count: 1,
+ });
+ t.messageIncludes(/\binventory received\b/iu);
+ },
+});
+
+function hasInventoryCounts(value: unknown): boolean {
+ if (typeof value !== "object" || value === null) return false;
+ const body = (value as { body?: unknown }).body;
+ if (typeof body !== "object" || body === null || Array.isArray(body)) return false;
+ return Object.values(body).some((count) => typeof count === "number");
+}
diff --git a/e2e/fixtures/agent-openapi-swagger/evals/petstore-swagger.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/petstore-swagger.eval.ts
new file mode 100644
index 000000000..61339c27a
--- /dev/null
+++ b/e2e/fixtures/agent-openapi-swagger/evals/petstore-swagger.eval.ts
@@ -0,0 +1,41 @@
+import { defineEval } from "eve/evals";
+
+const SEARCH_TOOL = "connection_search";
+const PETSTORE_INVENTORY_TOOL = "petstore__getInventory";
+
+export default defineEval({
+ tags: ["real-model"],
+ description:
+ "OpenAPI connection smoke: Swagger Petstore's Swagger 2.0 document exposes and calls getInventory.",
+
+ async test(t) {
+ const turn = await t.send(
+ [
+ "Use the `connection_search` tool to find the inventory operation in the `petstore` connection.",
+ "Then call `petstore__getInventory` exactly once with an empty object.",
+ "Reply with the exact words `inventory received` if the tool result contains inventory counts.",
+ ].join("\n"),
+ );
+
+ turn.calledTool(PETSTORE_INVENTORY_TOOL, {
+ output: hasInventoryCounts,
+ count: 1,
+ });
+
+ t.succeeded();
+ t.toolOrder([SEARCH_TOOL, PETSTORE_INVENTORY_TOOL]);
+ t.calledTool(SEARCH_TOOL);
+ t.calledTool(PETSTORE_INVENTORY_TOOL, {
+ output: hasInventoryCounts,
+ count: 1,
+ });
+ t.messageIncludes(/\binventory received\b/iu);
+ },
+});
+
+function hasInventoryCounts(value: unknown): boolean {
+ if (typeof value !== "object" || value === null) return false;
+ const body = (value as { body?: unknown }).body;
+ if (typeof body !== "object" || body === null || Array.isArray(body)) return false;
+ return Object.values(body).some((count) => typeof count === "number");
+}
diff --git a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts
deleted file mode 100644
index 07991fcd0..000000000
--- a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts
+++ /dev/null
@@ -1,72 +0,0 @@
-import { defineEval } from "eve/evals";
-
-const SEARCH_TOOL = "connection_search";
-const TFL_APPROVAL_JOURNEY_MODES_TOOL = "tfl-approval__Journey_Meta";
-
-export default defineEval({
- tags: ["real-model"],
- description:
- "OpenAPI connection HITL: an approval-gated TfL Swagger operation parks before execution.",
-
- async test(t) {
- const parked = await t.send(
- [
- "Use the `connection_search` tool with connection `tfl-approval` to find the TfL journey modes operation.",
- "Then call `tfl-approval__Journey_Meta` exactly once with an empty object.",
- "Wait for approval if requested.",
- "After the tool runs, reply with the exact words `bus` and `tube` if both mode names are present in the tool result.",
- ].join("\n"),
- );
- parked.expectOk();
-
- t.requireInputRequest({
- display: "confirmation",
- optionIds: ["approve", "deny"],
- toolName: TFL_APPROVAL_JOURNEY_MODES_TOOL,
- });
- parked.calledTool(TFL_APPROVAL_JOURNEY_MODES_TOOL, { status: "pending", count: 1 });
-
- const approved = await t.respondAll("approve");
- approved.expectOk();
-
- approved.event("action.result", {
- data: {
- result: { kind: "tool-result", toolName: TFL_APPROVAL_JOURNEY_MODES_TOOL },
- status: "completed",
- },
- count: 1,
- });
-
- t.succeeded();
- t.calledTool(SEARCH_TOOL);
- t.calledTool(TFL_APPROVAL_JOURNEY_MODES_TOOL, {
- output: hasBusAndTube,
- count: 1,
- });
- t.messageIncludes(/\bbus\b/iu);
- t.messageIncludes(/\btube\b/iu);
- },
-});
-
-function hasBusAndTube(value: unknown): boolean {
- if (typeof value !== "object" || value === null) return false;
- const modes = extractModeNames((value as { body?: unknown }).body);
- return modes.has("bus") && modes.has("tube");
-}
-
-function extractModeNames(body: unknown): Set<string> {
- const modes = new Set<string>();
- if (!Array.isArray(body)) {
- return modes;
- }
- for (const item of body) {
- if (typeof item !== "object" || item === null) {
- continue;
- }
- const modeName = (item as { modeName?: unknown }).modeName;
- if (typeof modeName === "string") {
- modes.add(modeName);
- }
- }
- return modes;
-}
diff --git a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts
deleted file mode 100644
index 4f1c68540..000000000
--- a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts
+++ /dev/null
@@ -1,58 +0,0 @@
-import { defineEval } from "eve/evals";
-
-const SEARCH_TOOL = "connection_search";
-const TFL_JOURNEY_MODES_TOOL = "tfl__Journey_Meta";
-
-export default defineEval({
- tags: ["real-model"],
- description:
- "OpenAPI connection smoke: TfL's Swagger 2.0 document exposes and calls Journey_Meta.",
-
- async test(t) {
- const turn = await t.send(
- [
- "Use the `connection_search` tool to find the TfL journey modes operation in the `tfl` connection.",
- "Then call `tfl__Journey_Meta` exactly once with an empty object.",
- "Reply with the exact words `bus` and `tube` if both mode names are present in the tool result.",
- ].join("\n"),
- );
-
- turn.calledTool(TFL_JOURNEY_MODES_TOOL, {
- output: hasBusAndTube,
- count: 1,
- });
-
- t.succeeded();
- t.toolOrder([SEARCH_TOOL, TFL_JOURNEY_MODES_TOOL]);
- t.calledTool(SEARCH_TOOL);
- t.calledTool(TFL_JOURNEY_MODES_TOOL, {
- output: hasBusAndTube,
- count: 1,
- });
- t.messageIncludes(/\bbus\b/iu);
- t.messageIncludes(/\btube\b/iu);
- },
-});
-
-function hasBusAndTube(value: unknown): boolean {
- if (typeof value !== "object" || value === null) return false;
- const modes = extractModeNames((value as { body?: unknown }).body);
- return modes.has("bus") && modes.has("tube");
-}
-
-function extractModeNames(body: unknown): Set<string> {
- const modes = new Set<string>();
- if (!Array.isArray(body)) {
- return modes;
- }
- for (const item of body) {
- if (typeof item !== "object" || item === null) {
- continue;
- }
- const modeName = (item as { modeName?: unknown }).modeName;
- if (typeof modeName === "string") {
- modes.add(modeName);
- }
- }
- return modes;
-}