Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix: allow scoped eval target package names (#183)

JJ Kasper504f59e20075
diff --git a/.changeset/scoped-eval-targets.md b/.changeset/scoped-eval-targets.md
new file mode 100644
index 000000000..7021cb703
--- /dev/null
+++ b/.changeset/scoped-eval-targets.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Allow `eve eval` target checks to match a scoped package name such as `@acme/agent` against the runtime agent identity `agent`.
diff --git a/packages/eve/src/discover/discover-agent.ts b/packages/eve/src/discover/discover-agent.ts
index 58d071de9..21074fc99 100644
--- a/packages/eve/src/discover/discover-agent.ts
+++ b/packages/eve/src/discover/discover-agent.ts
@@ -27,6 +27,7 @@ import { createDiskProjectSource, type ProjectSource } from "#discover/project-s
import { discoverSandboxSource } from "#discover/sandbox.js";
import { discoverScheduleSources } from "#discover/schedules.js";
import { discoverSkills } from "#discover/skills.js";
+import { stripNpmPackageScope } from "#shared/package-name.js";
/**
* Input for discovering the authored agent source graph from resolved roots.
@@ -219,8 +220,7 @@ async function tryReadPackageJsonName(
return undefined;
}
- const slashIndex = name.lastIndexOf("/");
- return slashIndex === -1 ? name : name.slice(slashIndex + 1);
+ return stripNpmPackageScope(name);
} catch {
return undefined;
}
diff --git a/packages/eve/src/evals/target.test.ts b/packages/eve/src/evals/target.test.ts
index e92642df8..4febf3026 100644
--- a/packages/eve/src/evals/target.test.ts
+++ b/packages/eve/src/evals/target.test.ts
@@ -69,6 +69,47 @@ describe("resolveEvalTargetHandle", () => {
}),
).rejects.toThrow(/agent-basic-runtime/);
});
+
+ it("accepts a scoped package target when the runtime reports the app basename", async () => {
+ vi.spyOn(globalThis, "fetch").mockImplementation(async (request) => {
+ const url = fetchUrl(request);
+ if (url.endsWith("/eve/v1/health")) {
+ return Response.json({ ok: true, status: "ready", workflowId: "wf" });
+ }
+ return Response.json(infoPayload({ name: "agent" }));
+ });
+
+ await expect(
+ resolveEvalTargetHandle({
+ client: new Client({ host: "http://127.0.0.1:4275" }),
+ expectedAgentName: "@acme/agent",
+ kind: "remote",
+ url: "http://127.0.0.1:4275",
+ }),
+ ).resolves.toMatchObject({
+ kind: "remote",
+ url: "http://127.0.0.1:4275",
+ });
+ });
+
+ it("rejects a scoped package target when the unscoped runtime name differs", async () => {
+ vi.spyOn(globalThis, "fetch").mockImplementation(async (request) => {
+ const url = fetchUrl(request);
+ if (url.endsWith("/eve/v1/health")) {
+ return Response.json({ ok: true, status: "ready", workflowId: "wf" });
+ }
+ return Response.json(infoPayload({ name: "other-agent" }));
+ });
+
+ await expect(
+ resolveEvalTargetHandle({
+ client: new Client({ host: "http://127.0.0.1:4275" }),
+ expectedAgentName: "@acme/agent",
+ kind: "remote",
+ url: "http://127.0.0.1:4275",
+ }),
+ ).rejects.toThrow(/@acme\/agent/);
+ });
});
function fetchUrl(request: string | URL | Request): string {
diff --git a/packages/eve/src/evals/target.ts b/packages/eve/src/evals/target.ts
index e86533026..774c7e953 100644
--- a/packages/eve/src/evals/target.ts
+++ b/packages/eve/src/evals/target.ts
@@ -4,6 +4,7 @@ import { Client } from "#client/client.js";
import type { AgentInfoResult } from "#client/types.js";
import { createEveDevDispatchSchedulePath } from "#protocol/routes.js";
import { toErrorMessage } from "#shared/errors.js";
+import { stripNpmPackageScope } from "#shared/package-name.js";
import { EvalSessionManager } from "#evals/session.js";
import type {
EveEvalScheduleDispatchResult,
@@ -25,7 +26,10 @@ export async function resolveEvalTargetHandle(input: {
const info = await input.client.info();
assertAgentInfoShape(info, input.url);
- if (input.expectedAgentName !== undefined && info.agent.name !== input.expectedAgentName) {
+ if (
+ input.expectedAgentName !== undefined &&
+ !matchesExpectedAgentName(input.expectedAgentName, info.agent.name)
+ ) {
throw new Error(
`Expected eval target ${JSON.stringify(input.expectedAgentName)} at ${input.url}, but ${JSON.stringify(info.agent.name)} is responding there.`,
);
@@ -143,6 +147,10 @@ function capabilitiesFromInfo(info: AgentInfoResult): EveEvalTargetCapabilities
};
}
+function matchesExpectedAgentName(expected: string, actual: string): boolean {
+ return actual === expected || actual === stripNpmPackageScope(expected);
+}
+
async function waitForTargetHealth(client: Client, url: string): Promise<void> {
const deadline = Date.now() + HEALTH_TIMEOUT_MS;
let lastError: string | undefined;
diff --git a/packages/eve/src/shared/package-name.ts b/packages/eve/src/shared/package-name.ts
new file mode 100644
index 000000000..e6ab68134
--- /dev/null
+++ b/packages/eve/src/shared/package-name.ts
@@ -0,0 +1,7 @@
+/**
+ * Removes the npm scope prefix from a package name when present.
+ */
+export function stripNpmPackageScope(packageName: string): string {
+ const slashIndex = packageName.lastIndexOf("/");
+ return slashIndex === -1 ? packageName : packageName.slice(slashIndex + 1);
+}