Comp AI CRM is an open source, CRM designed for AI agents. Agentic-first CRM.
TypeScript
import { db } from "@crm/db";
import { agentManifest } from "@crm/validation/agent-manifest";
import { defineEval } from "eve/evals";
import { equals, satisfies } from "eve/evals/expect";
export default defineEval({
description:
"Turn a short outcome request into a safe, review-only team agent through the durable CRM channel.",
tags: ["builder", "integration"],
timeoutMs: 180_000,
async test(t) {
const secret = process.env.AGENT_BRIDGE_SECRET?.trim();
if (
!process.env.DATABASE_URL ||
!secret ||
(!process.env.AI_GATEWAY_API_KEY && !process.env.VERCEL_OIDC_TOKEN)
) {
t.skip(
"Requires DATABASE_URL, AGENT_BRIDGE_SECRET, and an AI Gateway credential.",
);
return;
}
const suffix = crypto.randomUUID();
const userId = `builder-eval-user-${suffix}`;
let conversationId: string | null = null;
let sessionId: string | null = null;
try {
await db.user.create({
data: {
id: userId,
name: "Agent Builder Eval",
email: `${userId}@example.test`,
},
});
const conversation = await db.agentConversation.create({
data: {
kind: "BUILDER",
userId,
submissions: {
create: {
submittedById: userId,
clientRequestId: crypto.randomUUID(),
commandType: "CREATE_AGENT",
message: {
text: "Build me an agent that gives me a snapshot of our CRM accounts.",
resources: [],
},
},
},
},
select: { id: true },
});
conversationId = conversation.id;
const response = await t.target.fetch("/internal/crm/builder-dispatch", {
method: "POST",
headers: { authorization: `Bearer ${secret}` },
});
await t.require(response.status, equals(202));
sessionId = await waitForBuilderSession(conversation.id, t.signal);
await t.require(
sessionId,
satisfies<string | null>(
(value) => value !== null,
"builder session started",
),
);
const session = await t.target.attachSession(sessionId as string);
session.succeeded();
session.calledSubagent("agent_builder", { count: 1 });
session.notCalledTool("record_fact");
session.notCalledTool("record_job_change");
const saved = await db.agentConversation.findUniqueOrThrow({
where: { id: conversation.id },
select: {
agent: {
select: {
status: true,
versions: {
orderBy: { number: "desc" },
take: 1,
select: { status: true, manifest: true },
},
},
},
builderArtifacts: {
where: { status: "READY" },
select: { path: true },
},
},
});
t.check(saved.agent?.status, equals("DRAFT"));
t.check(saved.agent?.versions[0]?.status, equals("READY"));
t.check(saved.builderArtifacts.length, equals(3));
t.check(
saved.agent?.versions[0]?.manifest,
satisfies((value) => {
const parsed = agentManifest.safeParse(value);
return (
parsed.success &&
parsed.data.dataScope.mode === "WORKSPACE" &&
parsed.data.actions.length === 1 &&
parsed.data.actions[0]?.type === "run.summary"
);
}, "saved manifest is workspace-scoped and side-effect-free"),
);
} finally {
await cleanupBuilderEval(userId, conversationId, sessionId);
}
},
});
async function waitForBuilderSession(
conversationId: string,
signal: AbortSignal,
): Promise<string | null> {
for (let attempt = 0; attempt < 600; attempt += 1) {
if (signal.aborted) return null;
const conversation = await db.agentConversation.findUnique({
where: { id: conversationId },
select: { sessionId: true },
});
if (conversation?.sessionId) return conversation.sessionId;
await new Promise((resolve) => setTimeout(resolve, 100));
}
return null;
}
async function cleanupBuilderEval(
userId: string,
conversationId: string | null,
sessionId: string | null,
) {
const agents = await db.agentDefinition.findMany({
where: { createdById: userId },
select: { id: true },
});
const agentIds = agents.map((agent) => agent.id);
if (sessionId) await db.agentEvent.deleteMany({ where: { sessionId } });
if (conversationId) {
await db.agentBuilderArtifact.deleteMany({ where: { conversationId } });
}
if (agentIds.length > 0) {
await db.agentBuilderArtifact.deleteMany({
where: { version: { agentId: { in: agentIds } } },
});
await db.agentAuditEvent.deleteMany({
where: { agentId: { in: agentIds } },
});
await db.agentTrigger.deleteMany({ where: { agentId: { in: agentIds } } });
await db.agentDefinition.updateMany({
where: { id: { in: agentIds } },
data: { currentVersionId: null },
});
await db.agentVersion.deleteMany({ where: { agentId: { in: agentIds } } });
await db.agentDefinition.deleteMany({ where: { id: { in: agentIds } } });
}
if (conversationId) {
await db.agentConversation.deleteMany({ where: { id: conversationId } });
}
await db.user.deleteMany({ where: { id: userId } });
}