The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
import { randomUUID } from "node:crypto";
import { test, expect, type Route } from "@playwright/test";
const fulfill = (route: Route, body: unknown, status = 200) =>
route.fulfill({
status,
contentType: "application/json",
body: JSON.stringify(body),
});
test("AgentMail setup and email work through the normal task conversation", async ({
page,
request,
}) => {
await page.setViewportSize({ width: 1440, height: 1080 });
const settings = await request.patch("/api/instance/settings/experimental", {
data: { enableChatConnectors: false },
});
expect(settings.ok()).toBeTruthy();
const created = await request.post("/api/companies", {
data: { name: `AgentMail browser ${Date.now()}` },
});
expect(created.ok()).toBeTruthy();
const company = await created.json();
const agentResponse = await request.post(
`/api/companies/${company.id}/agents`,
{
data: {
name: "Mail agent",
role: "qa",
adapterType: "process",
adapterConfig: {
command: process.execPath,
args: ["-e", "process.exit(0)"],
},
},
},
);
expect(agentResponse.ok()).toBeTruthy();
const agent = await agentResponse.json();
const taskResponse = await request.post(
`/api/companies/${company.id}/issues`,
{ data: { title: "Customer email", status: "backlog" } },
);
expect(taskResponse.ok()).toBeTruthy();
const task = await taskResponse.json();
const inbox = {
id: randomUUID(),
companyId: company.id,
connectionId: randomUUID(),
assignedAgentId: agent.id,
address: "[email protected]",
status: "active",
receiveMode: "websocket",
lastError: null,
lastSyncAt: new Date().toISOString(),
};
const organizationConnectionId = randomUUID();
let connected = false;
let reservedInbox: typeof inbox | null = null;
const setupRequests: { idempotencyKey: string; username?: string }[] = [];
const addressTakenError = "This email address is already in use. Choose a different address.";
const permissionError = "AgentMail did not allow Paperclip to create an inbox-scoped API key. Check your API key permissions and AgentMail account limits, then try again.";
const sends: any[] = [];
const conversationId = randomUUID();
const thread = {
conversationId,
issueId: task.id,
endpoint: inbox,
subject: "Customer email",
messages: [
{
id: randomUUID(),
providerMessageId: "incoming-message",
from: "Customer <[email protected]>",
to: [inbox.address],
cc: ["[email protected]"],
bcc: ["[email protected]"],
subject: "Customer email",
direction: "inbound",
text: "Can you help?",
fullText: "Can you help?\nEarlier quoted context",
commentId: null,
attachmentIds: [],
timestamp: new Date().toISOString(),
automatic: false,
},
],
publications: [] as any[],
};
await page.route("**/api/instance/settings/experimental", (route) =>
fulfill(route, { enableChatConnectors: false }),
);
await page.route(`**/api/companies/${company.id}/tools/connections`, route =>
fulfill(route, { connections: [{
id: organizationConnectionId, name: "AgentMail", status: "active", enabled: true,
config: { provider: "agentmail", emailCredential: true }, createdAt: "2026-09-30T14:00:00Z",
}] }),
);
await page.route(`**/api/companies/${company.id}/chat-endpoints`, route =>
fulfill(route, reservedInbox ? [{ ...reservedInbox, provider: "agentmail", assignedAgentName: agent.name,
providerAccountLabel: reservedInbox.address }] : []),
);
await page.route(`**/api/tool-connections/${inbox.connectionId}`, route =>
fulfill(route, { config: { credentialConnectionId: organizationConnectionId } }),
);
await page.route(`**/api/tool-connections/${organizationConnectionId}/grants`, route =>
fulfill(route, { grants: [{ status: "active", kind: "organization" }], capabilities: { canConfigure: true } }),
);
await page.route(`**/api/tool-connections/${organizationConnectionId}/installs`, route =>
fulfill(route, { installs: [{ targetType: "agent", targetId: agent.id }] }),
);
await page.route(`**/api/chat-endpoints/${inbox.id}/conversations`, route =>
fulfill(route, [{ id: conversationId, externalLabel: "Customer email", issueId: task.id,
issueTitle: task.title, issueIdentifier: task.identifier, state: "active" }]),
);
await page.route(`**/api/chat-endpoints/${inbox.id}/activity?*`, route =>
fulfill(route, { items: [{ id: randomUUID(), kind: "delivery", status: "processed",
summary: "Email received", createdAt: new Date().toISOString() }], nextCursor: null }),
);
await page.route("**/api/**/email/**", async (route) => {
const url = new URL(route.request().url()),
method = route.request().method();
if (url.pathname.endsWith("/connections") && method === "GET") return fulfill(route, [
{ id: organizationConnectionId, label: "AgentMail account key", scope: "organization", createdAt: "2026-09-30T14:00:00Z" },
{ id: inbox.connectionId, label: "AgentMail inbox key", scope: "inbox", createdAt: "2026-10-01T14:00:00Z" },
]);
if (url.pathname.endsWith("/connections") && method === "POST") {
expect(route.request().postDataJSON()).toMatchObject({ apiKey: "inbox-test-key", agentIds: [agent.id], allAgents: false });
return fulfill(route, { id: inbox.connectionId }, 201);
}
if (url.pathname.endsWith("/check-address")) {
const body = route.request().postDataJSON();
return fulfill(route, { address: `${body.username}@${body.domain}`, status: body.username === "taken" ? "taken" : "unknown" });
}
if (url.pathname.endsWith(`/connections/${inbox.connectionId}/inspect`))
return fulfill(route, {
scope: { scope_type: "inbox" }, inboxes: [{ inbox_id: inbox.address }], domains: [],
});
if (url.pathname.endsWith("/inspect"))
return fulfill(route, {
scope: { scope_type: "organization" },
inboxes: [{ inbox_id: inbox.address }],
domains: [
{
domain_id: "domain-id",
domain: "verified.example.test",
status: "VERIFIED",
},
],
});
if (url.pathname.endsWith("/inboxes") && method === "GET")
return fulfill(route, connected ? [inbox] : reservedInbox ? [reservedInbox] : []);
if (url.pathname.endsWith("/inboxes") && method === "POST") {
const body = route.request().postDataJSON();
expect(body.receiveMode).toBe("websocket");
expect(body.assignedAgentId).toBe(agent.id);
expect(body.credentialConnectionId).toBe(organizationConnectionId);
setupRequests.push(body);
if (setupRequests.length === 1) return fulfill(route, { error: addressTakenError, code: "agentmail_address_taken",
details: { field: "username", providerStatus: 403, operation: "create_inbox" } }, 409);
if (setupRequests.length === 2) {
reservedInbox = { ...inbox, id: body.idempotencyKey, address: `${body.username}@${body.domain}`, status: "draft" };
return fulfill(route, {
error: permissionError,
details: { code: "agentmail_request_failed", providerStatus: 403, operation: "create_inbox_key" },
}, 422);
}
connected = true;
return fulfill(route, inbox, 201);
}
if (url.pathname.endsWith(`/tasks/${task.id}`))
return fulfill(route, thread);
if (url.pathname.endsWith("/send")) {
const input = route.request().postDataJSON();
sends.push(input);
const publication = {
id: input.idempotencyKey,
issueId: input.parentIssueId ? randomUUID() : task.id,
conversationId,
outcome: "queued",
error: null,
providerMessageId: null,
};
thread.publications.push(publication);
return fulfill(route, publication, 202);
}
return fulfill(route, null);
});
await page.route(`**/api/chat-endpoints/${inbox.id}`, (route) =>
fulfill(route, {
...inbox,
provider: "agentmail",
setup: { step: "complete" },
capabilities: {},
assignedAgentName: agent.name,
botExternalId: inbox.address,
}),
);
await page.goto(`/${company.issuePrefix}/apps`);
await page.getByRole("button", { name: /^(Connect|Add connection) AgentMail$/ }).click();
await expect(
page.getByRole("heading", { name: "Give an agent an email address" }),
).toBeVisible();
await page.getByLabel("Agent", { exact: true }).click();
await page.getByPlaceholder("Filter agents").fill("Mail agent");
await expect(page.getByRole("button", { name: "Select Mail agent", exact: true }).locator('[data-slot="agent-avatar"]')).toBeVisible();
await page.getByRole("button", { name: "Select Mail agent", exact: true }).click();
await expect(page.locator('#email-agent [data-slot="agent-avatar"]')).toBeVisible();
await expect(page.getByLabel("API key", { exact: true })).toHaveValue(organizationConnectionId);
// A deliberately selected restricted key stays at credentials until the user
// chooses a usable account key or explicitly requests that existing inbox.
await page.getByLabel("API key", { exact: true }).selectOption(inbox.connectionId);
await page.getByRole("button", { name: "Continue", exact: true }).click();
await expect(page.getByText(`That key only connects ${inbox.address}.`, { exact: false })).toBeVisible();
await expect(page.getByLabel("Mail agent’s email address", { exact: true })).toHaveCount(0);
await page.screenshot({ path: test.info().outputPath("agentmail-inbox-key-recovery.png"), fullPage: true });
await page.getByLabel("API key", { exact: true }).selectOption(organizationConnectionId);
await page.getByRole("button", { name: "Continue", exact: true }).click();
const addressField = page.getByLabel("Mail agent’s email address", { exact: true });
await expect(addressField).toBeEditable();
await expect(page.getByRole("heading", { name: "How it Works", exact: true })).toBeVisible();
await expect(page.getByLabel("Email domain", { exact: true })).toHaveValue("verified.example.test");
await page.getByLabel("Email domain", { exact: true }).selectOption("agentmail.to");
await expect(page.getByLabel("Email domain", { exact: true })).toHaveValue("agentmail.to");
await page.getByLabel("Email domain", { exact: true }).selectOption("verified.example.test");
await addressField.fill("taken");
await expect(page.getByRole("alert")).toHaveText(addressTakenError);
expect(setupRequests).toHaveLength(0);
await page.getByRole("button", { name: "[email protected]", exact: true }).click();
await expect(addressField).toHaveValue("taken-agent");
await expect(page.getByRole("alert")).toHaveCount(0);
await addressField.fill("mail-agent");
await expect(page.getByRole("button", { name: "Review email address" })).toHaveCount(0);
await page.getByText("Advanced options", { exact: true }).click();
await expect(page.getByLabel("Email domain", { exact: true }).locator("option", { hasText: "verified.example.test" })).toHaveCount(1);
await page.getByRole("button", { name: "Review trust settings" }).click();
const trustDialog = page.getByRole("dialog");
await trustDialog.getByRole("combobox").first().selectOption("low_trust_review");
await trustDialog.getByRole("combobox").nth(1).selectOption("root_issue");
await trustDialog.getByRole("combobox").nth(2).selectOption(task.id);
await trustDialog.getByRole("button", { name: "Save trust settings" }).click();
await expect(page.getByText("Low-trust review configured")).toBeVisible();
await page.getByRole("button", { name: "Review trust settings" }).click();
await page.getByRole("dialog").getByRole("combobox").first().selectOption("standard");
await page.getByRole("button", { name: "Save trust settings" }).click();
await expect(trustDialog).not.toBeVisible();
const savedAgent = await (await request.get(`/api/agents/${agent.id}`)).json();
expect(savedAgent.permissions.authorizationPolicy).toEqual({});
await expect(page.getByRole("link", { name: "Set up allowlists ↗" })).toHaveAttribute(
"href", "https://docs.agentmail.to/knowledge-base/allowlists-blocklists");
await page.getByText("Advanced options", { exact: true }).click();
await page.getByRole("button", { name: "Create email address", exact: true }).click();
await expect(page.getByRole("alert")).toHaveText(addressTakenError);
await expect(addressField).toHaveAttribute("aria-invalid", "true");
await expect(page.getByRole("button", { name: "Create email address", exact: true })).toBeDisabled();
await page.screenshot({ path: test.info().outputPath("agentmail-address-taken.png"), fullPage: true });
await addressField.fill("mail-agent-free");
await expect(page.getByRole("alert")).toHaveCount(0);
await page.reload();
await expect(addressField).toHaveValue("mail-agent-free");
await page.getByRole("button", { name: "Create email address", exact: true }).click();
await expect(page.getByText(permissionError, { exact: true })).toBeVisible();
await expect(page.getByText("Internal server error", { exact: true })).toHaveCount(0);
await expect(page.getByRole("heading", { name: "Your agent’s email is ready" })).toHaveCount(0);
await page.screenshot({ path: test.info().outputPath("agentmail-permission-denied.png"), fullPage: true });
await page.getByRole("link", { name: "Connectors", exact: true }).click();
await page.getByRole("button", { name: "Finish setup", exact: true }).click();
await expect(page.getByText("[email protected]", { exact: true })).toBeVisible();
await expect(addressField).toHaveCount(0);
await expect(page.getByRole("button", { name: "Choose a different address", exact: true })).toBeVisible();
await page.getByRole("button", { name: "Finish connecting", exact: true }).click();
await expect(page.getByRole("heading", { name: "Your agent’s email is ready" })).toBeVisible();
expect(setupRequests.map(input => input.username)).toEqual(["mail-agent", "mail-agent-free", undefined]);
expect(setupRequests[2]).toMatchObject({ inboxId: "[email protected]" });
expect(new Set(setupRequests.map(input => input.idempotencyKey)).size).toBe(1);
await page.getByRole("button", { name: "Email settings", exact: true }).click();
await expect(page.getByRole("heading", { name: inbox.address, exact: true })).toBeVisible();
const navigation = page.getByRole("navigation", { name: "Chat connection" });
await navigation.getByRole("link", { name: "Access", exact: true }).click();
await expect(page.getByText("Any human in the organization", { exact: true })).toBeVisible();
await expect(page.getByRole("button", { name: "Reconnect inbox", exact: true })).toHaveCount(0);
await navigation.getByRole("link", { name: "Conversations", exact: true }).click();
await expect(page.getByRole("list", { name: "Conversations" })).toBeVisible();
await expect(page.getByRole("list", { name: "Conversations" }).locator(`a[href="/${company.issuePrefix}/issues/${task.id}"]`)).toHaveAttribute("href", `/${company.issuePrefix}/issues/${task.id}`);
await page.reload();
await expect(page.getByRole("list", { name: "Conversations" })).toBeVisible();
await navigation.getByRole("link", { name: "Activity", exact: true }).click();
await expect(page.getByText("Email received", { exact: true })).toBeVisible();
await expect(page.getByRole("heading", { name: "Connection activity", exact: true })).toBeVisible();
await page.screenshot({ path: test.info().outputPath("agentmail-management-tabs.png"), fullPage: true });
await navigation.getByRole("link", { name: "Settings", exact: true }).click();
await expect(page.getByRole("heading", { name: "Receiving email", exact: true })).toBeVisible();
const copyAddress = page.getByRole("button", { name: "Copy email address", exact: true });
await expect(copyAddress).toHaveText(inbox.address);
await page.context().grantPermissions(["clipboard-read", "clipboard-write"]);
await copyAddress.click();
await expect(page.getByRole("status").filter({ hasText: "Copied!" })).toHaveCSS("opacity", "1");
expect(await page.evaluate(() => navigator.clipboard.readText())).toBe(inbox.address);
await expect(page.getByRole("link", { name: "View inbox", exact: true })).toHaveAttribute(
"href", `https://console.agentmail.to/dashboard/inboxes/${encodeURIComponent(inbox.address)}`);
await page.screenshot({ path: test.info().outputPath("agentmail-settings-copy-inbox.png"), fullPage: true });
await expect(page.getByLabel("New API key", { exact: true })).not.toBeVisible();
await page.locator("summary").filter({ hasText: "Reconnect inbox" }).click();
await expect(page.getByLabel("New API key", { exact: true })).toBeVisible();
await expect(page.getByRole("button", { name: "Reconnect inbox", exact: true })).toBeVisible();
await page.screenshot({ path: test.info().outputPath("agentmail-settings-reconnect.png"), fullPage: true });
await page.goto(`/${company.issuePrefix}/issues/${task.identifier}`);
const email = page.getByRole("article", { name: "Email received", exact: true });
await expect(email).toBeVisible();
await expect(email.getByText("Can you help?", { exact: true })).toBeVisible();
await expect(page.getByRole("button", {
name: /^(Internal comment|Email reply|Start email child task)$/,
})).toHaveCount(0);
await expect(email.getByText("Bcc: [email protected]")).not.toBeVisible();
await email.getByText("Email details", { exact: true }).click();
await expect(email.getByText("Bcc: [email protected]")).toBeVisible();
const composer = page.locator('[contenteditable="true"]').last();
await expect(composer).toBeEditable();
const instruction = "Please reply to the customer and confirm Friday delivery.";
await composer.fill(instruction);
await page.getByRole("button", { name: "Send", exact: true }).click();
await expect.poll(async () => {
const comments = await (await request.get(`/api/issues/${task.id}/comments`)).json();
return comments.some((comment: { body: string }) => comment.body.includes(instruction));
}).toBe(true);
// Task instructions persist normally; only an explicit agent action sends mail.
expect(sends).toHaveLength(0);
await page.screenshot({
path: test.info().outputPath("email-task-conversation.png"),
fullPage: true,
});
});