Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): MCP action logs go to the job log, with objective and api_job_id (#5092)

Rakshith Ramprakash979783635318
diff --git a/apps/api/src/controllers/v2/mcp-action-logs.ts b/apps/api/src/controllers/v2/mcp-action-logs.ts
index 703219f2f..c8cc030f6 100644
--- a/apps/api/src/controllers/v2/mcp-action-logs.ts
+++ b/apps/api/src/controllers/v2/mcp-action-logs.ts
@@ -1,14 +1,14 @@
import { Request, Response } from "express";
import { config } from "../../config";
-import { db, dbRr } from "../../db/connection";
+import { db } from "../../db/connection";
import { ErrorResponse, RequestWithAuth, TeamFlags } from "./types";
import {
authorizeMcpActionLogViewer,
listMcpActionLogs,
McpActionLogAuthorizationError,
+ McpActionLogUnavailableError,
McpActionLogValidationError,
normalizeMcpActionLogInput,
- purgeMcpActionLogsForTeam,
recordMcpActionLog,
resolveMcpActionLogTeamPolicy,
validateMcpActionLogActor,
@@ -44,14 +44,13 @@ export async function ingestMcpActionLogController(
});
}
if (isMcpActionLogZdrTeam(team.flags)) {
- await purgeMcpActionLogsForTeam(db, input.team_id);
return res.status(202).json({
success: true,
disposition: "zero-data-retention",
id: null,
});
}
- const result = await recordMcpActionLog(db, input);
+ const result = await recordMcpActionLog(input);
return res.status(202).json({ success: true, ...result });
} catch (error) {
if (error instanceof McpActionLogValidationError) {
@@ -91,7 +90,6 @@ export async function listMcpActionLogsController(
});
}
if (isMcpActionLogZdrTeam(team.flags)) {
- await purgeMcpActionLogsForTeam(db, req.auth.team_id);
return res.status(200).json({
success: true,
data: [],
@@ -99,7 +97,7 @@ export async function listMcpActionLogsController(
});
}
const parsedLimit = Number.parseInt(String(req.query.limit ?? "50"), 10);
- const result = await listMcpActionLogs(dbRr, req.auth.team_id, {
+ const result = await listMcpActionLogs(req.auth.team_id, {
limit: Number.isFinite(parsedLimit) ? parsedLimit : 50,
cursor: typeof req.query.cursor === "string" ? req.query.cursor : null,
});
@@ -111,6 +109,9 @@ export async function listMcpActionLogsController(
if (error instanceof McpActionLogValidationError) {
return res.status(400).json({ success: false, error: error.message });
}
+ if (error instanceof McpActionLogUnavailableError) {
+ return res.status(503).json({ success: false, error: error.message });
+ }
throw error;
}
}
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
index 74bc02da7..0eacb9f01 100644
--- a/apps/api/src/db/schema/public.ts
+++ b/apps/api/src/db/schema/public.ts
@@ -442,38 +442,6 @@ export const products = pgTable("products", {
type: text("type"),
});
-export const mcp_action_logs = pgTable(
- "mcp_action_logs",
- {
- id: uuid("id").defaultRandom().primaryKey(),
- team_id: uuid("team_id").notNull(),
- user_id: uuid("user_id"),
- api_key_id: bigintNum("api_key_id"),
- oauth_client_id: text("oauth_client_id"),
- auth_type: text("auth_type").notNull(),
- tool_name: text("tool_name").notNull(),
- status: text("status").notNull(),
- request_id: uuid("request_id").notNull(),
- client_name: text("client_name"),
- client_version: text("client_version"),
- error_class: text("error_class"),
- resource: text("resource").notNull(),
- created_at: ts("created_at").notNull().defaultNow(),
- expires_at: ts("expires_at").notNull(),
- },
- table => [
- // Mirrors the DB migration and is required by recordMcpActionLog's
- // ON CONFLICT (team_id, request_id) idempotency contract.
- unique("mcp_action_logs_team_request_unique").on(
- table.team_id,
- table.request_id,
- ),
- // Mirrors the DB migration and keeps the retention worker's bounded
- // expiry scan indexed as the table grows.
- index("mcp_action_logs_expires_idx").on(table.expires_at),
- ],
-);
-
export const search_feedback = pgTable("search_feedback", {
id: uuid("id").notNull().defaultRandom(),
search_id: uuid("search_id"),
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 31969064e..3d5a5e20a 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -28,8 +28,6 @@ import { v2Router } from "./routes/v2";
import { exchangeRouter } from "./routes/exchange";
import { labsRouter } from "./routes/labs";
import { registerMcpActionLogIngestRoute } from "./routes/mcp-action-logs";
-import { startMcpActionLogRetentionWorkerIfEnabled } from "./services/mcp/action-logs";
-import { db } from "./db/connection";
import { nuqShutdown } from "./services/worker/nuq";
import { getErrorContactMessage } from "./lib/deployment";
import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";
@@ -154,10 +152,6 @@ async function startServer(port = DEFAULT_PORT) {
throw error;
}
- const mcpActionLogRetention = startMcpActionLogRetentionWorkerIfEnabled({
- enabled: config.MCP_ACTION_LOG_STORAGE_ENABLED,
- db,
- });
const server = app.listen(Number(port), HOST, (error?: Error) => {
if (error) {
logger.error("Failed to start HTTP server", { error, port, host: HOST });
@@ -169,7 +163,6 @@ async function startServer(port = DEFAULT_PORT) {
const exitHandler = async () => {
logger.info("SIGTERM signal received: closing HTTP server");
- mcpActionLogRetention?.stop();
if (config.IS_KUBERNETES) {
// Account for GCE load balancer drain timeout
logger.info("Waiting 60s for GCE load balancer drain timeout");
diff --git a/apps/api/src/lib/mcp-action-log-config.test.ts b/apps/api/src/lib/mcp-action-log-config.test.ts
index ae47aa40c..f0151a6ad 100644
--- a/apps/api/src/lib/mcp-action-log-config.test.ts
+++ b/apps/api/src/lib/mcp-action-log-config.test.ts
@@ -22,6 +22,7 @@ describe("MCP action log configuration", () => {
expect(errors.map(error => error.path)).toEqual([
"MCP_ACTION_LOG_WRITES_ENABLED",
"MCP_ACTION_LOG_SECRET",
+ "PUBSUB_CREDENTIALS",
]);
});
@@ -31,6 +32,7 @@ describe("MCP action log configuration", () => {
MCP_ACTION_LOG_STORAGE_ENABLED: true,
MCP_ACTION_LOG_WRITES_ENABLED: true,
MCP_ACTION_LOG_SECRET: "secret",
+ PUBSUB_CREDENTIALS: "credentials",
USE_DB_AUTHENTICATION: true,
}),
).toEqual([]);
diff --git a/apps/api/src/lib/mcp-action-log-config.ts b/apps/api/src/lib/mcp-action-log-config.ts
index 8b8e879ca..a717bc9b1 100644
--- a/apps/api/src/lib/mcp-action-log-config.ts
+++ b/apps/api/src/lib/mcp-action-log-config.ts
@@ -2,6 +2,7 @@ type McpActionLogConfigInput = {
MCP_ACTION_LOG_STORAGE_ENABLED?: boolean;
MCP_ACTION_LOG_WRITES_ENABLED?: boolean;
MCP_ACTION_LOG_SECRET?: string;
+ PUBSUB_CREDENTIALS?: string;
USE_DB_AUTHENTICATION?: boolean;
};
@@ -39,5 +40,14 @@ export function getMcpActionLogConfigErrors(config: McpActionLogConfigInput) {
"MCP action log writes require a non-empty MCP_ACTION_LOG_SECRET",
});
}
+ if (
+ config.MCP_ACTION_LOG_WRITES_ENABLED === true &&
+ !config.PUBSUB_CREDENTIALS?.trim()
+ ) {
+ errors.push({
+ path: "PUBSUB_CREDENTIALS",
+ message: "MCP action log writes require the job-log publisher",
+ });
+ }
return errors;
}
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 9f0f332ea..6cfabebe6 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -1412,3 +1412,36 @@ async function logDeepResearchInternal(deepResearch: LoggedDeepResearch) {
await saveDeepResearchToGCS(deepResearch, logger);
}
}
+
+type LoggedMcpActionLog = {
+ id: string;
+ team_id: string;
+ request_id: string;
+ [field: string]: unknown;
+};
+
+/**
+ * One hosted MCP tool call from the internal action-log ingest. `id` is the
+ * MCP request id, so retried deliveries collapse into one row.
+ */
+export async function logMcpActionLog(action: LoggedMcpActionLog) {
+ return withLogSpan(
+ {
+ operation: "mcp_action",
+ table: "mcp_action_logs",
+ id: action.id,
+ requestId: action.request_id,
+ },
+ () => logMcpActionLogInternal(action),
+ );
+}
+
+async function logMcpActionLogInternal(action: LoggedMcpActionLog) {
+ const logger = _logger.child({
+ module: "log_job",
+ method: "logMcpActionLog",
+ requestId: action.request_id,
+ teamId: action.team_id,
+ });
+ await robustInsert("mcp_action_logs", action, logger);
+}
diff --git a/apps/api/src/services/mcp/action-logs-controller.test.ts b/apps/api/src/services/mcp/action-logs-controller.test.ts
index 1da76f3d2..e96ec0488 100644
--- a/apps/api/src/services/mcp/action-logs-controller.test.ts
+++ b/apps/api/src/services/mcp/action-logs-controller.test.ts
@@ -6,9 +6,8 @@ const mocks = vi.hoisted(() => ({
replicaDb: { name: "replica" },
normalize: vi.fn((body: any) => body),
validate: vi.fn(),
- record: vi.fn().mockResolvedValue({ disposition: "stored", id: "log-id" }),
+ record: vi.fn().mockResolvedValue({ disposition: "accepted", id: "log-id" }),
policy: vi.fn(),
- purge: vi.fn(),
authorize: vi.fn(),
list: vi.fn().mockResolvedValue({ data: [], nextCursor: null }),
}));
@@ -17,6 +16,8 @@ vi.mock("../../db/connection", () => ({
db: mocks.primaryDb,
dbRr: mocks.replicaDb,
}));
+vi.mock("../logging/log_job", () => ({ logMcpActionLog: vi.fn() }));
+vi.mock("../../lib/clickhouse-client", () => ({ clickhouseClient: null }));
vi.mock("./action-logs", async importOriginal => {
const actual = await importOriginal<typeof import("./action-logs")>();
return {
@@ -25,7 +26,6 @@ vi.mock("./action-logs", async importOriginal => {
validateMcpActionLogActor: mocks.validate,
recordMcpActionLog: mocks.record,
resolveMcpActionLogTeamPolicy: mocks.policy,
- purgeMcpActionLogsForTeam: mocks.purge,
authorizeMcpActionLogViewer: mocks.authorize,
listMcpActionLogs: mocks.list,
};
@@ -35,7 +35,10 @@ import {
ingestMcpActionLogController,
listMcpActionLogsController,
} from "../../controllers/v2/mcp-action-logs";
-import { McpActionLogAuthorizationError } from "./action-logs";
+import {
+ McpActionLogAuthorizationError,
+ McpActionLogUnavailableError,
+} from "./action-logs";
function response() {
const res: any = {};
@@ -49,7 +52,7 @@ describe("MCP action log controllers", () => {
vi.clearAllMocks();
mocks.policy.mockResolvedValue({ flags: null });
config.MCP_ACTION_LOG_STORAGE_ENABLED = true;
- mocks.record.mockResolvedValue({ disposition: "stored", id: "log-id" });
+ mocks.record.mockResolvedValue({ disposition: "accepted", id: "log-id" });
mocks.list.mockResolvedValue({ data: [], nextCursor: null });
});
@@ -62,7 +65,6 @@ describe("MCP action log controllers", () => {
);
expect(mocks.validate).toHaveBeenCalledTimes(1);
expect(mocks.policy).toHaveBeenCalledWith(mocks.primaryDb, "team");
- expect(mocks.purge).toHaveBeenCalledWith(mocks.primaryDb, "team");
expect(mocks.record).not.toHaveBeenCalled();
expect(res.status).toHaveBeenCalledWith(202);
expect(res.json).toHaveBeenCalledWith(
@@ -88,8 +90,33 @@ describe("MCP action log controllers", () => {
res,
);
expect(mocks.validate).toHaveBeenCalledTimes(1);
- expect(mocks.record).toHaveBeenCalledTimes(1);
+ expect(mocks.record).toHaveBeenCalledWith({ team_id: "team" });
expect(res.status).toHaveBeenCalledWith(202);
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({ disposition: "accepted", id: "log-id" }),
+ );
+ });
+
+ it("fails the delivery when the log cannot be published", async () => {
+ mocks.record.mockRejectedValueOnce(new Error("publish failed"));
+ const res = response();
+ await ingestMcpActionLogController(
+ { body: { team_id: "team" } } as any,
+ res,
+ );
+ expect(res.status).toHaveBeenCalledWith(500);
+ });
+
+ it("reports listing as unavailable when storage is not configured", async () => {
+ mocks.list.mockRejectedValueOnce(
+ new McpActionLogUnavailableError("MCP action log storage is unavailable"),
+ );
+ const res = response();
+ await listMcpActionLogsController(
+ { auth: { team_id: "team" }, acuc: { api_key_id: 42 }, query: {} } as any,
+ res,
+ );
+ expect(res.status).toHaveBeenCalledWith(503);
});
it("authorizes listing from the authenticated API key id", async () => {
@@ -104,11 +131,7 @@ describe("MCP action log controllers", () => {
);
expect(mocks.authorize).toHaveBeenCalledWith(mocks.primaryDb, "team", 42);
expect(mocks.policy).toHaveBeenCalledWith(mocks.primaryDb, "team");
- expect(mocks.list).toHaveBeenCalledWith(
- mocks.replicaDb,
- "team",
- expect.any(Object),
- );
+ expect(mocks.list).toHaveBeenCalledWith("team", expect.any(Object));
});
it("prefers the exact bigint API-key ID when authentication provides it", async () => {
@@ -176,7 +199,6 @@ describe("MCP action log controllers", () => {
res,
);
expect(mocks.authorize).toHaveBeenCalledTimes(1);
- expect(mocks.purge).toHaveBeenCalledWith(mocks.primaryDb, "team");
expect(mocks.list).not.toHaveBeenCalled();
expect(res.json).toHaveBeenCalledWith({
success: true,
diff --git a/apps/api/src/services/mcp/action-logs.test.ts b/apps/api/src/services/mcp/action-logs.test.ts
index 8c32f68b2..663c3e0f3 100644
--- a/apps/api/src/services/mcp/action-logs.test.ts
+++ b/apps/api/src/services/mcp/action-logs.test.ts
@@ -1,6 +1,18 @@
import { describe, expect, it, vi } from "vitest";
import { PgDialect } from "drizzle-orm/pg-core";
+const ch = vi.hoisted(() => ({
+ client: { query: vi.fn() } as { query: ReturnType<typeof vi.fn> } | null,
+ logMcpActionLog: vi.fn(),
+}));
+vi.mock("../../lib/clickhouse-client", () => ({
+ get clickhouseClient() {
+ return ch.client;
+ },
+}));
+vi.mock("../logging/log_job", () => ({
+ logMcpActionLog: ch.logMcpActionLog,
+}));
vi.mock("../../db/rpc", () => ({
authCreditUsageChunkFromTeam: vi.fn((db: any, teamId: string) =>
db.getTeamPolicy(teamId),
@@ -9,14 +21,10 @@ vi.mock("../../db/rpc", () => ({
import {
authorizeMcpActionLogViewer,
decodeMcpActionLogCursor,
- encodeMcpActionLogCursor,
listMcpActionLogs,
normalizeMcpActionLogInput,
- purgeMcpActionLogsForTeam,
recordMcpActionLog,
resolveMcpActionLogTeamPolicy,
- startMcpActionLogRetentionWorker,
- startMcpActionLogRetentionWorkerIfEnabled,
validateMcpActionLogActor,
} from "./action-logs";
@@ -131,6 +139,72 @@ describe("MCP action log contract", () => {
);
});
+ it("accepts the hosted MCP search resource", () => {
+ expect(
+ input({ resource: "https://mcp.firecrawl.dev/v2/mcp-search" }).resource,
+ ).toBe("https://mcp.firecrawl.dev/v2/mcp-search");
+ });
+
+ it("stores an optional, normalized objective with secret-like values redacted", () => {
+ expect(input()).toMatchObject({ objective: null, api_job_id: null });
+ expect(
+ input({
+ tool_name: "firecrawl_find_tools",
+ objective: "Pick a tool to extract product prices",
+ }),
+ ).toMatchObject({
+ objective: "Pick a tool to extract product prices",
+ api_job_id: null,
+ });
+ expect(input({ objective: null }).objective).toBeNull();
+ expect(input({ objective: " " }).objective).toBeNull();
+ expect(
+ input({ objective: " Compare pricing pages\nfor three vendors " })
+ .objective,
+ ).toBe("Compare pricing pages for three vendors");
+ expect(
+ input({ objective: "Check docs using Bearer abc.def and fc-123abc" })
+ .objective,
+ ).toBe("Check docs using [redacted] and [redacted]");
+ expect(input({ objective: "Find\u0085the\u009fdocs" }).objective).toBe(
+ "Find the docs",
+ );
+ const started = performance.now();
+ expect(input({ objective: "eyJ-".repeat(16_000) }).objective).toBe(
+ "[redacted]",
+ );
+ expect(performance.now() - started).toBeLessThan(500);
+ const cutJwt = `${`fc-${"x".repeat(96)} `.repeat(39)}eyJhbGciOi.eyJ${"a".repeat(200)}.c2ln`;
+ expect(input({ objective: cutJwt }).objective).toBe(
+ `${"[redacted] ".repeat(39)}[redacted]`,
+ );
+ expect(input({ objective: "x".repeat(1200) }).objective).toBe(
+ "x".repeat(1000),
+ );
+ expect(input({ objective: `${"x".repeat(999)}\u{1F600}` }).objective).toBe(
+ "x".repeat(999),
+ );
+ expect(() => input({ objective: 42 })).toThrow(
+ "objective must be a string",
+ );
+ });
+
+ it("stores an optional API job ID with the standard text limits", () => {
+ expect(input({ api_job_id: " 0190a5d4-7b10-7cc3 " }).api_job_id).toBe(
+ "0190a5d4-7b10-7cc3",
+ );
+ expect(input({ api_job_id: "" }).api_job_id).toBeNull();
+ expect(() => input({ api_job_id: "j".repeat(129) })).toThrow(
+ "api_job_id must be at most 128 characters",
+ );
+ expect(() => input({ api_job_id: "fc-secret" })).toThrow(
+ "api_job_id must not contain secret-like values",
+ );
+ expect(() => input({ api_job_id: 7 })).toThrow(
+ "api_job_id must be a string",
+ );
+ });
+
it("validates API-key and OAuth actors against the claimed team", async () => {
const apiKeyDb = {
select: vi.fn(() => ({
@@ -198,24 +272,30 @@ describe("MCP action log contract", () => {
expect(primaryDb.getTeamPolicy).toHaveBeenCalledWith(TEAM_ID);
});
- it("reports stored and duplicate deliveries correctly", async () => {
- const returning = vi.fn().mockResolvedValue([]);
- const onConflictDoNothing = vi.fn(() => ({ returning }));
- const values = vi.fn(() => ({ onConflictDoNothing }));
- const db = { insert: () => ({ values }) };
-
- await expect(recordMcpActionLog(db as any, input())).resolves.toEqual({
- disposition: "duplicate",
- id: null,
- });
- expect(onConflictDoNothing).toHaveBeenCalledTimes(1);
- expect((values.mock.calls as any)[0][0]).not.toHaveProperty("user_agent");
-
- returning.mockResolvedValueOnce([{ id: REQUEST_ID }]);
- await expect(recordMcpActionLog(db as any, input())).resolves.toEqual({
- disposition: "stored",
+ it("publishes accepted events keyed by the MCP request ID", async () => {
+ ch.logMcpActionLog.mockReset().mockResolvedValue(undefined);
+ await expect(
+ recordMcpActionLog(
+ input({
+ objective: "Find the refund policy",
+ api_job_id: REQUEST_ID,
+ user_agent: "Claude/1.0",
+ }),
+ ),
+ ).resolves.toEqual({ disposition: "accepted", id: REQUEST_ID });
+ const row = ch.logMcpActionLog.mock.calls[0][0];
+ expect(row).toMatchObject({
id: REQUEST_ID,
+ request_id: REQUEST_ID,
+ team_id: TEAM_ID,
+ api_key_id: "123",
+ objective: "Find the refund policy",
+ api_job_id: REQUEST_ID,
});
+ expect(row).not.toHaveProperty("user_agent");
+
+ ch.logMcpActionLog.mockRejectedValueOnce(new Error("publish failed"));
+ await expect(recordMcpActionLog(input())).rejects.toThrow("publish failed");
});
it("requires the API-key owner to be a team admin or owner for listing", async () => {
@@ -258,96 +338,79 @@ describe("MCP action log contract", () => {
).rejects.toThrow("An owner-bound API key is required");
});
- it("purges all retained rows for a team on the supplied primary DB", async () => {
- const where = vi.fn().mockResolvedValue(undefined);
- const db = { delete: vi.fn(() => ({ where })) };
- await purgeMcpActionLogsForTeam(db as any, TEAM_ID);
- expect(db.delete).toHaveBeenCalledTimes(1);
- expect(where).toHaveBeenCalledTimes(1);
- });
-
- it("runs bounded retention independently on an unrefed, stoppable timer", async () => {
- const db = { execute: vi.fn().mockResolvedValue(undefined) };
- let callback: (() => void) | undefined;
- const timer = { unref: vi.fn() };
- const clear = vi.fn();
- const worker = startMcpActionLogRetentionWorker({
- db,
- intervalMs: 1234,
- setIntervalFn: (next, interval) => {
- callback = next;
- expect(interval).toBe(1234);
- return timer;
- },
- clearIntervalFn: clear,
- });
- await worker.ready;
- expect(db.execute).toHaveBeenCalled();
- expect(timer.unref).toHaveBeenCalledTimes(1);
- callback?.();
- await vi.waitFor(() =>
- expect(db.execute.mock.calls.length).toBeGreaterThan(1),
- );
- worker.stop();
- expect(clear).toHaveBeenCalledWith(timer);
- });
-
- it("does not touch storage when retention is disabled", () => {
- const db = { execute: vi.fn() };
- const setIntervalFn = vi.fn();
- expect(
- startMcpActionLogRetentionWorkerIfEnabled({
- enabled: false,
- db,
- setIntervalFn,
- }),
- ).toBeNull();
- expect(db.execute).not.toHaveBeenCalled();
- expect(setIntervalFn).not.toHaveBeenCalled();
- });
-
- it("paginates non-expired rows with stable cursors", async () => {
+ it("paginates non-expired ClickHouse rows with stable cursors", async () => {
const rows = [
{
id: REQUEST_ID,
api_key_id: "9007199254740993",
- created_at: "2026-07-10T10:03:00.000Z",
+ created_at: "2026-07-10T10:03:00.123456789Z",
+ },
+ {
+ id: USER_ID,
+ api_key_id: null,
+ created_at: "2026-07-10T10:02:00.000000000Z",
},
- { id: USER_ID, api_key_id: null, created_at: "2026-07-10T10:02:00.000Z" },
];
- const execute = vi.fn().mockResolvedValue(undefined);
- const db = {
- execute,
- select: () => ({
- from() {
- return this;
- },
- where() {
- return this;
- },
- orderBy() {
- return this;
- },
- limit: () => Promise.resolve(rows),
- }),
+ ch.client = {
+ query: vi.fn().mockResolvedValue({ json: async () => rows }),
};
- const result = await listMcpActionLogs(db as any, TEAM_ID, { limit: 1 });
+ const result = await listMcpActionLogs(TEAM_ID, { limit: 1 });
expect(result.data).toEqual(rows.slice(0, 1));
expect(result.data[0].api_key_id).toBe("9007199254740993");
expect(decodeMcpActionLogCursor(result.nextCursor!)).toEqual({
id: rows[0].id,
created_at: rows[0].created_at,
});
- expect(execute).not.toHaveBeenCalled();
+ const first = ch.client.query.mock.calls[0][0];
+ expect(first.query).toContain("FROM mcp_action_logs AS log FINAL");
+ expect(first.query).toContain("log.team_id = {teamId: UUID}");
+ expect(first.query).toContain("INTERVAL 30 DAY");
+ expect(first.query).not.toContain("{beforeId: UUID}");
+ expect(first.query_params).toEqual({ teamId: TEAM_ID, limit: 2 });
- const cursor = encodeMcpActionLogCursor(rows[0]);
- expect(decodeMcpActionLogCursor(cursor)).toEqual({
- id: rows[0].id,
- created_at: rows[0].created_at,
+ await listMcpActionLogs(TEAM_ID, { limit: 500, cursor: result.nextCursor });
+ const second = ch.client.query.mock.calls[1][0];
+ expect(second.query).toContain(
+ "(log.created_at, log.id) < (parseDateTime64BestEffort({before: String}, 9), {beforeId: UUID})",
+ );
+ expect(second.query_params).toEqual({
+ teamId: TEAM_ID,
+ limit: 101,
+ before: rows[0].created_at,
+ beforeId: rows[0].id,
});
+
expect(() => decodeMcpActionLogCursor("bad")).toThrow("cursor is invalid");
+ await expect(listMcpActionLogs(TEAM_ID, { cursor: "bad" })).rejects.toThrow(
+ "cursor is invalid",
+ );
- const complete = await listMcpActionLogs(db as any, TEAM_ID, { limit: 2 });
+ const complete = await listMcpActionLogs(TEAM_ID, { limit: 2 });
expect(complete.data[1].api_key_id).toBeNull();
+ expect(complete.nextCursor).toBeNull();
+ });
+
+ it("never selects the objective or API job ID for the team-readable list", async () => {
+ ch.client = { query: vi.fn().mockResolvedValue({ json: async () => [] }) };
+ await listMcpActionLogs(TEAM_ID);
+ const sql = ch.client.query.mock.calls[0][0].query as string;
+ const columns = sql.slice(0, sql.indexOf(" FROM "));
+ expect(columns).toContain("tool_name");
+ expect(columns).not.toContain("objective");
+ expect(columns).not.toContain("api_job_id");
+ });
+
+ it("reports storage as unavailable when the read fails", async () => {
+ ch.client = { query: vi.fn().mockRejectedValue(new Error("timeout")) };
+ await expect(listMcpActionLogs(TEAM_ID)).rejects.toThrow(
+ "MCP action log storage is unavailable",
+ );
+ });
+
+ it("reports storage as unavailable without ClickHouse", async () => {
+ ch.client = null;
+ await expect(listMcpActionLogs(TEAM_ID)).rejects.toThrow(
+ "MCP action log storage is unavailable",
+ );
});
});
diff --git a/apps/api/src/services/mcp/action-logs.ts b/apps/api/src/services/mcp/action-logs.ts
index a773e5ff0..b1dfcbf7d 100644
--- a/apps/api/src/services/mcp/action-logs.ts
+++ b/apps/api/src/services/mcp/action-logs.ts
@@ -1,21 +1,25 @@
-import { and, desc, eq, gt, lt, or, sql } from "drizzle-orm";
+import { and, eq, sql } from "drizzle-orm";
import * as schema from "../../db/schema";
import { authCreditUsageChunkFromTeam } from "../../db/rpc";
+import { clickhouseClient } from "../../lib/clickhouse-client";
import { logger } from "../../lib/logger";
+import { logMcpActionLog } from "../logging/log_job";
const STATUSES = ["success", "error"] as const;
const AUTH_TYPES = ["oauth", "api-key"] as const;
const RESOURCES = new Set([
"https://mcp.firecrawl.dev/v2/mcp",
"https://mcp.firecrawl.dev/v2/mcp-oauth",
+ "https://mcp.firecrawl.dev/v2/mcp-search",
]);
const UUID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const POSTGRES_BIGINT_MAX = 9_223_372_036_854_775_807n;
const SECRET_PATTERN =
/(?:\bBearer\s+\S+|\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b|\b(?:sk-|fc-|fco_|fcr_|fcmcp_)[A-Za-z0-9_-]+)/i;
+const SECRET_PATTERN_ALL = new RegExp(SECRET_PATTERN.source, "gi");
+const OBJECTIVE_MAX_LENGTH = 1000;
const RETENTION_DAYS = 30;
-const CLEANUP_BATCH_SIZE = 500;
const ALLOWED_FIELDS = new Set([
"team_id",
"user_id",
@@ -30,10 +34,13 @@ const ALLOWED_FIELDS = new Set([
"client_version",
"error_class",
"resource",
+ "objective",
+ "api_job_id",
]);
export class McpActionLogValidationError extends Error {}
export class McpActionLogAuthorizationError extends Error {}
+export class McpActionLogUnavailableError extends Error {}
function invalid(message: string): never {
throw new McpActionLogValidationError(message);
@@ -77,7 +84,22 @@ function text(value: unknown, field: string, max = 128, required = false) {
return normalized || null;
}
-type McpActionLogInput = {
+function objective(value: unknown) {
+ if (value == null) return null;
+ if (typeof value !== "string") invalid("objective must be a string");
+ const normalized = value
+ .slice(0, OBJECTIVE_MAX_LENGTH * 4)
+ .replace(/[\u0000-\u001F\u007F-\u009F]+/g, " ")
+ .replace(SECRET_PATTERN_ALL, "[redacted]")
+ .replace(/\beyJ[A-Za-z0-9_.-]*$/, "[redacted]")
+ .trim()
+ .slice(0, OBJECTIVE_MAX_LENGTH)
+ .replace(/[\uD800-\uDBFF]$/, "")
+ .trim();
+ return normalized || null;
+}
+
+export type McpActionLogInput = {
team_id: string;
user_id: string | null;
api_key_id: string | null;
@@ -90,6 +112,24 @@ type McpActionLogInput = {
client_version: string | null;
error_class: string | null;
resource: string;
+ objective: string | null;
+ api_job_id: string | null;
+};
+
+type McpActionLogRow = {
+ id: string;
+ user_id: string | null;
+ api_key_id: string | null;
+ oauth_client_id: string | null;
+ auth_type: string;
+ tool_name: string;
+ status: string;
+ request_id: string;
+ client_name: string | null;
+ client_version: string | null;
+ error_class: string | null;
+ resource: string;
+ created_at: string;
};
export function normalizeMcpActionLogInput(
@@ -138,6 +178,8 @@ export function normalizeMcpActionLogInput(
client_version: text(payload.client_version, "client_version"),
error_class: text(payload.error_class, "error_class"),
resource,
+ objective: objective(payload.objective),
+ api_job_id: text(payload.api_job_id, "api_job_id"),
};
}
@@ -195,101 +237,9 @@ export async function resolveMcpActionLogTeamPolicy(db: any, teamId: string) {
return { flags: rows[0].flags ?? null };
}
-async function cleanupExpiredMcpActionLogs(db: any) {
- await db.execute(sql`
- with expired as (
- select id from mcp_action_logs
- where expires_at <= now()
- order by expires_at asc
- limit ${CLEANUP_BATCH_SIZE}
- )
- delete from mcp_action_logs
- where id in (select id from expired)
- `);
-}
-
-export async function purgeMcpActionLogsForTeam(db: any, teamId: string) {
- await db
- .delete(schema.mcp_action_logs)
- .where(eq(schema.mcp_action_logs.team_id, teamId));
-}
-
-type RetentionTimer = {
- unref?: () => void;
-};
-
-export function startMcpActionLogRetentionWorker(options: {
- db: any;
- intervalMs?: number;
- setIntervalFn?: (callback: () => void, intervalMs: number) => RetentionTimer;
- clearIntervalFn?: (timer: RetentionTimer) => void;
-}) {
- const intervalMs = options.intervalMs ?? 5 * 60 * 1000;
- const setIntervalFn =
- options.setIntervalFn ?? ((callback, ms) => setInterval(callback, ms));
- const clearIntervalFn =
- options.clearIntervalFn ??
- (timer => clearInterval(timer as NodeJS.Timeout));
- let running = false;
-
- const run = async () => {
- if (running) return;
- running = true;
- try {
- await cleanupExpiredMcpActionLogs(options.db);
- } catch (error) {
- logger.warn("Failed to clean up expired MCP action logs", { error });
- } finally {
- running = false;
- }
- };
-
- const timer = setIntervalFn(() => void run(), intervalMs);
- timer.unref?.();
- const ready = run();
-
- return {
- ready,
- run,
- stop: () => clearIntervalFn(timer),
- };
-}
-
-export function startMcpActionLogRetentionWorkerIfEnabled(options: {
- enabled: boolean;
- db: any;
- intervalMs?: number;
- setIntervalFn?: (callback: () => void, intervalMs: number) => RetentionTimer;
- clearIntervalFn?: (timer: RetentionTimer) => void;
-}) {
- if (!options.enabled) return null;
- return startMcpActionLogRetentionWorker(options);
-}
-
-export async function recordMcpActionLog(db: any, input: McpActionLogInput) {
- const expiresAt = new Date(
- Date.now() + RETENTION_DAYS * 24 * 60 * 60 * 1000,
- ).toISOString();
- const rows = await db
- .insert(schema.mcp_action_logs)
- .values({
- ...input,
- api_key_id:
- input.api_key_id === null
- ? null
- : sql`cast(${input.api_key_id} as bigint)`,
- expires_at: expiresAt,
- })
- .onConflictDoNothing({
- target: [
- schema.mcp_action_logs.team_id,
- schema.mcp_action_logs.request_id,
- ],
- })
- .returning({ id: schema.mcp_action_logs.id });
- return rows[0]
- ? { disposition: "stored" as const, id: rows[0].id }
- : { disposition: "duplicate" as const, id: null };
+export async function recordMcpActionLog(input: McpActionLogInput) {
+ await logMcpActionLog({ id: input.request_id, ...input });
+ return { disposition: "accepted" as const, id: input.request_id };
}
export async function authorizeMcpActionLogViewer(
@@ -345,16 +295,9 @@ export async function authorizeMcpActionLogViewer(
return { userId: ownerId, role };
}
-export function encodeMcpActionLogCursor(row: {
- created_at: Date | string;
- id: string;
-}) {
- const createdAt =
- row.created_at instanceof Date
- ? row.created_at.toISOString()
- : row.created_at;
+function encodeMcpActionLogCursor(row: { created_at: string; id: string }) {
return Buffer.from(
- JSON.stringify({ created_at: createdAt, id: row.id }),
+ JSON.stringify({ created_at: row.created_at, id: row.id }),
).toString("base64url");
}
@@ -377,56 +320,39 @@ export function decodeMcpActionLogCursor(cursor: string) {
}
export async function listMcpActionLogs(
- db: any,
teamId: string,
options: { limit?: number; cursor?: string | null } = {},
) {
+ if (!clickhouseClient) {
+ throw new McpActionLogUnavailableError(
+ "MCP action log storage is unavailable",
+ );
+ }
const limit = Math.min(Math.max(options.limit ?? 50, 1), 100);
const cursor = options.cursor
? decodeMcpActionLogCursor(options.cursor)
: null;
- const cursorClause = cursor
- ? or(
- lt(schema.mcp_action_logs.created_at, cursor.created_at),
- and(
- eq(schema.mcp_action_logs.created_at, cursor.created_at),
- lt(schema.mcp_action_logs.id, cursor.id),
- ),
- )
- : undefined;
- const rows = await db
- .select({
- id: schema.mcp_action_logs.id,
- user_id: schema.mcp_action_logs.user_id,
- // API-key identifiers are bigint in Postgres. Read them as text so the
- // JSON boundary never rounds values beyond JavaScript's safe integer range.
- api_key_id: sql<
- string | null
- >`${schema.mcp_action_logs.api_key_id}::text`,
- oauth_client_id: schema.mcp_action_logs.oauth_client_id,
- auth_type: schema.mcp_action_logs.auth_type,
- tool_name: schema.mcp_action_logs.tool_name,
- status: schema.mcp_action_logs.status,
- request_id: schema.mcp_action_logs.request_id,
- client_name: schema.mcp_action_logs.client_name,
- client_version: schema.mcp_action_logs.client_version,
- error_class: schema.mcp_action_logs.error_class,
- resource: schema.mcp_action_logs.resource,
- created_at: schema.mcp_action_logs.created_at,
- })
- .from(schema.mcp_action_logs)
- .where(
- and(
- eq(schema.mcp_action_logs.team_id, teamId),
- gt(schema.mcp_action_logs.expires_at, new Date().toISOString()),
- cursorClause,
- ),
- )
- .orderBy(
- desc(schema.mcp_action_logs.created_at),
- desc(schema.mcp_action_logs.id),
- )
- .limit(limit + 1);
+ let rows: McpActionLogRow[];
+ try {
+ const result = await clickhouseClient.query({
+ // The explicit column list is the team-readable contract: objective and
+ // api_job_id stay out of it. Filters name `log.` columns because the
+ // formatted created_at alias would otherwise shadow the stored value.
+ query: `SELECT id, user_id, toString(log.api_key_id) AS api_key_id, oauth_client_id, auth_type, tool_name, status, request_id, client_name, client_version, error_class, resource, formatDateTime(log.created_at, '%Y-%m-%dT%H:%i:%S.%fZ', 'UTC') AS created_at FROM mcp_action_logs AS log FINAL WHERE log.team_id = {teamId: UUID} AND log.created_at > now64(9) - INTERVAL ${RETENTION_DAYS} DAY${cursor ? " AND (log.created_at, log.id) < (parseDateTime64BestEffort({before: String}, 9), {beforeId: UUID})" : ""} ORDER BY log.created_at DESC, log.id DESC LIMIT {limit: UInt32}`,
+ query_params: {
+ teamId,
+ limit: limit + 1,
+ ...(cursor ? { before: cursor.created_at, beforeId: cursor.id } : {}),
+ },
+ format: "JSONEachRow",
+ });
+ rows = await result.json<McpActionLogRow>();
+ } catch (error) {
+ logger.warn("Failed to read MCP action logs", { error });
+ throw new McpActionLogUnavailableError(
+ "MCP action log storage is unavailable",
+ );
+ }
return {
data: rows.slice(0, limit),
nextCursor: