Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): MCP action logs go to the job log, with objective and api_job_id (#5092)
Rakshith Ramprakash979783635318
diff --git a/apps/api/src/controllers/v2/mcp-action-logs.ts b/apps/api/src/controllers/v2/mcp-action-logs.tsindex 703219f2f..c8cc030f6 100644--- a/apps/api/src/controllers/v2/mcp-action-logs.ts+++ b/apps/api/src/controllers/v2/mcp-action-logs.ts@@ -1,14 +1,14 @@import { Request, Response } from "express";import { config } from "../../config";-import { db, dbRr } from "../../db/connection";+import { db } from "../../db/connection";import { ErrorResponse, RequestWithAuth, TeamFlags } from "./types";import {authorizeMcpActionLogViewer,listMcpActionLogs,McpActionLogAuthorizationError,+ McpActionLogUnavailableError,McpActionLogValidationError,normalizeMcpActionLogInput,- purgeMcpActionLogsForTeam,recordMcpActionLog,resolveMcpActionLogTeamPolicy,validateMcpActionLogActor,@@ -44,14 +44,13 @@ export async function ingestMcpActionLogController(});}if (isMcpActionLogZdrTeam(team.flags)) {- await purgeMcpActionLogsForTeam(db, input.team_id);return res.status(202).json({success: true,disposition: "zero-data-retention",id: null,});}- const result = await recordMcpActionLog(db, input);+ const result = await recordMcpActionLog(input);return res.status(202).json({ success: true, ...result });} catch (error) {if (error instanceof McpActionLogValidationError) {@@ -91,7 +90,6 @@ export async function listMcpActionLogsController(});}if (isMcpActionLogZdrTeam(team.flags)) {- await purgeMcpActionLogsForTeam(db, req.auth.team_id);return res.status(200).json({success: true,data: [],@@ -99,7 +97,7 @@ export async function listMcpActionLogsController(});}const parsedLimit = Number.parseInt(String(req.query.limit ?? "50"), 10);- const result = await listMcpActionLogs(dbRr, req.auth.team_id, {+ const result = await listMcpActionLogs(req.auth.team_id, {limit: Number.isFinite(parsedLimit) ? parsedLimit : 50,cursor: typeof req.query.cursor === "string" ? req.query.cursor : null,});@@ -111,6 +109,9 @@ export async function listMcpActionLogsController(if (error instanceof McpActionLogValidationError) {return res.status(400).json({ success: false, error: error.message });}+ if (error instanceof McpActionLogUnavailableError) {+ return res.status(503).json({ success: false, error: error.message });+ }throw error;}}diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex 74bc02da7..0eacb9f01 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -442,38 +442,6 @@ export const products = pgTable("products", {type: text("type"),});-export const mcp_action_logs = pgTable(- "mcp_action_logs",- {- id: uuid("id").defaultRandom().primaryKey(),- team_id: uuid("team_id").notNull(),- user_id: uuid("user_id"),- api_key_id: bigintNum("api_key_id"),- oauth_client_id: text("oauth_client_id"),- auth_type: text("auth_type").notNull(),- tool_name: text("tool_name").notNull(),- status: text("status").notNull(),- request_id: uuid("request_id").notNull(),- client_name: text("client_name"),- client_version: text("client_version"),- error_class: text("error_class"),- resource: text("resource").notNull(),- created_at: ts("created_at").notNull().defaultNow(),- expires_at: ts("expires_at").notNull(),- },- table => [- // Mirrors the DB migration and is required by recordMcpActionLog's- // ON CONFLICT (team_id, request_id) idempotency contract.- unique("mcp_action_logs_team_request_unique").on(- table.team_id,- table.request_id,- ),- // Mirrors the DB migration and keeps the retention worker's bounded- // expiry scan indexed as the table grows.- index("mcp_action_logs_expires_idx").on(table.expires_at),- ],-);-export const search_feedback = pgTable("search_feedback", {id: uuid("id").notNull().defaultRandom(),search_id: uuid("search_id"),diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 31969064e..3d5a5e20a 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -28,8 +28,6 @@ import { v2Router } from "./routes/v2";import { exchangeRouter } from "./routes/exchange";import { labsRouter } from "./routes/labs";import { registerMcpActionLogIngestRoute } from "./routes/mcp-action-logs";-import { startMcpActionLogRetentionWorkerIfEnabled } from "./services/mcp/action-logs";-import { db } from "./db/connection";import { nuqShutdown } from "./services/worker/nuq";import { getErrorContactMessage } from "./lib/deployment";import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";@@ -154,10 +152,6 @@ async function startServer(port = DEFAULT_PORT) {throw error;}- const mcpActionLogRetention = startMcpActionLogRetentionWorkerIfEnabled({- enabled: config.MCP_ACTION_LOG_STORAGE_ENABLED,- db,- });const server = app.listen(Number(port), HOST, (error?: Error) => {if (error) {logger.error("Failed to start HTTP server", { error, port, host: HOST });@@ -169,7 +163,6 @@ async function startServer(port = DEFAULT_PORT) {const exitHandler = async () => {logger.info("SIGTERM signal received: closing HTTP server");- mcpActionLogRetention?.stop();if (config.IS_KUBERNETES) {// Account for GCE load balancer drain timeoutlogger.info("Waiting 60s for GCE load balancer drain timeout");diff --git a/apps/api/src/lib/mcp-action-log-config.test.ts b/apps/api/src/lib/mcp-action-log-config.test.tsindex ae47aa40c..f0151a6ad 100644--- a/apps/api/src/lib/mcp-action-log-config.test.ts+++ b/apps/api/src/lib/mcp-action-log-config.test.ts@@ -22,6 +22,7 @@ describe("MCP action log configuration", () => {expect(errors.map(error => error.path)).toEqual(["MCP_ACTION_LOG_WRITES_ENABLED","MCP_ACTION_LOG_SECRET",+ "PUBSUB_CREDENTIALS",]);});@@ -31,6 +32,7 @@ describe("MCP action log configuration", () => {MCP_ACTION_LOG_STORAGE_ENABLED: true,MCP_ACTION_LOG_WRITES_ENABLED: true,MCP_ACTION_LOG_SECRET: "secret",+ PUBSUB_CREDENTIALS: "credentials",USE_DB_AUTHENTICATION: true,}),).toEqual([]);diff --git a/apps/api/src/lib/mcp-action-log-config.ts b/apps/api/src/lib/mcp-action-log-config.tsindex 8b8e879ca..a717bc9b1 100644--- a/apps/api/src/lib/mcp-action-log-config.ts+++ b/apps/api/src/lib/mcp-action-log-config.ts@@ -2,6 +2,7 @@ type McpActionLogConfigInput = {MCP_ACTION_LOG_STORAGE_ENABLED?: boolean;MCP_ACTION_LOG_WRITES_ENABLED?: boolean;MCP_ACTION_LOG_SECRET?: string;+ PUBSUB_CREDENTIALS?: string;USE_DB_AUTHENTICATION?: boolean;};@@ -39,5 +40,14 @@ export function getMcpActionLogConfigErrors(config: McpActionLogConfigInput) {"MCP action log writes require a non-empty MCP_ACTION_LOG_SECRET",});}+ if (+ config.MCP_ACTION_LOG_WRITES_ENABLED === true &&+ !config.PUBSUB_CREDENTIALS?.trim()+ ) {+ errors.push({+ path: "PUBSUB_CREDENTIALS",+ message: "MCP action log writes require the job-log publisher",+ });+ }return errors;}diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 9f0f332ea..6cfabebe6 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -1412,3 +1412,36 @@ async function logDeepResearchInternal(deepResearch: LoggedDeepResearch) {await saveDeepResearchToGCS(deepResearch, logger);}}++type LoggedMcpActionLog = {+ id: string;+ team_id: string;+ request_id: string;+ [field: string]: unknown;+};++/**+ * One hosted MCP tool call from the internal action-log ingest. `id` is the+ * MCP request id, so retried deliveries collapse into one row.+ */+export async function logMcpActionLog(action: LoggedMcpActionLog) {+ return withLogSpan(+ {+ operation: "mcp_action",+ table: "mcp_action_logs",+ id: action.id,+ requestId: action.request_id,+ },+ () => logMcpActionLogInternal(action),+ );+}++async function logMcpActionLogInternal(action: LoggedMcpActionLog) {+ const logger = _logger.child({+ module: "log_job",+ method: "logMcpActionLog",+ requestId: action.request_id,+ teamId: action.team_id,+ });+ await robustInsert("mcp_action_logs", action, logger);+}diff --git a/apps/api/src/services/mcp/action-logs-controller.test.ts b/apps/api/src/services/mcp/action-logs-controller.test.tsindex 1da76f3d2..e96ec0488 100644--- a/apps/api/src/services/mcp/action-logs-controller.test.ts+++ b/apps/api/src/services/mcp/action-logs-controller.test.ts@@ -6,9 +6,8 @@ const mocks = vi.hoisted(() => ({replicaDb: { name: "replica" },normalize: vi.fn((body: any) => body),validate: vi.fn(),- record: vi.fn().mockResolvedValue({ disposition: "stored", id: "log-id" }),+ record: vi.fn().mockResolvedValue({ disposition: "accepted", id: "log-id" }),policy: vi.fn(),- purge: vi.fn(),authorize: vi.fn(),list: vi.fn().mockResolvedValue({ data: [], nextCursor: null }),}));@@ -17,6 +16,8 @@ vi.mock("../../db/connection", () => ({db: mocks.primaryDb,dbRr: mocks.replicaDb,}));+vi.mock("../logging/log_job", () => ({ logMcpActionLog: vi.fn() }));+vi.mock("../../lib/clickhouse-client", () => ({ clickhouseClient: null }));vi.mock("./action-logs", async importOriginal => {const actual = await importOriginal<typeof import("./action-logs")>();return {@@ -25,7 +26,6 @@ vi.mock("./action-logs", async importOriginal => {validateMcpActionLogActor: mocks.validate,recordMcpActionLog: mocks.record,resolveMcpActionLogTeamPolicy: mocks.policy,- purgeMcpActionLogsForTeam: mocks.purge,authorizeMcpActionLogViewer: mocks.authorize,listMcpActionLogs: mocks.list,};@@ -35,7 +35,10 @@ import {ingestMcpActionLogController,listMcpActionLogsController,} from "../../controllers/v2/mcp-action-logs";-import { McpActionLogAuthorizationError } from "./action-logs";+import {+ McpActionLogAuthorizationError,+ McpActionLogUnavailableError,+} from "./action-logs";function response() {const res: any = {};@@ -49,7 +52,7 @@ describe("MCP action log controllers", () => {vi.clearAllMocks();mocks.policy.mockResolvedValue({ flags: null });config.MCP_ACTION_LOG_STORAGE_ENABLED = true;- mocks.record.mockResolvedValue({ disposition: "stored", id: "log-id" });+ mocks.record.mockResolvedValue({ disposition: "accepted", id: "log-id" });mocks.list.mockResolvedValue({ data: [], nextCursor: null });});@@ -62,7 +65,6 @@ describe("MCP action log controllers", () => {);expect(mocks.validate).toHaveBeenCalledTimes(1);expect(mocks.policy).toHaveBeenCalledWith(mocks.primaryDb, "team");- expect(mocks.purge).toHaveBeenCalledWith(mocks.primaryDb, "team");expect(mocks.record).not.toHaveBeenCalled();expect(res.status).toHaveBeenCalledWith(202);expect(res.json).toHaveBeenCalledWith(@@ -88,8 +90,33 @@ describe("MCP action log controllers", () => {res,);expect(mocks.validate).toHaveBeenCalledTimes(1);- expect(mocks.record).toHaveBeenCalledTimes(1);+ expect(mocks.record).toHaveBeenCalledWith({ team_id: "team" });expect(res.status).toHaveBeenCalledWith(202);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({ disposition: "accepted", id: "log-id" }),+ );+ });++ it("fails the delivery when the log cannot be published", async () => {+ mocks.record.mockRejectedValueOnce(new Error("publish failed"));+ const res = response();+ await ingestMcpActionLogController(+ { body: { team_id: "team" } } as any,+ res,+ );+ expect(res.status).toHaveBeenCalledWith(500);+ });++ it("reports listing as unavailable when storage is not configured", async () => {+ mocks.list.mockRejectedValueOnce(+ new McpActionLogUnavailableError("MCP action log storage is unavailable"),+ );+ const res = response();+ await listMcpActionLogsController(+ { auth: { team_id: "team" }, acuc: { api_key_id: 42 }, query: {} } as any,+ res,+ );+ expect(res.status).toHaveBeenCalledWith(503);});it("authorizes listing from the authenticated API key id", async () => {@@ -104,11 +131,7 @@ describe("MCP action log controllers", () => {);expect(mocks.authorize).toHaveBeenCalledWith(mocks.primaryDb, "team", 42);expect(mocks.policy).toHaveBeenCalledWith(mocks.primaryDb, "team");- expect(mocks.list).toHaveBeenCalledWith(- mocks.replicaDb,- "team",- expect.any(Object),- );+ expect(mocks.list).toHaveBeenCalledWith("team", expect.any(Object));});it("prefers the exact bigint API-key ID when authentication provides it", async () => {@@ -176,7 +199,6 @@ describe("MCP action log controllers", () => {res,);expect(mocks.authorize).toHaveBeenCalledTimes(1);- expect(mocks.purge).toHaveBeenCalledWith(mocks.primaryDb, "team");expect(mocks.list).not.toHaveBeenCalled();expect(res.json).toHaveBeenCalledWith({success: true,diff --git a/apps/api/src/services/mcp/action-logs.test.ts b/apps/api/src/services/mcp/action-logs.test.tsindex 8c32f68b2..663c3e0f3 100644--- a/apps/api/src/services/mcp/action-logs.test.ts+++ b/apps/api/src/services/mcp/action-logs.test.ts@@ -1,6 +1,18 @@import { describe, expect, it, vi } from "vitest";import { PgDialect } from "drizzle-orm/pg-core";+const ch = vi.hoisted(() => ({+ client: { query: vi.fn() } as { query: ReturnType<typeof vi.fn> } | null,+ logMcpActionLog: vi.fn(),+}));+vi.mock("../../lib/clickhouse-client", () => ({+ get clickhouseClient() {+ return ch.client;+ },+}));+vi.mock("../logging/log_job", () => ({+ logMcpActionLog: ch.logMcpActionLog,+}));vi.mock("../../db/rpc", () => ({authCreditUsageChunkFromTeam: vi.fn((db: any, teamId: string) =>db.getTeamPolicy(teamId),@@ -9,14 +21,10 @@ vi.mock("../../db/rpc", () => ({import {authorizeMcpActionLogViewer,decodeMcpActionLogCursor,- encodeMcpActionLogCursor,listMcpActionLogs,normalizeMcpActionLogInput,- purgeMcpActionLogsForTeam,recordMcpActionLog,resolveMcpActionLogTeamPolicy,- startMcpActionLogRetentionWorker,- startMcpActionLogRetentionWorkerIfEnabled,validateMcpActionLogActor,} from "./action-logs";@@ -131,6 +139,72 @@ describe("MCP action log contract", () => {);});+ it("accepts the hosted MCP search resource", () => {+ expect(+ input({ resource: "https://mcp.firecrawl.dev/v2/mcp-search" }).resource,+ ).toBe("https://mcp.firecrawl.dev/v2/mcp-search");+ });++ it("stores an optional, normalized objective with secret-like values redacted", () => {+ expect(input()).toMatchObject({ objective: null, api_job_id: null });+ expect(+ input({+ tool_name: "firecrawl_find_tools",+ objective: "Pick a tool to extract product prices",+ }),+ ).toMatchObject({+ objective: "Pick a tool to extract product prices",+ api_job_id: null,+ });+ expect(input({ objective: null }).objective).toBeNull();+ expect(input({ objective: " " }).objective).toBeNull();+ expect(+ input({ objective: " Compare pricing pages\nfor three vendors " })+ .objective,+ ).toBe("Compare pricing pages for three vendors");+ expect(+ input({ objective: "Check docs using Bearer abc.def and fc-123abc" })+ .objective,+ ).toBe("Check docs using [redacted] and [redacted]");+ expect(input({ objective: "Find\u0085the\u009fdocs" }).objective).toBe(+ "Find the docs",+ );+ const started = performance.now();+ expect(input({ objective: "eyJ-".repeat(16_000) }).objective).toBe(+ "[redacted]",+ );+ expect(performance.now() - started).toBeLessThan(500);+ const cutJwt = `${`fc-${"x".repeat(96)} `.repeat(39)}eyJhbGciOi.eyJ${"a".repeat(200)}.c2ln`;+ expect(input({ objective: cutJwt }).objective).toBe(+ `${"[redacted] ".repeat(39)}[redacted]`,+ );+ expect(input({ objective: "x".repeat(1200) }).objective).toBe(+ "x".repeat(1000),+ );+ expect(input({ objective: `${"x".repeat(999)}\u{1F600}` }).objective).toBe(+ "x".repeat(999),+ );+ expect(() => input({ objective: 42 })).toThrow(+ "objective must be a string",+ );+ });++ it("stores an optional API job ID with the standard text limits", () => {+ expect(input({ api_job_id: " 0190a5d4-7b10-7cc3 " }).api_job_id).toBe(+ "0190a5d4-7b10-7cc3",+ );+ expect(input({ api_job_id: "" }).api_job_id).toBeNull();+ expect(() => input({ api_job_id: "j".repeat(129) })).toThrow(+ "api_job_id must be at most 128 characters",+ );+ expect(() => input({ api_job_id: "fc-secret" })).toThrow(+ "api_job_id must not contain secret-like values",+ );+ expect(() => input({ api_job_id: 7 })).toThrow(+ "api_job_id must be a string",+ );+ });+it("validates API-key and OAuth actors against the claimed team", async () => {const apiKeyDb = {select: vi.fn(() => ({@@ -198,24 +272,30 @@ describe("MCP action log contract", () => {expect(primaryDb.getTeamPolicy).toHaveBeenCalledWith(TEAM_ID);});- it("reports stored and duplicate deliveries correctly", async () => {- const returning = vi.fn().mockResolvedValue([]);- const onConflictDoNothing = vi.fn(() => ({ returning }));- const values = vi.fn(() => ({ onConflictDoNothing }));- const db = { insert: () => ({ values }) };-- await expect(recordMcpActionLog(db as any, input())).resolves.toEqual({- disposition: "duplicate",- id: null,- });- expect(onConflictDoNothing).toHaveBeenCalledTimes(1);- expect((values.mock.calls as any)[0][0]).not.toHaveProperty("user_agent");-- returning.mockResolvedValueOnce([{ id: REQUEST_ID }]);- await expect(recordMcpActionLog(db as any, input())).resolves.toEqual({- disposition: "stored",+ it("publishes accepted events keyed by the MCP request ID", async () => {+ ch.logMcpActionLog.mockReset().mockResolvedValue(undefined);+ await expect(+ recordMcpActionLog(+ input({+ objective: "Find the refund policy",+ api_job_id: REQUEST_ID,+ user_agent: "Claude/1.0",+ }),+ ),+ ).resolves.toEqual({ disposition: "accepted", id: REQUEST_ID });+ const row = ch.logMcpActionLog.mock.calls[0][0];+ expect(row).toMatchObject({id: REQUEST_ID,+ request_id: REQUEST_ID,+ team_id: TEAM_ID,+ api_key_id: "123",+ objective: "Find the refund policy",+ api_job_id: REQUEST_ID,});+ expect(row).not.toHaveProperty("user_agent");++ ch.logMcpActionLog.mockRejectedValueOnce(new Error("publish failed"));+ await expect(recordMcpActionLog(input())).rejects.toThrow("publish failed");});it("requires the API-key owner to be a team admin or owner for listing", async () => {@@ -258,96 +338,79 @@ describe("MCP action log contract", () => {).rejects.toThrow("An owner-bound API key is required");});- it("purges all retained rows for a team on the supplied primary DB", async () => {- const where = vi.fn().mockResolvedValue(undefined);- const db = { delete: vi.fn(() => ({ where })) };- await purgeMcpActionLogsForTeam(db as any, TEAM_ID);- expect(db.delete).toHaveBeenCalledTimes(1);- expect(where).toHaveBeenCalledTimes(1);- });-- it("runs bounded retention independently on an unrefed, stoppable timer", async () => {- const db = { execute: vi.fn().mockResolvedValue(undefined) };- let callback: (() => void) | undefined;- const timer = { unref: vi.fn() };- const clear = vi.fn();- const worker = startMcpActionLogRetentionWorker({- db,- intervalMs: 1234,- setIntervalFn: (next, interval) => {- callback = next;- expect(interval).toBe(1234);- return timer;- },- clearIntervalFn: clear,- });- await worker.ready;- expect(db.execute).toHaveBeenCalled();- expect(timer.unref).toHaveBeenCalledTimes(1);- callback?.();- await vi.waitFor(() =>- expect(db.execute.mock.calls.length).toBeGreaterThan(1),- );- worker.stop();- expect(clear).toHaveBeenCalledWith(timer);- });-- it("does not touch storage when retention is disabled", () => {- const db = { execute: vi.fn() };- const setIntervalFn = vi.fn();- expect(- startMcpActionLogRetentionWorkerIfEnabled({- enabled: false,- db,- setIntervalFn,- }),- ).toBeNull();- expect(db.execute).not.toHaveBeenCalled();- expect(setIntervalFn).not.toHaveBeenCalled();- });-- it("paginates non-expired rows with stable cursors", async () => {+ it("paginates non-expired ClickHouse rows with stable cursors", async () => {const rows = [{id: REQUEST_ID,api_key_id: "9007199254740993",- created_at: "2026-07-10T10:03:00.000Z",+ created_at: "2026-07-10T10:03:00.123456789Z",+ },+ {+ id: USER_ID,+ api_key_id: null,+ created_at: "2026-07-10T10:02:00.000000000Z",},- { id: USER_ID, api_key_id: null, created_at: "2026-07-10T10:02:00.000Z" },];- const execute = vi.fn().mockResolvedValue(undefined);- const db = {- execute,- select: () => ({- from() {- return this;- },- where() {- return this;- },- orderBy() {- return this;- },- limit: () => Promise.resolve(rows),- }),+ ch.client = {+ query: vi.fn().mockResolvedValue({ json: async () => rows }),};- const result = await listMcpActionLogs(db as any, TEAM_ID, { limit: 1 });+ const result = await listMcpActionLogs(TEAM_ID, { limit: 1 });expect(result.data).toEqual(rows.slice(0, 1));expect(result.data[0].api_key_id).toBe("9007199254740993");expect(decodeMcpActionLogCursor(result.nextCursor!)).toEqual({id: rows[0].id,created_at: rows[0].created_at,});- expect(execute).not.toHaveBeenCalled();+ const first = ch.client.query.mock.calls[0][0];+ expect(first.query).toContain("FROM mcp_action_logs AS log FINAL");+ expect(first.query).toContain("log.team_id = {teamId: UUID}");+ expect(first.query).toContain("INTERVAL 30 DAY");+ expect(first.query).not.toContain("{beforeId: UUID}");+ expect(first.query_params).toEqual({ teamId: TEAM_ID, limit: 2 });- const cursor = encodeMcpActionLogCursor(rows[0]);- expect(decodeMcpActionLogCursor(cursor)).toEqual({- id: rows[0].id,- created_at: rows[0].created_at,+ await listMcpActionLogs(TEAM_ID, { limit: 500, cursor: result.nextCursor });+ const second = ch.client.query.mock.calls[1][0];+ expect(second.query).toContain(+ "(log.created_at, log.id) < (parseDateTime64BestEffort({before: String}, 9), {beforeId: UUID})",+ );+ expect(second.query_params).toEqual({+ teamId: TEAM_ID,+ limit: 101,+ before: rows[0].created_at,+ beforeId: rows[0].id,});+expect(() => decodeMcpActionLogCursor("bad")).toThrow("cursor is invalid");+ await expect(listMcpActionLogs(TEAM_ID, { cursor: "bad" })).rejects.toThrow(+ "cursor is invalid",+ );- const complete = await listMcpActionLogs(db as any, TEAM_ID, { limit: 2 });+ const complete = await listMcpActionLogs(TEAM_ID, { limit: 2 });expect(complete.data[1].api_key_id).toBeNull();+ expect(complete.nextCursor).toBeNull();+ });++ it("never selects the objective or API job ID for the team-readable list", async () => {+ ch.client = { query: vi.fn().mockResolvedValue({ json: async () => [] }) };+ await listMcpActionLogs(TEAM_ID);+ const sql = ch.client.query.mock.calls[0][0].query as string;+ const columns = sql.slice(0, sql.indexOf(" FROM "));+ expect(columns).toContain("tool_name");+ expect(columns).not.toContain("objective");+ expect(columns).not.toContain("api_job_id");+ });++ it("reports storage as unavailable when the read fails", async () => {+ ch.client = { query: vi.fn().mockRejectedValue(new Error("timeout")) };+ await expect(listMcpActionLogs(TEAM_ID)).rejects.toThrow(+ "MCP action log storage is unavailable",+ );+ });++ it("reports storage as unavailable without ClickHouse", async () => {+ ch.client = null;+ await expect(listMcpActionLogs(TEAM_ID)).rejects.toThrow(+ "MCP action log storage is unavailable",+ );});});diff --git a/apps/api/src/services/mcp/action-logs.ts b/apps/api/src/services/mcp/action-logs.tsindex a773e5ff0..b1dfcbf7d 100644--- a/apps/api/src/services/mcp/action-logs.ts+++ b/apps/api/src/services/mcp/action-logs.ts@@ -1,21 +1,25 @@-import { and, desc, eq, gt, lt, or, sql } from "drizzle-orm";+import { and, eq, sql } from "drizzle-orm";import * as schema from "../../db/schema";import { authCreditUsageChunkFromTeam } from "../../db/rpc";+import { clickhouseClient } from "../../lib/clickhouse-client";import { logger } from "../../lib/logger";+import { logMcpActionLog } from "../logging/log_job";const STATUSES = ["success", "error"] as const;const AUTH_TYPES = ["oauth", "api-key"] as const;const RESOURCES = new Set(["https://mcp.firecrawl.dev/v2/mcp","https://mcp.firecrawl.dev/v2/mcp-oauth",+ "https://mcp.firecrawl.dev/v2/mcp-search",]);const UUID_PATTERN =/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;const POSTGRES_BIGINT_MAX = 9_223_372_036_854_775_807n;const SECRET_PATTERN =/(?:\bBearer\s+\S+|\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b|\b(?:sk-|fc-|fco_|fcr_|fcmcp_)[A-Za-z0-9_-]+)/i;+const SECRET_PATTERN_ALL = new RegExp(SECRET_PATTERN.source, "gi");+const OBJECTIVE_MAX_LENGTH = 1000;const RETENTION_DAYS = 30;-const CLEANUP_BATCH_SIZE = 500;const ALLOWED_FIELDS = new Set(["team_id","user_id",@@ -30,10 +34,13 @@ const ALLOWED_FIELDS = new Set(["client_version","error_class","resource",+ "objective",+ "api_job_id",]);export class McpActionLogValidationError extends Error {}export class McpActionLogAuthorizationError extends Error {}+export class McpActionLogUnavailableError extends Error {}function invalid(message: string): never {throw new McpActionLogValidationError(message);@@ -77,7 +84,22 @@ function text(value: unknown, field: string, max = 128, required = false) {return normalized || null;}-type McpActionLogInput = {+function objective(value: unknown) {+ if (value == null) return null;+ if (typeof value !== "string") invalid("objective must be a string");+ const normalized = value+ .slice(0, OBJECTIVE_MAX_LENGTH * 4)+ .replace(/[\u0000-\u001F\u007F-\u009F]+/g, " ")+ .replace(SECRET_PATTERN_ALL, "[redacted]")+ .replace(/\beyJ[A-Za-z0-9_.-]*$/, "[redacted]")+ .trim()+ .slice(0, OBJECTIVE_MAX_LENGTH)+ .replace(/[\uD800-\uDBFF]$/, "")+ .trim();+ return normalized || null;+}++export type McpActionLogInput = {team_id: string;user_id: string | null;api_key_id: string | null;@@ -90,6 +112,24 @@ type McpActionLogInput = {client_version: string | null;error_class: string | null;resource: string;+ objective: string | null;+ api_job_id: string | null;+};++type McpActionLogRow = {+ id: string;+ user_id: string | null;+ api_key_id: string | null;+ oauth_client_id: string | null;+ auth_type: string;+ tool_name: string;+ status: string;+ request_id: string;+ client_name: string | null;+ client_version: string | null;+ error_class: string | null;+ resource: string;+ created_at: string;};export function normalizeMcpActionLogInput(@@ -138,6 +178,8 @@ export function normalizeMcpActionLogInput(client_version: text(payload.client_version, "client_version"),error_class: text(payload.error_class, "error_class"),resource,+ objective: objective(payload.objective),+ api_job_id: text(payload.api_job_id, "api_job_id"),};}@@ -195,101 +237,9 @@ export async function resolveMcpActionLogTeamPolicy(db: any, teamId: string) {return { flags: rows[0].flags ?? null };}-async function cleanupExpiredMcpActionLogs(db: any) {- await db.execute(sql`- with expired as (- select id from mcp_action_logs- where expires_at <= now()- order by expires_at asc- limit ${CLEANUP_BATCH_SIZE}- )- delete from mcp_action_logs- where id in (select id from expired)- `);-}--export async function purgeMcpActionLogsForTeam(db: any, teamId: string) {- await db- .delete(schema.mcp_action_logs)- .where(eq(schema.mcp_action_logs.team_id, teamId));-}--type RetentionTimer = {- unref?: () => void;-};--export function startMcpActionLogRetentionWorker(options: {- db: any;- intervalMs?: number;- setIntervalFn?: (callback: () => void, intervalMs: number) => RetentionTimer;- clearIntervalFn?: (timer: RetentionTimer) => void;-}) {- const intervalMs = options.intervalMs ?? 5 * 60 * 1000;- const setIntervalFn =- options.setIntervalFn ?? ((callback, ms) => setInterval(callback, ms));- const clearIntervalFn =- options.clearIntervalFn ??- (timer => clearInterval(timer as NodeJS.Timeout));- let running = false;-- const run = async () => {- if (running) return;- running = true;- try {- await cleanupExpiredMcpActionLogs(options.db);- } catch (error) {- logger.warn("Failed to clean up expired MCP action logs", { error });- } finally {- running = false;- }- };-- const timer = setIntervalFn(() => void run(), intervalMs);- timer.unref?.();- const ready = run();-- return {- ready,- run,- stop: () => clearIntervalFn(timer),- };-}--export function startMcpActionLogRetentionWorkerIfEnabled(options: {- enabled: boolean;- db: any;- intervalMs?: number;- setIntervalFn?: (callback: () => void, intervalMs: number) => RetentionTimer;- clearIntervalFn?: (timer: RetentionTimer) => void;-}) {- if (!options.enabled) return null;- return startMcpActionLogRetentionWorker(options);-}--export async function recordMcpActionLog(db: any, input: McpActionLogInput) {- const expiresAt = new Date(- Date.now() + RETENTION_DAYS * 24 * 60 * 60 * 1000,- ).toISOString();- const rows = await db- .insert(schema.mcp_action_logs)- .values({- ...input,- api_key_id:- input.api_key_id === null- ? null- : sql`cast(${input.api_key_id} as bigint)`,- expires_at: expiresAt,- })- .onConflictDoNothing({- target: [- schema.mcp_action_logs.team_id,- schema.mcp_action_logs.request_id,- ],- })- .returning({ id: schema.mcp_action_logs.id });- return rows[0]- ? { disposition: "stored" as const, id: rows[0].id }- : { disposition: "duplicate" as const, id: null };+export async function recordMcpActionLog(input: McpActionLogInput) {+ await logMcpActionLog({ id: input.request_id, ...input });+ return { disposition: "accepted" as const, id: input.request_id };}export async function authorizeMcpActionLogViewer(@@ -345,16 +295,9 @@ export async function authorizeMcpActionLogViewer(return { userId: ownerId, role };}-export function encodeMcpActionLogCursor(row: {- created_at: Date | string;- id: string;-}) {- const createdAt =- row.created_at instanceof Date- ? row.created_at.toISOString()- : row.created_at;+function encodeMcpActionLogCursor(row: { created_at: string; id: string }) {return Buffer.from(- JSON.stringify({ created_at: createdAt, id: row.id }),+ JSON.stringify({ created_at: row.created_at, id: row.id }),).toString("base64url");}@@ -377,56 +320,39 @@ export function decodeMcpActionLogCursor(cursor: string) {}export async function listMcpActionLogs(- db: any,teamId: string,options: { limit?: number; cursor?: string | null } = {},) {+ if (!clickhouseClient) {+ throw new McpActionLogUnavailableError(+ "MCP action log storage is unavailable",+ );+ }const limit = Math.min(Math.max(options.limit ?? 50, 1), 100);const cursor = options.cursor? decodeMcpActionLogCursor(options.cursor): null;- const cursorClause = cursor- ? or(- lt(schema.mcp_action_logs.created_at, cursor.created_at),- and(- eq(schema.mcp_action_logs.created_at, cursor.created_at),- lt(schema.mcp_action_logs.id, cursor.id),- ),- )- : undefined;- const rows = await db- .select({- id: schema.mcp_action_logs.id,- user_id: schema.mcp_action_logs.user_id,- // API-key identifiers are bigint in Postgres. Read them as text so the- // JSON boundary never rounds values beyond JavaScript's safe integer range.- api_key_id: sql<- string | null- >`${schema.mcp_action_logs.api_key_id}::text`,- oauth_client_id: schema.mcp_action_logs.oauth_client_id,- auth_type: schema.mcp_action_logs.auth_type,- tool_name: schema.mcp_action_logs.tool_name,- status: schema.mcp_action_logs.status,- request_id: schema.mcp_action_logs.request_id,- client_name: schema.mcp_action_logs.client_name,- client_version: schema.mcp_action_logs.client_version,- error_class: schema.mcp_action_logs.error_class,- resource: schema.mcp_action_logs.resource,- created_at: schema.mcp_action_logs.created_at,- })- .from(schema.mcp_action_logs)- .where(- and(- eq(schema.mcp_action_logs.team_id, teamId),- gt(schema.mcp_action_logs.expires_at, new Date().toISOString()),- cursorClause,- ),- )- .orderBy(- desc(schema.mcp_action_logs.created_at),- desc(schema.mcp_action_logs.id),- )- .limit(limit + 1);+ let rows: McpActionLogRow[];+ try {+ const result = await clickhouseClient.query({+ // The explicit column list is the team-readable contract: objective and+ // api_job_id stay out of it. Filters name `log.` columns because the+ // formatted created_at alias would otherwise shadow the stored value.+ query: `SELECT id, user_id, toString(log.api_key_id) AS api_key_id, oauth_client_id, auth_type, tool_name, status, request_id, client_name, client_version, error_class, resource, formatDateTime(log.created_at, '%Y-%m-%dT%H:%i:%S.%fZ', 'UTC') AS created_at FROM mcp_action_logs AS log FINAL WHERE log.team_id = {teamId: UUID} AND log.created_at > now64(9) - INTERVAL ${RETENTION_DAYS} DAY${cursor ? " AND (log.created_at, log.id) < (parseDateTime64BestEffort({before: String}, 9), {beforeId: UUID})" : ""} ORDER BY log.created_at DESC, log.id DESC LIMIT {limit: UInt32}`,+ query_params: {+ teamId,+ limit: limit + 1,+ ...(cursor ? { before: cursor.created_at, beforeId: cursor.id } : {}),+ },+ format: "JSONEachRow",+ });+ rows = await result.json<McpActionLogRow>();+ } catch (error) {+ logger.warn("Failed to read MCP action logs", { error });+ throw new McpActionLogUnavailableError(+ "MCP action log storage is unavailable",+ );+ }return {data: rows.slice(0, limit),nextCursor: