Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(mcp): accept error_message and target in the MCP action-log ingest (#5502)
claude[bot]5246476e883f
diff --git a/apps/api/src/services/mcp/action-logs.test.ts b/apps/api/src/services/mcp/action-logs.test.tsindex e5fbda8aa..be30bd861 100644--- a/apps/api/src/services/mcp/action-logs.test.ts+++ b/apps/api/src/services/mcp/action-logs.test.ts@@ -205,6 +205,108 @@ describe("MCP action log contract", () => {);});+ it("stores an optional error message and target, redacted and capped", () => {+ expect(input()).toMatchObject({ error_message: null, target: null });+ expect(+ input({+ status: "error",+ error_message: "Request failed with status code 408: Scrape timed out",+ target: "https://example.com/pricing",+ }),+ ).toMatchObject({+ error_message: "Request failed with status code 408: Scrape timed out",+ target: "https://example.com/pricing",+ });+ expect(input({ error_message: "", target: " " })).toMatchObject({+ error_message: null,+ target: null,+ });+ expect(+ input({ error_message: "Unauthorized:\nBearer abc.def for fc-123abc" })+ .error_message,+ ).toBe("Unauthorized: [redacted] for [redacted]");+ expect(+ input({ target: "https://example.com/?key=fc-123abc&page=2" }).target,+ ).toBe("https://example.com/?key=[redacted]&page=[redacted]");+ expect(+ input({+ target:+ "https://user:[email protected]/oauth/callback?code=c0de&state=s7ate&flag#access_token=t",+ }).target,+ ).toBe(+ "https://[redacted]@example.com/oauth/callback?code=[redacted]&state=[redacted]&[redacted]#[redacted]",+ );+ expect(+ input({+ target:+ "https://files.example.com/scl/fi/k8sj2h4l9q0w1e3r5t7y/report.pdf?dl=0",+ }).target,+ ).toBe(+ "https://files.example.com/scl/fi/[redacted]/report.pdf?dl=[redacted]",+ );+ expect(+ input({ target: "https://example.com/reset-password/abc" }).target,+ ).toBe("https://example.com/reset-password/[redacted]");+ expect(+ input({ target: "https://example.com/blog/how-to-build-an-api-in-2024" })+ .target,+ ).toBe("https://example.com/blog/how-to-build-an-api-in-2024");+ expect(+ input({+ error_message:+ "Authorization: Basic dXNlcjpwYXNz; otp=123456 resetToken: r3s3t verification_code=99 bad ?code=abc&state=def",+ }).error_message,+ ).toBe(+ "Authorization: [redacted]; otp=[redacted] resetToken: [redacted] verification_code=[redacted] bad ?code=[redacted]&state=[redacted]",+ );+ expect(input({ error_message: "Status code: 500" }).error_message).toBe(+ "Status code: 500",+ );+ expect(+ input({+ error_message:+ "{'password': 'hunter2', 'api_key': 'abc123secret'} {'Authorization': 'Basic dXNlcjpwYXNz'} {\"token\"=>\"t0k\"}",+ }).error_message,+ ).toBe(+ "{'password': [redacted], 'api_key': [redacted]} {'Authorization': [redacted]} {\"token\"=>[redacted]}",+ );+ expect(+ input({ error_message: "401 for Basic dXNlcjpwYXNz on Basic research" })+ .error_message,+ ).toBe("401 for [redacted] on Basic research");+ expect(+ input({+ error_message: "password=>Basic dXNlcjpwYXNz cred: Basic dXNlcjpwYXNz",+ }).error_message,+ ).toBe("password=>[redacted] cred: [redacted]");+ expect(input({ error_message: "401 for Basic YTo=" }).error_message).toBe(+ "401 for [redacted]",+ );+ for (const [message, expected] of [+ ["password=Basic dXNlcjpwYXNz", "password=[redacted]"],+ ["x-auth: Basic dXNlcjpwYXNz", "x-auth: [redacted]"],+ ["password=Basic c2VjcmV0dG9rZW4", "password=[redacted]"],+ ["auth_token: Token abc123tok", "auth_token: [redacted]"],+ ["secret=Bearer abc123tok", "secret=[redacted]"],+ ["password=hunter2 Basic research", "password=[redacted] Basic research"],+ ]) {+ expect(input({ error_message: message }).error_message).toBe(expected);+ }+ expect(input({ error_message: "x".repeat(400) }).error_message).toBe(+ "x".repeat(300),+ );+ expect(input({ target: "q".repeat(600) }).target).toBe("q".repeat(512));+ expect(input({ target: `${"q".repeat(511)}\u{1F600}` }).target).toBe(+ "q".repeat(511),+ );+ expect(() => input({ error_message: 500 })).toThrow(+ "error_message must be a string",+ );+ expect(() => input({ target: ["https://example.com"] })).toThrow(+ "target must be a string",+ );+ });+it("stores an optional coarse error code and rejects anything but snake_case", () => {expect(input().error_code).toBeNull();expect(@@ -517,6 +619,8 @@ describe("MCP action log contract", () => {objective: "Find the refund policy",api_job_id: REQUEST_ID,error_code: "rate_limited",+ error_message: "Rate limit exceeded",+ target: "https://example.com/pricing",user_agent: "Claude/1.0",mcp_origin: "mcp-ua-codex-cli-rs",mcp_client: "codex-cli-rs",@@ -541,6 +645,8 @@ describe("MCP action log contract", () => {objective: "Find the refund policy",api_job_id: REQUEST_ID,error_code: "rate_limited",+ error_message: "Rate limit exceeded",+ target: "https://example.com/pricing",mcp_origin: "mcp-ua-codex-cli-rs",mcp_client: "codex-cli-rs",mcp_client_source: "user_agent",@@ -651,7 +757,7 @@ describe("MCP action log contract", () => {expect(complete.nextCursor).toBeNull();});- it("never selects the objective or API job ID for the team-readable list", async () => {+ it("never selects call content or the API job ID for the team-readable list", async () => {ch.client = { query: vi.fn().mockResolvedValue({ json: async () => [] }) };await listMcpActionLogs(TEAM_ID);const sql = ch.client.query.mock.calls[0][0].query as string;@@ -659,6 +765,8 @@ describe("MCP action log contract", () => {expect(columns).toContain("tool_name");expect(columns).not.toContain("objective");expect(columns).not.toContain("api_job_id");+ expect(columns).not.toContain("error_message");+ expect(columns).not.toMatch(/\btarget\b/);});it("reports storage as unavailable when the read fails", async () => {diff --git a/apps/api/src/services/mcp/action-logs.ts b/apps/api/src/services/mcp/action-logs.tsindex 24cafd89c..fa80abf80 100644--- a/apps/api/src/services/mcp/action-logs.ts+++ b/apps/api/src/services/mcp/action-logs.ts@@ -18,7 +18,23 @@ const POSTGRES_BIGINT_MAX = 9_223_372_036_854_775_807n;const SECRET_PATTERN =/(?:\bBearer\s+\S+|\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b|\b(?:sk-|fc-|fco_|fcr_|fcmcp_)[A-Za-z0-9_-]+)/i;const SECRET_PATTERN_ALL = new RegExp(SECRET_PATTERN.source, "gi");+// After any secret key, a leading auth scheme (Basic, Bearer, Token, ...) is+// consumed with its credential so no part of "password=>Basic <b64>" survives.+const DETAIL_SECRET_PAIR = new RegExp(+ "(\\b(?:[A-Za-z0-9]*[_.-]?)(?:api[_-]?key|apikey|key|access[_-]?token|accesstoken|auth[_-]?token|token|client[_-]?secret|secret|password|passwd|passcode|pwd|auth|authorization|signature|sig|credentials?|session[_-]?id|sessionid|otp|nonce|verifier|cookie|jwt|(?:auth|oauth|authorization|verification|confirmation|reset|access|one[_-]?time)[_-]?code)[\"']?)(\\s*(?:=>|[=:])\\s*)(?:\"[^\"]*\"|'[^']*'|(?:(?:(?<=authorization[\"']?\\s*(?:=>|[=:])\\s*)[A-Za-z]+|Basic|Bearer|Token|Digest|Bot|ApiKey|Negotiate|NTLM)\\s+)?[^\\s&#\"',;]+)",+ "gi",+);+// A Basic credential is base64 of "user:password"; only one that decodes+// to a colon pair is redacted, so prose such as "Basic research" is kept.+const DETAIL_BASIC_AUTH = /\bBasic\s+([A-Za-z0-9+/]{2,}={0,2})/gi;+const DETAIL_QUERY_PAIR = /([?&][^\s=&#?]+=)[^\s&#]+/g;+const DETAIL_URL = /\b[a-z][a-z0-9+.-]*:\/\/[^\s"'<>`]+/gi;+const DETAIL_TOKEN_RUN = /[A-Za-z0-9]{16,}/g;+const DETAIL_TOKEN_PATH =+ /^(?:s|share|shared|invite|invitation|reset|reset[_-]?password|password[_-]?reset|verify|verification|confirm|confirmation|activate|magic|magic[_-]?link|token|auth|login|unsubscribe|callback|otp)$/i;const OBJECTIVE_MAX_LENGTH = 1000;+const ERROR_MESSAGE_MAX_LENGTH = 300;+const TARGET_MAX_LENGTH = 512;const ERROR_CODE_PATTERN = /^(?=.{2,64}$)[a-z][a-z0-9]*(?:_[a-z0-9]+)*$/;const MCP_CLIENT_SOURCES = ["client_info", "user_agent", "oauth"] as const;const RATE_LIMIT_RETRY_OUTCOMES = [@@ -51,6 +67,8 @@ const ALLOWED_FIELDS = new Set(["client_version","error_class","error_code",+ "error_message",+ "target","resource","objective","api_job_id",@@ -125,16 +143,95 @@ function errorCode(value: unknown) {return value;}-function objective(value: unknown) {+function tokenLikeSegment(segment: string) {+ return (segment.match(DETAIL_TOKEN_RUN) ?? []).some(+ run => /\d/.test(run) && /[A-Za-z]/.test(run),+ );+}++/**+ * Keeps a URL's scheme, host and plain path: userinfo, every query value, the+ * fragment and token-like path segments become [redacted].+ */+function scrubbedUrl(url: string) {+ const hashAt = url.indexOf("#");+ const fragment =+ hashAt === -1 || hashAt === url.length - 1 ? "" : "#[redacted]";+ const beforeHash = hashAt === -1 ? url : url.slice(0, hashAt);+ const queryAt = beforeHash.indexOf("?");+ const base = queryAt === -1 ? beforeHash : beforeHash.slice(0, queryAt);+ const query =+ queryAt === -1+ ? ""+ : "?" ++ beforeHash+ .slice(queryAt + 1)+ .split("&")+ .map(pair => {+ const eq = pair.indexOf("=");+ if (eq === -1) return pair && "[redacted]";+ return eq === pair.length - 1+ ? pair+ : pair.slice(0, eq + 1) + "[redacted]";+ })+ .join("&");+ const match = /^([^:]+:\/\/)([^/]*)(.*)$/.exec(base);+ if (!match) return base + query + fragment;+ const [, scheme, authority, path] = match;+ const host = authority.includes("@")+ ? "[redacted]" + authority.slice(authority.lastIndexOf("@"))+ : authority;+ const segments = path.split("/");+ const scrubbedPath = segments+ .map((segment, i) =>+ tokenLikeSegment(segment) ||+ (i > 0 && DETAIL_TOKEN_PATH.test(segments[i - 1]))+ ? "[redacted]"+ : segment,+ )+ .join("/");+ return scheme + host + scrubbedPath + query + fragment;+}++/**+ * Error messages and targets also lose URL userinfo, query values, fragments,+ * token-like path segments, bare query-string values, secret=value pairs+ * (keys bare or quoted, as in JSON or Python and Ruby dicts) and Basic+ * credentials.+ */+function scrubbedDetail(text: string) {+ return text+ .replace(SECRET_PATTERN_ALL, "[redacted]")+ .replace(DETAIL_URL, scrubbedUrl)+ .replace(DETAIL_QUERY_PAIR, "$1[redacted]")+ .replace(DETAIL_BASIC_AUTH, (match, token: string) =>+ Buffer.from(token, "base64").toString("latin1").includes(":")+ ? "[redacted]"+ : match,+ )+ .replace(DETAIL_SECRET_PAIR, "$1$2[redacted]");+}++/**+ * Free text from the call (objective, error message, target URL):+ * control characters become spaces, secret-like values become [redacted],+ * and the result is cut to `max` characters rather than rejected.+ */+function redactedText(+ value: unknown,+ field: string,+ max: number,+ scrub: (text: string) => string = text => text,+) {if (value == null) return null;- if (typeof value !== "string") invalid("objective must be a string");- const normalized = value- .slice(0, OBJECTIVE_MAX_LENGTH * 4)- .replace(/[\u0000-\u001F\u007F-\u009F]+/g, " ")+ if (typeof value !== "string") invalid(`${field} must be a string`);+ const normalized = scrub(+ value.slice(0, max * 4).replace(/[\u0000-\u001F\u007F-\u009F]+/g, " "),+ ).replace(SECRET_PATTERN_ALL, "[redacted]").replace(/\beyJ[A-Za-z0-9_.-]*$/, "[redacted]").trim()- .slice(0, OBJECTIVE_MAX_LENGTH)+ .slice(0, max).replace(/[\uD800-\uDBFF]$/, "").trim();return normalized || null;@@ -211,6 +308,8 @@ type McpActionLogInput = {client_version: string | null;error_class: string | null;error_code: string | null;+ error_message: string | null;+ target: string | null;resource: string;objective: string | null;api_job_id: string | null;@@ -311,8 +410,24 @@ export function normalizeMcpActionLogInput(client_version: text(payload.client_version, "client_version"),error_class: text(payload.error_class, "error_class"),error_code: errorCode(payload.error_code),+ error_message: redactedText(+ payload.error_message,+ "error_message",+ ERROR_MESSAGE_MAX_LENGTH,+ scrubbedDetail,+ ),+ target: redactedText(+ payload.target,+ "target",+ TARGET_MAX_LENGTH,+ scrubbedDetail,+ ),resource,- objective: objective(payload.objective),+ objective: redactedText(+ payload.objective,+ "objective",+ OBJECTIVE_MAX_LENGTH,+ ),api_job_id: text(payload.api_job_id, "api_job_id"),mcp_origin: text(payload.mcp_origin, "mcp_origin"),mcp_client: text(payload.mcp_client, "mcp_client"),@@ -481,8 +596,8 @@ export async function listMcpActionLogs(let rows: McpActionLogRow[];try {const result = await clickhouseClient.query({- // The explicit column list is the team-readable contract: objective and- // api_job_id stay out of it. Filters name `log.` columns because the+ // The explicit column list is the team-readable contract: objective,+ // api_job_id, error_message and target stay out of it. Filters name `log.` columns because the// formatted created_at alias would otherwise shadow the stored value.query: `SELECT id, user_id, toString(log.api_key_id) AS api_key_id, oauth_client_id, auth_type, tool_name, status, request_id, client_name, client_version, error_class, resource, formatDateTime(log.created_at, '%Y-%m-%dT%H:%i:%S.%fZ', 'UTC') AS created_at FROM mcp_action_logs AS log FINAL WHERE log.team_id = {teamId: UUID} AND log.created_at > now64(9) - INTERVAL ${RETENTION_DAYS} DAY${cursor ? " AND (log.created_at, log.id) < (parseDateTime64BestEffort({before: String}, 9), {beforeId: UUID})" : ""} ORDER BY log.created_at DESC, log.id DESC LIMIT {limit: UInt32}`,query_params: {