Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): record provider scrapes in the activity log (#4679)
Developers Digest89f69ede1b34
diff --git a/apps/api/src/controllers/v1/__tests__/activity.test.ts b/apps/api/src/controllers/v1/__tests__/activity.test.tsindex 62ce61eae..edcb07b81 100644--- a/apps/api/src/controllers/v1/__tests__/activity.test.ts+++ b/apps/api/src/controllers/v1/__tests__/activity.test.ts@@ -76,7 +76,7 @@ describe("activityController", () => {expect(endpointRes.json).toHaveBeenCalledWith({success: false,error:- "Invalid endpoint filter. Must be one of: scrape, crawl, batch_scrape, search, extract, llmstxt, deep_research, map, agent, browser, interact",+ "Invalid endpoint filter. Must be one of: alexandria, scrape, crawl, batch_scrape, search, extract, llmstxt, deep_research, map, agent, browser, interact",});const cursorRes = makeRes();@@ -230,4 +230,15 @@ describe("activityController", () => {error: "Failed to fetch activity.",});});++ it("accepts the Alexandria activity filter and includes historical scrape rows", async () => {+ mockRows([]);+ const res = makeRes();+ await activityController(makeReq({ endpoint: "alexandria" }), res);+ expect(res.status).toHaveBeenCalledWith(200);+ expect(mocks.query).toHaveBeenCalled();+ expect(mocks.query.mock.calls.at(-1)?.[0].query).toContain(+ "(kind = 'alexandria' OR (kind = 'scrape' AND startsWith(target_hint, 'alexandria:')))",+ );+ });});diff --git a/apps/api/src/controllers/v1/activity.ts b/apps/api/src/controllers/v1/activity.tsindex 860739734..398d2f8fe 100644--- a/apps/api/src/controllers/v1/activity.ts+++ b/apps/api/src/controllers/v1/activity.ts@@ -32,6 +32,7 @@ function toClickHouseDateTime(value: string): string {}const VALID_ENDPOINTS = [+ "alexandria","scrape","crawl","batch_scrape",@@ -126,7 +127,11 @@ export async function activityController(limit: limit + 1,};- if (endpoint) {+ if (endpoint === "alexandria") {+ conditions.push(+ "(kind = 'alexandria' OR (kind = 'scrape' AND startsWith(target_hint, 'alexandria:')))",+ );+ } else if (endpoint) {conditions.push("kind = {endpoint: String}");queryParams.endpoint = endpoint;}diff --git a/apps/api/src/controllers/v2/scrape-alexandria.test.ts b/apps/api/src/controllers/v2/scrape-alexandria.test.tsindex 0334d084e..34af95d21 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.test.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.test.ts@@ -1,11 +1,18 @@import express from "express";import request from "supertest";-const mocks = vi.hoisted(() => ({ retrieve: vi.fn(), log: vi.fn() }));+const mocks = vi.hoisted(() => ({+ retrieve: vi.fn(),+ log: vi.fn(),+ scrapeLog: vi.fn(),+}));vi.mock("../../services/alexandria/retrieve", () => ({REQUEST_ID_PATTERN: /^[A-Za-z0-9._:-]{1,128}$/,retrieveProviders: mocks.retrieve,}));-vi.mock("../../services/logging/log_job", () => ({ logRequest: mocks.log }));+vi.mock("../../services/logging/log_job", () => ({+ logRequest: mocks.log,+ logProviderScrape: mocks.scrapeLog,+}));vi.mock("../../lib/key-restriction", () => ({checkKeyFormatRestriction: async () => ({ allowed: true }),}));@@ -54,6 +61,7 @@ const result = (results: unknown[], executed = true) => ({beforeEach(() => {vi.clearAllMocks();mocks.log.mockResolvedValue(undefined);+ mocks.scrapeLog.mockResolvedValue(undefined);mocks.retrieve.mockResolvedValue(result([{ ...call, creditsCost: 0, data: {} }]),);@@ -92,9 +100,20 @@ it("returns the Scrape contract, shares identity with the legacy route, and logs...second,scrapeId: undefined,});+ expect(mocks.scrapeLog).toHaveBeenCalledTimes(1);+ expect(mocks.scrapeLog).toHaveBeenCalledWith(+ expect.objectContaining({+ id: "scrape-1",+ request_id: "scrape-1",+ target: "alexandria:fred/categories/category",+ credits_cost: 0,+ is_successful: true,+ time_taken: expect.any(Number),+ }),+ );expect(mocks.log).toHaveBeenCalledTimes(1);expect(mocks.log).toHaveBeenCalledWith(- expect.objectContaining({ kind: "scrape", jobAccess: false }),+ expect.objectContaining({ kind: "alexandria", jobAccess: false }),);});@@ -144,4 +163,127 @@ it("only lets trusted agent interop bypass billing, and prefers its request id",expect.objectContaining({ requestId: "agent-id", bypassBilling: true }),);expect(mocks.log).not.toHaveBeenCalled();+ expect(mocks.scrapeLog).not.toHaveBeenCalled();+});++it("returns before logging finishes but orders the child after the parent", async () => {+ let finish!: () => void;+ mocks.log.mockReturnValue(+ new Promise<void>(resolve => {+ finish = resolve;+ }),+ );+ const response = await request(app)+ .post("/v2/scrape")+ .send({ alexandria: call });+ expect(response.status).toBe(200);+ expect(mocks.scrapeLog).not.toHaveBeenCalled();+ finish();+ await vi.waitFor(() => expect(mocks.scrapeLog).toHaveBeenCalledTimes(1));+});++it("logs executed non-200 responses with known credits and failure details", async () => {+ mocks.retrieve.mockResolvedValue({+ executed: true,+ scrapeId: "failed-1",+ status: 503,+ body: { error: "Provider unavailable", creditsCost: 2 },+ });+ const response = await request(app)+ .post("/v2/scrape")+ .send({ alexandria: call });+ expect(response.status).toBe(503);+ expect(mocks.scrapeLog).toHaveBeenCalledWith(+ expect.objectContaining({+ id: "failed-1",+ request_id: "failed-1",+ is_successful: false,+ error: "Provider unavailable",+ credits_cost: 2,+ time_taken: expect.any(Number),+ }),+ );+});++it("does not log activity when retrieval rejects", async () => {+ mocks.retrieve.mockRejectedValue(new Error("Unavailable"));+ const response = await request(app)+ .post("/v2/scrape")+ .send({ alexandria: call });+ expect(response.status).toBe(503);+ expect(response.body).toEqual({+ success: false,+ error: "Provider request unavailable. Retry with the same x-request-id.",+ });+ expect(mocks.log).not.toHaveBeenCalled();+ expect(mocks.scrapeLog).not.toHaveBeenCalled();+});++it("logs per-tool errors even when the response is HTTP 200", async () => {+ mocks.retrieve.mockResolvedValue(+ result([+ {+ ...call,+ creditsCost: 0,+ error: {+ code: "provider_unavailable",+ message: "Provider unavailable",+ status: 503,+ },+ },+ ]),+ );+ const response = await request(app)+ .post("/v2/scrape")+ .send({ alexandria: call });+ expect(response.status).toBe(200);+ expect(mocks.scrapeLog).toHaveBeenCalledWith(+ expect.objectContaining({+ is_successful: false,+ error: "Provider unavailable",+ credits_cost: 0,+ }),+ );+});++it("uses a failure fallback when no error message is available", async () => {+ mocks.retrieve.mockResolvedValue({+ executed: true,+ scrapeId: "failed-1",+ status: 503,+ body: {},+ });+ const response = await request(app)+ .post("/v2/scrape")+ .send({ alexandria: call });+ expect(response.status).toBe(503);+ expect(mocks.scrapeLog).toHaveBeenCalledWith(+ expect.objectContaining({+ is_successful: false,+ error: "Provider request failed (503).",+ credits_cost: 0,+ }),+ );+});++it("executes with supplied arguments but stores only tool identifiers in activity options", async () => {+ const supplied = {+ ...call,+ options: {+ query: "private customer data",+ token: "secret-token",+ nested: { authorization: "Bearer secret" },+ },+ };+ const response = await request(app)+ .post("/v2/scrape")+ .send({ alexandria: supplied });+ expect(response.status).toBe(200);+ expect(mocks.retrieve).toHaveBeenCalledWith(+ expect.objectContaining({ calls: [supplied] }),+ );+ expect(mocks.scrapeLog).toHaveBeenCalledTimes(1);+ expect(mocks.scrapeLog.mock.calls[0][0].options).toEqual({+ alexandria: [{ provider: call.provider, capability: call.capability }],+ });});diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsindex 9f71e48b9..4d9cbddb1 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -9,7 +9,7 @@ import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";import { checkKeyFormatRestriction } from "../../lib/key-restriction";import { orgIdFromAcuc } from "../../lib/team-org";-import { logRequest } from "../../services/logging/log_job";+import { logProviderScrape, logRequest } from "../../services/logging/log_job";import {callsSchema,callSchema,@@ -119,6 +119,8 @@ export async function providerScrapeController(.status(503).json({ success: false, error: "This endpoint is not available." });+ const target = `alexandria:${body.alexandria.map(call => `${call.provider}/${call.capability}`).join(",")}`;+ const startedAt = Date.now();let result;try {result = await retrieveProviders({@@ -144,25 +146,71 @@ export async function providerScrapeController(error: "Provider request unavailable. Retry with the same x-request-id.",});}- if (result.executed && !body.__agentInterop)- void logRequest({- id: result.scrapeId,- kind: "scrape",- api_version: "v2",- external_request_id: externalRequestId(req),- team_id: req.auth.team_id,- api_key_id: req.acuc.api_key_id ?? null,- origin: body.origin,- integration: body.integration ?? null,- target_hint: `alexandria:${body.alexandria.map(call => `${call.provider}/${call.capability}`).join(",")}`,- zeroDataRetention: false,- jobAccess: false,- }).catch(error =>- logger.warn("Provider request logging failed", {+ const timeTaken = (Date.now() - startedAt) / 1000;+ if (result.executed && !body.__agentInterop) {+ const apiKeyId = req.acuc.api_key_id ?? null;+ const served = answerSchema.safeParse(result.body);+ const failure = result.body as {+ error?: unknown;+ creditsCost?: unknown;+ } | null;+ const successful =+ result.status === 200 &&+ served.success &&+ served.data.results.every(item => !item.error);+ const error = served.success+ ? served.data.results.find(item => item.error)?.error?.message+ : typeof failure?.error === "string"+ ? failure.error+ : undefined;+ const credits = served.success+ ? served.data.creditsCost+ : failure?.creditsCost;+ // Preserve parent/child ordering without delaying the provider response.+ void (async () => {+ await logRequest({+ id: result.scrapeId,+ kind: "alexandria",+ api_version: "v2",+ external_request_id: externalRequestId(req),+ team_id: req.auth.team_id,+ api_key_id: apiKeyId,+ origin: body.origin,+ integration: body.integration ?? null,+ target_hint: target,+ zeroDataRetention: false,+ jobAccess: false,+ });+ await logProviderScrape({+ id: result.scrapeId,+ request_id: result.scrapeId,+ target,+ team_id: req.auth.team_id,+ options: {+ alexandria: body.alexandria.map(({ provider, capability }) => ({+ provider,+ capability,+ })),+ },+ time_taken: timeTaken,+ credits_cost:+ typeof credits === "number" &&+ Number.isFinite(credits) &&+ credits >= 0+ ? credits+ : 0,+ is_successful: successful,+ error: successful+ ? undefined+ : (error ?? `Provider request failed (${result.status}).`),+ });+ })().catch(error =>+ logger.warn("Provider activity logging failed", {error,scrapeId: result.scrapeId,}),);+ }if (result.status !== 200) return res.status(result.status).json(result.body);const answer = answerSchema.parse(result.body);diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 17979ba15..d4c57370b 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -475,6 +475,7 @@ async function robustInsert(type LoggedRequest = {id: string;kind:+ | "alexandria"| "scrape"| "crawl"| "batch_scrape"@@ -884,6 +885,68 @@ async function logScrapeInternal(}}+type LoggedProviderScrape = {+ id: string;+ request_id: string;+ target: string;+ team_id: string;+ options: unknown;+ time_taken: number;+ credits_cost: number;+ is_successful: boolean;+ error?: string;+};++export async function logProviderScrape(scrape: LoggedProviderScrape) {+ return withLogSpan(+ {+ operation: "provider_scrape",+ table: "scrapes",+ id: scrape.id,+ requestId: scrape.request_id,+ zeroDataRetention: false,+ },+ () => logProviderScrapeInternal(scrape),+ );+}++async function logProviderScrapeInternal(scrape: LoggedProviderScrape) {+ const logger = _logger.child({+ module: "log_job",+ method: "logProviderScrape",+ scrapeId: scrape.id,+ requestId: scrape.request_id,+ teamId: scrape.team_id,+ });+ const storedTeamId =+ keylessTeamUuid(scrape.team_id) ??+ (scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")+ ? previewTeamId+ : scrape.team_id);++ await robustInsert(+ "scrapes",+ {+ id: scrape.id,+ request_id: scrape.request_id,+ url: scrape.target,+ is_successful: scrape.is_successful,+ error: scrape.error ?? null,+ time_taken: scrape.time_taken,+ team_id: storedTeamId,+ options: scrape.options,+ cost_tracking: null,+ pdf_num_pages: null,+ credits_cost: scrape.credits_cost,+ monitor_id: null,+ monitor_check_id: null,+ content_type: null,+ },+ false,+ logger,+ );+}+type LoggedCrawl = {id: string;request_id: string;