Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): record provider scrapes in the activity log (#4679)

Developers Digest89f69ede1b34
diff --git a/apps/api/src/controllers/v1/__tests__/activity.test.ts b/apps/api/src/controllers/v1/__tests__/activity.test.ts
index 62ce61eae..edcb07b81 100644
--- a/apps/api/src/controllers/v1/__tests__/activity.test.ts
+++ b/apps/api/src/controllers/v1/__tests__/activity.test.ts
@@ -76,7 +76,7 @@ describe("activityController", () => {
expect(endpointRes.json).toHaveBeenCalledWith({
success: false,
error:
- "Invalid endpoint filter. Must be one of: scrape, crawl, batch_scrape, search, extract, llmstxt, deep_research, map, agent, browser, interact",
+ "Invalid endpoint filter. Must be one of: alexandria, scrape, crawl, batch_scrape, search, extract, llmstxt, deep_research, map, agent, browser, interact",
});
const cursorRes = makeRes();
@@ -230,4 +230,15 @@ describe("activityController", () => {
error: "Failed to fetch activity.",
});
});
+
+ it("accepts the Alexandria activity filter and includes historical scrape rows", async () => {
+ mockRows([]);
+ const res = makeRes();
+ await activityController(makeReq({ endpoint: "alexandria" }), res);
+ expect(res.status).toHaveBeenCalledWith(200);
+ expect(mocks.query).toHaveBeenCalled();
+ expect(mocks.query.mock.calls.at(-1)?.[0].query).toContain(
+ "(kind = 'alexandria' OR (kind = 'scrape' AND startsWith(target_hint, 'alexandria:')))",
+ );
+ });
});
diff --git a/apps/api/src/controllers/v1/activity.ts b/apps/api/src/controllers/v1/activity.ts
index 860739734..398d2f8fe 100644
--- a/apps/api/src/controllers/v1/activity.ts
+++ b/apps/api/src/controllers/v1/activity.ts
@@ -32,6 +32,7 @@ function toClickHouseDateTime(value: string): string {
}
const VALID_ENDPOINTS = [
+ "alexandria",
"scrape",
"crawl",
"batch_scrape",
@@ -126,7 +127,11 @@ export async function activityController(
limit: limit + 1,
};
- if (endpoint) {
+ if (endpoint === "alexandria") {
+ conditions.push(
+ "(kind = 'alexandria' OR (kind = 'scrape' AND startsWith(target_hint, 'alexandria:')))",
+ );
+ } else if (endpoint) {
conditions.push("kind = {endpoint: String}");
queryParams.endpoint = endpoint;
}
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.test.ts b/apps/api/src/controllers/v2/scrape-alexandria.test.ts
index 0334d084e..34af95d21 100644
--- a/apps/api/src/controllers/v2/scrape-alexandria.test.ts
+++ b/apps/api/src/controllers/v2/scrape-alexandria.test.ts
@@ -1,11 +1,18 @@
import express from "express";
import request from "supertest";
-const mocks = vi.hoisted(() => ({ retrieve: vi.fn(), log: vi.fn() }));
+const mocks = vi.hoisted(() => ({
+ retrieve: vi.fn(),
+ log: vi.fn(),
+ scrapeLog: vi.fn(),
+}));
vi.mock("../../services/alexandria/retrieve", () => ({
REQUEST_ID_PATTERN: /^[A-Za-z0-9._:-]{1,128}$/,
retrieveProviders: mocks.retrieve,
}));
-vi.mock("../../services/logging/log_job", () => ({ logRequest: mocks.log }));
+vi.mock("../../services/logging/log_job", () => ({
+ logRequest: mocks.log,
+ logProviderScrape: mocks.scrapeLog,
+}));
vi.mock("../../lib/key-restriction", () => ({
checkKeyFormatRestriction: async () => ({ allowed: true }),
}));
@@ -54,6 +61,7 @@ const result = (results: unknown[], executed = true) => ({
beforeEach(() => {
vi.clearAllMocks();
mocks.log.mockResolvedValue(undefined);
+ mocks.scrapeLog.mockResolvedValue(undefined);
mocks.retrieve.mockResolvedValue(
result([{ ...call, creditsCost: 0, data: {} }]),
);
@@ -92,9 +100,20 @@ it("returns the Scrape contract, shares identity with the legacy route, and logs
...second,
scrapeId: undefined,
});
+ expect(mocks.scrapeLog).toHaveBeenCalledTimes(1);
+ expect(mocks.scrapeLog).toHaveBeenCalledWith(
+ expect.objectContaining({
+ id: "scrape-1",
+ request_id: "scrape-1",
+ target: "alexandria:fred/categories/category",
+ credits_cost: 0,
+ is_successful: true,
+ time_taken: expect.any(Number),
+ }),
+ );
expect(mocks.log).toHaveBeenCalledTimes(1);
expect(mocks.log).toHaveBeenCalledWith(
- expect.objectContaining({ kind: "scrape", jobAccess: false }),
+ expect.objectContaining({ kind: "alexandria", jobAccess: false }),
);
});
@@ -144,4 +163,127 @@ it("only lets trusted agent interop bypass billing, and prefers its request id",
expect.objectContaining({ requestId: "agent-id", bypassBilling: true }),
);
expect(mocks.log).not.toHaveBeenCalled();
+ expect(mocks.scrapeLog).not.toHaveBeenCalled();
+});
+
+it("returns before logging finishes but orders the child after the parent", async () => {
+ let finish!: () => void;
+ mocks.log.mockReturnValue(
+ new Promise<void>(resolve => {
+ finish = resolve;
+ }),
+ );
+ const response = await request(app)
+ .post("/v2/scrape")
+ .send({ alexandria: call });
+ expect(response.status).toBe(200);
+ expect(mocks.scrapeLog).not.toHaveBeenCalled();
+ finish();
+ await vi.waitFor(() => expect(mocks.scrapeLog).toHaveBeenCalledTimes(1));
+});
+
+it("logs executed non-200 responses with known credits and failure details", async () => {
+ mocks.retrieve.mockResolvedValue({
+ executed: true,
+ scrapeId: "failed-1",
+ status: 503,
+ body: { error: "Provider unavailable", creditsCost: 2 },
+ });
+ const response = await request(app)
+ .post("/v2/scrape")
+ .send({ alexandria: call });
+ expect(response.status).toBe(503);
+ expect(mocks.scrapeLog).toHaveBeenCalledWith(
+ expect.objectContaining({
+ id: "failed-1",
+ request_id: "failed-1",
+ is_successful: false,
+ error: "Provider unavailable",
+ credits_cost: 2,
+ time_taken: expect.any(Number),
+ }),
+ );
+});
+
+it("does not log activity when retrieval rejects", async () => {
+ mocks.retrieve.mockRejectedValue(new Error("Unavailable"));
+ const response = await request(app)
+ .post("/v2/scrape")
+ .send({ alexandria: call });
+ expect(response.status).toBe(503);
+ expect(response.body).toEqual({
+ success: false,
+ error: "Provider request unavailable. Retry with the same x-request-id.",
+ });
+ expect(mocks.log).not.toHaveBeenCalled();
+ expect(mocks.scrapeLog).not.toHaveBeenCalled();
+});
+
+it("logs per-tool errors even when the response is HTTP 200", async () => {
+ mocks.retrieve.mockResolvedValue(
+ result([
+ {
+ ...call,
+ creditsCost: 0,
+ error: {
+ code: "provider_unavailable",
+ message: "Provider unavailable",
+ status: 503,
+ },
+ },
+ ]),
+ );
+ const response = await request(app)
+ .post("/v2/scrape")
+ .send({ alexandria: call });
+ expect(response.status).toBe(200);
+ expect(mocks.scrapeLog).toHaveBeenCalledWith(
+ expect.objectContaining({
+ is_successful: false,
+ error: "Provider unavailable",
+ credits_cost: 0,
+ }),
+ );
+});
+
+it("uses a failure fallback when no error message is available", async () => {
+ mocks.retrieve.mockResolvedValue({
+ executed: true,
+ scrapeId: "failed-1",
+ status: 503,
+ body: {},
+ });
+ const response = await request(app)
+ .post("/v2/scrape")
+ .send({ alexandria: call });
+ expect(response.status).toBe(503);
+ expect(mocks.scrapeLog).toHaveBeenCalledWith(
+ expect.objectContaining({
+ is_successful: false,
+ error: "Provider request failed (503).",
+ credits_cost: 0,
+ }),
+ );
+});
+
+it("executes with supplied arguments but stores only tool identifiers in activity options", async () => {
+ const supplied = {
+ ...call,
+ options: {
+ query: "private customer data",
+ token: "secret-token",
+ nested: { authorization: "Bearer secret" },
+ },
+ };
+ const response = await request(app)
+ .post("/v2/scrape")
+ .send({ alexandria: supplied });
+ expect(response.status).toBe(200);
+ expect(mocks.retrieve).toHaveBeenCalledWith(
+ expect.objectContaining({ calls: [supplied] }),
+ );
+ expect(mocks.scrapeLog).toHaveBeenCalledTimes(1);
+ expect(mocks.scrapeLog.mock.calls[0][0].options).toEqual({
+ alexandria: [{ provider: call.provider, capability: call.capability }],
+ });
});
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.ts
index 9f71e48b9..4d9cbddb1 100644
--- a/apps/api/src/controllers/v2/scrape-alexandria.ts
+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts
@@ -9,7 +9,7 @@ import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { checkKeyFormatRestriction } from "../../lib/key-restriction";
import { orgIdFromAcuc } from "../../lib/team-org";
-import { logRequest } from "../../services/logging/log_job";
+import { logProviderScrape, logRequest } from "../../services/logging/log_job";
import {
callsSchema,
callSchema,
@@ -119,6 +119,8 @@ export async function providerScrapeController(
.status(503)
.json({ success: false, error: "This endpoint is not available." });
+ const target = `alexandria:${body.alexandria.map(call => `${call.provider}/${call.capability}`).join(",")}`;
+ const startedAt = Date.now();
let result;
try {
result = await retrieveProviders({
@@ -144,25 +146,71 @@ export async function providerScrapeController(
error: "Provider request unavailable. Retry with the same x-request-id.",
});
}
- if (result.executed && !body.__agentInterop)
- void logRequest({
- id: result.scrapeId,
- kind: "scrape",
- api_version: "v2",
- external_request_id: externalRequestId(req),
- team_id: req.auth.team_id,
- api_key_id: req.acuc.api_key_id ?? null,
- origin: body.origin,
- integration: body.integration ?? null,
- target_hint: `alexandria:${body.alexandria.map(call => `${call.provider}/${call.capability}`).join(",")}`,
- zeroDataRetention: false,
- jobAccess: false,
- }).catch(error =>
- logger.warn("Provider request logging failed", {
+ const timeTaken = (Date.now() - startedAt) / 1000;
+ if (result.executed && !body.__agentInterop) {
+ const apiKeyId = req.acuc.api_key_id ?? null;
+ const served = answerSchema.safeParse(result.body);
+ const failure = result.body as {
+ error?: unknown;
+ creditsCost?: unknown;
+ } | null;
+ const successful =
+ result.status === 200 &&
+ served.success &&
+ served.data.results.every(item => !item.error);
+ const error = served.success
+ ? served.data.results.find(item => item.error)?.error?.message
+ : typeof failure?.error === "string"
+ ? failure.error
+ : undefined;
+ const credits = served.success
+ ? served.data.creditsCost
+ : failure?.creditsCost;
+ // Preserve parent/child ordering without delaying the provider response.
+ void (async () => {
+ await logRequest({
+ id: result.scrapeId,
+ kind: "alexandria",
+ api_version: "v2",
+ external_request_id: externalRequestId(req),
+ team_id: req.auth.team_id,
+ api_key_id: apiKeyId,
+ origin: body.origin,
+ integration: body.integration ?? null,
+ target_hint: target,
+ zeroDataRetention: false,
+ jobAccess: false,
+ });
+ await logProviderScrape({
+ id: result.scrapeId,
+ request_id: result.scrapeId,
+ target,
+ team_id: req.auth.team_id,
+ options: {
+ alexandria: body.alexandria.map(({ provider, capability }) => ({
+ provider,
+ capability,
+ })),
+ },
+ time_taken: timeTaken,
+ credits_cost:
+ typeof credits === "number" &&
+ Number.isFinite(credits) &&
+ credits >= 0
+ ? credits
+ : 0,
+ is_successful: successful,
+ error: successful
+ ? undefined
+ : (error ?? `Provider request failed (${result.status}).`),
+ });
+ })().catch(error =>
+ logger.warn("Provider activity logging failed", {
error,
scrapeId: result.scrapeId,
}),
);
+ }
if (result.status !== 200) return res.status(result.status).json(result.body);
const answer = answerSchema.parse(result.body);
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 17979ba15..d4c57370b 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -475,6 +475,7 @@ async function robustInsert(
type LoggedRequest = {
id: string;
kind:
+ | "alexandria"
| "scrape"
| "crawl"
| "batch_scrape"
@@ -884,6 +885,68 @@ async function logScrapeInternal(
}
}
+type LoggedProviderScrape = {
+ id: string;
+ request_id: string;
+ target: string;
+ team_id: string;
+ options: unknown;
+ time_taken: number;
+ credits_cost: number;
+ is_successful: boolean;
+ error?: string;
+};
+
+export async function logProviderScrape(scrape: LoggedProviderScrape) {
+ return withLogSpan(
+ {
+ operation: "provider_scrape",
+ table: "scrapes",
+ id: scrape.id,
+ requestId: scrape.request_id,
+ zeroDataRetention: false,
+ },
+ () => logProviderScrapeInternal(scrape),
+ );
+}
+
+async function logProviderScrapeInternal(scrape: LoggedProviderScrape) {
+ const logger = _logger.child({
+ module: "log_job",
+ method: "logProviderScrape",
+ scrapeId: scrape.id,
+ requestId: scrape.request_id,
+ teamId: scrape.team_id,
+ });
+ const storedTeamId =
+ keylessTeamUuid(scrape.team_id) ??
+ (scrape.team_id === "preview" || scrape.team_id?.startsWith("preview_")
+ ? previewTeamId
+ : scrape.team_id);
+
+ await robustInsert(
+ "scrapes",
+ {
+ id: scrape.id,
+ request_id: scrape.request_id,
+ url: scrape.target,
+ is_successful: scrape.is_successful,
+ error: scrape.error ?? null,
+ time_taken: scrape.time_taken,
+ team_id: storedTeamId,
+ options: scrape.options,
+ cost_tracking: null,
+ pdf_num_pages: null,
+ credits_cost: scrape.credits_cost,
+ monitor_id: null,
+ monitor_check_id: null,
+ content_type: null,
+ },
+ false,
+ logger,
+ );
+}
+
type LoggedCrawl = {
id: string;
request_id: string;