Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { z } from "zod";
import type { Response } from "express";
import { randomUUID } from "node:crypto";
import { v7 as uuidv7 } from "uuid";
import { config } from "../../config";
import { logger } from "../../lib/logger";
import { isAgentInteropSecretValid } from "../../lib/agent-interop";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { keylessProviderToolsBody } from "../../lib/keyless";
import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { billingCountryForTeam } from "../../lib/billing-country";
import { checkKeyFormatRestriction } from "../../lib/key-restriction";
import { orgIdFromAcuc } from "../../lib/team-org";
import { issueFeedbackToken } from "../../lib/alexandria-feedback-token";
import { logProviderScrape, logRequest } from "../../services/logging/log_job";
import {
markAlexandriaActivity,
markAlexandriaCalls,
} from "../../lib/alexandria-activity";
import {
discoveryForCalls,
recordDiscoveredTools,
} from "../../lib/alexandria-discovery";
import {
callsSchema,
callSchema,
answerSchema,
} from "../../services/alexandria/contracts";
import {
REQUEST_ID_PATTERN,
exchangeRequestId,
retrieveProviders,
} from "../../services/alexandria/retrieve";
import type { RequestWithAuth } from "./types";
const FIND_TOOLS_LOGGED_OPTIONS = [
"query",
"providers",
"categories",
"groups",
"capabilities",
"level",
"limit",
"offset",
"expand",
];
const isFindTools = (call: { provider?: string; capability?: string }) =>
call.provider === "firecrawl" && call.capability === "find-tools";
const findToolsItems = (data: unknown): unknown[] => {
const items = (data as { items?: unknown } | null)?.items;
return Array.isArray(items) ? items : [];
};
const agentInteropSchema = z.strictObject({
auth: z.string(),
requestId: z.string(),
shouldBill: z.boolean(),
boostConcurrency: z.boolean().optional(),
hostedMcp: z.boolean().optional(),
});
const providerScrapeSchema = z.strictObject({
alexandria: z.preprocess(
value => (Array.isArray(value) ? value : [value]),
callsSchema,
),
timeout: z
.number()
.int()
.positive()
.default(120_000)
.transform(value => Math.min(value, 120_000)),
origin: z.string().default("api"),
integration: z.string().nullable().optional(),
zeroDataRetention: z.boolean().optional(),
maxCredits: z.number().int().nonnegative().safe().optional(),
__agentInterop: agentInteropSchema.optional(),
});
export async function providerScrapeController(
req: RequestWithAuth<any, any, any>,
res: Response,
legacy = false,
) {
const legacyZeroDataRetention = {
zeroDataRetention: z.boolean().optional(),
__agentInterop: agentInteropSchema.optional(),
};
const legacyBody = legacy
? z
.union([
callSchema.extend(legacyZeroDataRetention),
z.strictObject({ requests: callsSchema, ...legacyZeroDataRetention }),
])
.parse(req.body)
: null;
const body = providerScrapeSchema.parse(
legacyBody
? (({ zeroDataRetention, __agentInterop, ...rest }) => ({
alexandria: "requests" in rest ? rest.requests : [rest],
zeroDataRetention,
__agentInterop,
}))(legacyBody)
: req.body,
);
if (body.__agentInterop) {
if (!config.AGENT_INTEROP_SECRET)
return res
.status(403)
.json({ success: false, error: "Agent interop is not enabled." });
if (!isAgentInteropSecretValid(body.__agentInterop.auth))
return res
.status(403)
.json({ success: false, error: "Invalid agent interop." });
}
const requestId = req.get("x-request-id") ?? randomUUID();
if (REQUEST_ID_PATTERN.test(requestId))
res.setHeader("x-request-id", requestId);
if (req.auth.team_id.startsWith("preview_keyless_")) {
applyAgentAuthDiscoveryHeader(res);
return res
.status(403)
.json(keylessProviderToolsBody(req.auth.team_id, req));
}
if (!req.acuc)
return res.status(403).json({
success: false,
error: "This endpoint is not enabled for this team.",
});
// Exchange refuses capabilities that cannot honour ZDR, per call.
const zeroDataRetention =
getScrapeZDR(req.acuc.flags) === "forced" ||
body.zeroDataRetention === true;
if (
(req as any).agentIndexOnly ||
["pending", "blocked"].includes(req.acuc._agentSponsor?.status ?? "")
)
return res.status(403).json({
success: false,
code: "AGENT_INDEX_ONLY",
error: "Verify this API key before executing provider tools.",
});
const restriction = await checkKeyFormatRestriction(
["json"],
[],
req.acuc.api_key_id,
req.acuc.flags,
);
if (!restriction.allowed)
return res
.status(restriction.status)
.json({ success: false, error: restriction.error });
if (!config.FIRE_EXCHANGE_URL)
return res
.status(503)
.json({ success: false, error: "This endpoint is not available." });
const target = `alexandria:${body.alexandria.map(call => `${call.provider}/${call.capability}`).join(",")}`;
const discovery =
zeroDataRetention || body.__agentInterop
? Promise.resolve(null)
: discoveryForCalls(req.auth.team_id, body.alexandria).catch(error => {
logger.warn("Alexandria discovery lookup failed", { error });
return null;
});
const startedAt = Date.now();
let result;
try {
result = await retrieveProviders({
teamId: req.auth.team_id,
orgId: orgIdFromAcuc(req.acuc),
apiKeyId: req.acuc.api_key_id ?? null,
apiKeyIdText: req.acuc.api_key_id_text ?? null,
flags: req.acuc.flags,
calls: body.alexandria,
resultAuthorization: req.get("authorization"),
requestId,
agentRequestId: body.__agentInterop?.requestId,
hostedMcp: req.auth.hostedMcp,
scrapeId: uuidv7(),
timeoutMs: body.timeout,
bypassBilling: body.__agentInterop?.shouldBill === false,
zeroDataRetention,
clientIp: req.ip,
billingCountry: await billingCountryForTeam(req.auth.team_id),
rateLimitMultiplier: req.acuc.rate_limit_multiplier,
...(body.maxCredits === undefined ? {} : { maxCredits: body.maxCredits }),
});
} catch (error) {
logger.error("Provider scrape unavailable", {
error,
teamId: req.auth.team_id,
});
return res.status(503).json({
success: false,
error: "Provider request unavailable. Retry with the same x-request-id.",
});
}
const timeTaken = (Date.now() - startedAt) / 1000;
markAlexandriaActivity(req.auth.team_id);
const usageRequestId = exchangeRequestId(req.auth.team_id, requestId);
if (result.executed && !zeroDataRetention)
markAlexandriaCalls(
req.auth.team_id,
body.alexandria.map(({ provider, capability }) => ({
requestId: usageRequestId,
provider,
capability,
})),
);
if (result.executed && !body.__agentInterop) {
const apiKeyId = req.acuc.api_key_id ?? null;
const served = answerSchema.safeParse(result.body);
const failure = result.body as {
error?: unknown;
creditsCost?: unknown;
} | null;
const successful =
result.status === 200 &&
served.success &&
served.data.results.every(item => !item.error);
const error = served.success
? served.data.results.find(item => item.error)?.error?.message
: typeof failure?.error === "string"
? failure.error
: undefined;
const credits = served.success
? served.data.creditsCost
: failure?.creditsCost;
if (served.success && !zeroDataRetention)
recordDiscoveredTools(
req.auth.team_id,
"find_tools",
result.scrapeId,
served.data.results.flatMap(item =>
isFindTools(item)
? findToolsItems(item.data).flatMap(tool => {
const { provider, capability } = (tool ?? {}) as {
provider?: unknown;
capability?: unknown;
};
return typeof provider === "string" &&
typeof capability === "string"
? [{ provider, capability }]
: [];
})
: [],
),
);
// Preserve parent/child ordering without delaying the provider response.
void (async () => {
// Bounded so a slow Redis never holds back the activity log.
const discovered = await Promise.race([
discovery,
new Promise<null>(resolve =>
setTimeout(() => resolve(null), 500).unref(),
),
]);
await logRequest({
id: result.scrapeId,
kind: "alexandria",
api_version: "v2",
external_request_id: externalRequestId(req),
team_id: req.auth.team_id,
api_key_id: apiKeyId,
origin: body.origin,
integration: body.integration ?? null,
target_hint: target,
zeroDataRetention,
jobAccess: false,
});
await logProviderScrape({
id: result.scrapeId,
request_id: result.scrapeId,
target,
team_id: req.auth.team_id,
options: {
alexandria: body.alexandria.map(
({ provider, capability, options }, i) => ({
provider,
capability,
...(served.success && served.data.results[i]
? {
code: served.data.results[i].error?.code ?? null,
creditsCost: served.data.results[i].creditsCost ?? 0,
...(served.data.results[i].provenance
? { provenance: served.data.results[i].provenance }
: {}),
}
: {}),
...(isFindTools({ provider, capability })
? {
options: {
...Object.fromEntries(
Object.entries(options).filter(([key]) =>
FIND_TOOLS_LOGGED_OPTIONS.includes(key),
),
),
...(Array.isArray(options.urls)
? { urlCount: options.urls.length }
: {}),
},
...(served.success
? {
returned: findToolsItems(served.data.results[i]?.data)
.length,
}
: {}),
}
: {}),
...(discovered && Object.keys(discovered[i]).length
? { discovery: discovered[i] }
: {}),
}),
),
},
time_taken: timeTaken,
credits_cost:
typeof credits === "number" &&
Number.isFinite(credits) &&
credits >= 0
? credits
: 0,
is_successful: successful,
error: successful
? undefined
: (error ?? `Provider request failed (${result.status}).`),
zeroDataRetention,
});
})().catch(error =>
logger.warn("Provider activity logging failed", {
error,
scrapeId: result.scrapeId,
}),
);
}
if (result.status !== 200) {
const refused = result.body as {
code?: unknown;
retryAfterSeconds?: unknown;
} | null;
if (
(refused?.code === "index_daily_limit" ||
refused?.code === "index_rate_limit") &&
typeof refused.retryAfterSeconds === "number"
)
res.setHeader("Retry-After", String(refused.retryAfterSeconds));
return res.status(result.status).json(result.body);
}
const exchangeAnswer = answerSchema.parse(result.body);
const parsed = {
...exchangeAnswer,
results: exchangeAnswer.results.map(
({ provenance: _provenance, ...item }) => item,
),
};
const answer = zeroDataRetention
? parsed
: {
...parsed,
results: parsed.results.map((item, i) => {
const provider = item.provider ?? body.alexandria[i]?.provider;
const capability = item.capability ?? body.alexandria[i]?.capability;
const feedbackToken =
provider && capability
? issueFeedbackToken({
requestId: usageRequestId,
teamId: req.auth.team_id,
provider,
capability,
})
: undefined;
return feedbackToken ? { ...item, feedbackToken } : item;
}),
};
if (legacy) {
if (legacyBody && "requests" in legacyBody) return res.json(answer);
const first = answer.results[0];
if (first.error) {
const status = first.error.status ?? 502;
return res.status(status >= 400 && status <= 599 ? status : 502).json({
success: false,
code: first.error.code,
error: first.error.message,
...(first.feedbackToken ? { feedbackToken: first.feedbackToken } : {}),
});
}
return res.json({ success: true, ...first });
}
return res.json({
success: true,
scrape_id: result.scrapeId,
data: { alexandria: answer.results, creditsCost: answer.creditsCost },
});
}