Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add replica-safe SIEM Logging (#4122)

Gergő Móricz28c8d30207fc
diff --git a/apps/api/.env.example b/apps/api/.env.example
index 90fdab093..9554111c9 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -95,6 +95,12 @@ MENU_EXTRACTION_SERVICE_URL=
PROXY_SERVER=
PROXY_USERNAME=
PROXY_PASSWORD=
+
+# Enterprise SIEM Logging delivery. The encryption key must be 32 random bytes
+# encoded as 64 hex characters or base64. Route both Entra and Azure Monitor
+# ingestion calls through the dedicated partner egress proxy.
+SIEM_LOGGING_ENCRYPTION_KEY=
+PARTNER_EGRESS_PROXY_URL=
# set if you'd like to block media requests to save proxy bandwidth
BLOCK_MEDIA=
diff --git a/apps/api/src/__tests__/snips/v2/team-siem-logging.test.ts b/apps/api/src/__tests__/snips/v2/team-siem-logging.test.ts
new file mode 100644
index 000000000..6376ae37f
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/team-siem-logging.test.ts
@@ -0,0 +1,107 @@
+import request from "supertest";
+import {
+ describeIf,
+ idmux,
+ type Identity,
+ TEST_API_URL,
+ TEST_PRODUCTION,
+} from "../lib";
+
+async function getConfig(identity: Identity) {
+ return request(TEST_API_URL)
+ .get("/v2/team/siem")
+ .set("Authorization", `Bearer ${identity.apiKey}`);
+}
+
+async function putConfig(identity: Identity, body: unknown) {
+ return request(TEST_API_URL)
+ .put("/v2/team/siem")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send(body as object);
+}
+
+describeIf(TEST_PRODUCTION)("Team SIEM logging config API", () => {
+ it("rejects a team without the enterprise flag", async () => {
+ const identity = await idmux({ name: "team-siem-logging/no-flag" });
+ const response = await getConfig(identity);
+
+ expect(response.statusCode).toBe(403);
+ expect(response.body.error).toContain("enterprise feature");
+ });
+
+ describeIf(!!process.env.SIEM_LOGGING_ENCRYPTION_KEY)(
+ "with the enterprise flag",
+ () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "team-siem-logging/flagged",
+ flags: { siemLogging: true },
+ });
+ });
+
+ it("stores an Azure destination while keeping the secret write-only", async () => {
+ const response = await putConfig(identity, {
+ enabled: true,
+ destination: {
+ type: "azure_sentinel",
+ tenantId: "00000000-0000-0000-0000-000000000001",
+ clientId: "00000000-0000-0000-0000-000000000002",
+ clientSecret: "never-return-this-value",
+ dceUrl: "https://firecrawl-test.eastus.ingest.monitor.azure.com",
+ dcrImmutableId: "dcr-00000000000000000000000000000000",
+ streamName: "Custom-FirecrawlScrapeActivity",
+ },
+ });
+
+ // A rollout can deploy API code before the out-of-band DDL. Match the
+ // existing enterprise-config snips by making that state explicit.
+ if (response.statusCode === 500) {
+ console.warn(
+ "siem_logging_config table is unavailable; skipping config round-trip",
+ );
+ return;
+ }
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.data).toMatchObject({
+ configured: true,
+ enabled: true,
+ destination: {
+ type: "azure_sentinel",
+ clientSecretConfigured: true,
+ },
+ });
+ expect(JSON.stringify(response.body)).not.toContain(
+ "never-return-this-value",
+ );
+
+ const read = await getConfig(identity);
+ expect(read.statusCode).toBe(200);
+ expect(JSON.stringify(read.body)).not.toContain(
+ "never-return-this-value",
+ );
+ });
+
+ it("rejects non-Azure ingestion endpoints", async () => {
+ const response = await putConfig(identity, {
+ enabled: true,
+ destination: {
+ type: "azure_sentinel",
+ tenantId: "tenant",
+ clientId: "client",
+ clientSecret: "secret",
+ dceUrl: "https://example.com/collect",
+ dcrImmutableId: "dcr-id",
+ streamName: "Custom-FirecrawlScrapeActivity",
+ },
+ });
+
+ expect(response.statusCode).toBe(400);
+ expect(response.body.success).toBe(false);
+ });
+ },
+ );
+});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index e29e1dbac..827e8bf06 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -81,6 +81,12 @@ const configSchema = z.object({
.positive()
.default(6 * 60 * 60),
+ // Organization SIEM logging delivery. The encryption key must decode to
+ // exactly 32 bytes; validation happens when a secret is encrypted/decrypted
+ // so self-hosted deployments that do not use this feature need no key.
+ SIEM_LOGGING_ENCRYPTION_KEY: z.string().optional(),
+ PARTNER_EGRESS_PROXY_URL: z.string().url().optional(),
+
// API Keys & Authentication
BULL_AUTH_KEY: z.string().optional(),
OPENAI_API_KEY: z.string().optional(),
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index d20906b9f..1e69f7bce 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -44,6 +44,8 @@ import {
} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
+import { CrawlDenialError } from "../../lib/error";
export async function batchScrapeController(
req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -108,6 +110,7 @@ export async function batchScrapeController(
let urls: string[] = req.body.urls;
let unnormalizedURLs = preNormalizedBody.urls;
let invalidURLs: string[] | undefined = undefined;
+ const locallyBlockedURLs: string[] = [];
if (req.body.ignoreInvalidURLs) {
invalidURLs = [];
@@ -129,21 +132,39 @@ export async function batchScrapeController(
unnormalizedURLs.push(u);
} else {
invalidURLs.push(u);
+ locallyBlockedURLs.push(nu);
}
} catch (_) {
invalidURLs.push(u);
}
}
} else {
- if (
- req.body.urls?.some((url: string) =>
+ const blockedURLs =
+ req.body.urls?.filter((url: string) =>
isUrlBlocked(url, req.acuc?.flags ?? null, {
team_id: req.auth.team_id,
org_id: req.acuc?.org_id ?? null,
origin: req.body.origin ?? null,
}),
- )
- ) {
+ ) ?? [];
+ if (blockedURLs.length > 0) {
+ locallyBlockedURLs.push(...blockedURLs);
+ emitRejectedScrapeActivityEvents(
+ locallyBlockedURLs.map(url => ({
+ scrapeId: uuidv7(),
+ requestId: id,
+ endpoint: "batch_scrape",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.auditMetadata,
+ url,
+ error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention: zeroDataRetention ?? false,
+ })),
+ );
+ locallyBlockedURLs.length = 0;
if (!res.headersSent) {
return res.status(403).json({
success: false,
@@ -153,6 +174,22 @@ export async function batchScrapeController(
}
}
+ emitRejectedScrapeActivityEvents(
+ locallyBlockedURLs.map(url => ({
+ scrapeId: uuidv7(),
+ requestId: id,
+ endpoint: "batch_scrape",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.auditMetadata,
+ url,
+ error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention: zeroDataRetention ?? false,
+ })),
+ );
+
// Threat protection: reject/report blocked URLs at enqueue time so they
// never consume scrape slots (mirrors the isUrlBlocked handling above).
if (threatProtection.policy) {
@@ -184,6 +221,25 @@ export async function batchScrapeController(
);
});
}
+ emitRejectedScrapeActivityEvents(
+ blocked.map(blockedUrl => ({
+ scrapeId: uuidv7(),
+ requestId: id,
+ endpoint: "batch_scrape",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.auditMetadata,
+ url: blockedUrl.url,
+ error: new UnsafeDomainBlockedError(
+ blockedUrl.url,
+ blockedUrl.decision,
+ ),
+ threatDecisions: [blockedUrl.decision],
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention: zeroDataRetention ?? false,
+ })),
+ );
if (req.body.ignoreInvalidURLs) {
const blockedSet = new Set(blocked.map(x => x.url));
const keptUnnormalized: string[] = [];
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index acdf87c15..4557802d5 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -35,6 +35,7 @@ import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";
+import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -42,6 +43,9 @@ export async function crawlController(
) {
const preNormalizedBody = req.body;
req.body = crawlRequestSchema.parse(req.body);
+ const id = uuidv7();
+ const zeroDataRetention =
+ getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;
const threatProtection = await resolveThreatProtection({
teamId: req.auth.team_id,
@@ -91,6 +95,20 @@ export async function crawlController(
});
}
const error = new UnsafeDomainBlockedError(req.body.url, decision);
+ emitRejectedScrapeActivityEvent({
+ scrapeId: uuidv7(),
+ requestId: id,
+ endpoint: "crawl",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.scrapeOptions?.auditMetadata,
+ url: req.body.url,
+ error,
+ threatDecisions: [decision],
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention: zeroDataRetention ?? false,
+ });
return res.status(403).json({
success: false,
code: error.code,
@@ -112,10 +130,6 @@ export async function crawlController(
});
}
- const zeroDataRetention =
- getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;
-
- const id = uuidv7();
const logger = _logger.child({
crawlId: id,
module: "api/v1",
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index b1b778a31..9b4fbf893 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -30,6 +30,8 @@ import {
import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
+import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
+import { CrawlDenialError } from "../../lib/error";
async function oldExtract(
req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
res: Response<ExtractResponse>,
@@ -111,6 +113,7 @@ export async function extractController(
const selfHosted = config.USE_DB_AUTHENTICATION !== true;
const originalRequest = { ...req.body };
req.body = extractRequestSchema.parse(req.body);
+ const extractId = uuidv7();
if (getScrapeZDR(req.acuc?.flags) === "forced") {
return res.status(400).json({
@@ -129,6 +132,22 @@ export async function extractController(
}),
) ?? [];
+ emitRejectedScrapeActivityEvents(
+ invalidURLs.map(url => ({
+ scrapeId: uuidv7(),
+ requestId: extractId,
+ endpoint: "extract",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.scrapeOptions?.auditMetadata,
+ url,
+ error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention: false,
+ })),
+ );
+
const createdAt = Date.now();
if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) {
@@ -181,6 +200,27 @@ export async function extractController(
});
}
if (blocked.length > 0) {
+ emitRejectedScrapeActivityEvents(
+ blocked
+ .filter(blockedUrl => !invalidURLs.includes(blockedUrl.url))
+ .map(blockedUrl => ({
+ scrapeId: uuidv7(),
+ requestId: extractId,
+ endpoint: "extract",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.scrapeOptions?.auditMetadata,
+ url: blockedUrl.url,
+ error: new UnsafeDomainBlockedError(
+ blockedUrl.url,
+ blockedUrl.decision,
+ ),
+ threatDecisions: [blockedUrl.decision],
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention: false,
+ })),
+ );
if (req.body.ignoreInvalidURLs) {
invalidURLs.push(...blocked.map(x => x.url));
} else {
@@ -195,8 +235,6 @@ export async function extractController(
}
}
- const extractId = uuidv7();
-
_logger.info("Extract starting...", {
request: req.body,
originalRequest,
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index b4db0a1bc..3600d5f5b 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -20,6 +20,7 @@ import { BrandingProfile } from "../../types/branding";
import { ProductProfile } from "../../types/product";
import { MenuProfile } from "../../types/menu";
import { threatProtectionOverrideSchema } from "../../lib/threat-protection/config";
+import { auditMetadataSchema } from "../../lib/siem-logging/types";
type Format =
| "markdown"
@@ -532,6 +533,7 @@ const baseScrapeOptions = z.strictObject({
// Enterprise: per-request field-level override of the org's threat
// protection policy. Gated on the team flag + org config (checkPermissions).
threatProtection: threatProtectionOverrideSchema.optional(),
+ auditMetadata: auditMetadataSchema.optional(),
// @deprecated
__experimental_cache: z.boolean().prefault(false).optional(),
__searchPreviewToken: z.string().optional(),
@@ -978,6 +980,7 @@ const mapRequestSchemaBase = crawlerOptions
location: locationSchema,
headers: z.record(z.string(), z.string()).optional(),
threatProtection: threatProtectionOverrideSchema.optional(),
+ auditMetadata: auditMetadataSchema.optional(),
});
export const mapRequestSchema = mapRequestSchemaBase.strict();
@@ -1275,6 +1278,7 @@ export type TeamFlags = {
ignoreRobots?: "disabled" | "allowed" | "forced";
customRobotsAgent?: "disabled" | "allowed";
threatProtection?: "disabled" | "allowed" | "forced";
+ siemLogging?: boolean;
unblockedDomains?: string[];
forceZDR?: boolean;
allowZDR?: boolean;
diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.ts
index 9cd86ef7b..86ee7f346 100644
--- a/apps/api/src/controllers/v2/agent.ts
+++ b/apps/api/src/controllers/v2/agent.ts
@@ -18,6 +18,7 @@ import {
import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
+import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
export async function agentController(
req: RequestWithAuth<{}, AgentResponse, AgentRequest>,
@@ -97,6 +98,25 @@ export async function agentController(
}
const first = blocked[0];
const error = new UnsafeDomainBlockedError(first.url, first.decision);
+ emitRejectedScrapeActivityEvents(
+ blocked.map(blockedUrl => ({
+ scrapeId: uuidv7(),
+ requestId: agentId,
+ endpoint: "agent",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.auditMetadata,
+ url: blockedUrl.url,
+ error: new UnsafeDomainBlockedError(
+ blockedUrl.url,
+ blockedUrl.decision,
+ ),
+ threatDecisions: [blockedUrl.decision],
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention: false,
+ })),
+ );
return res.status(403).json({
success: false,
code: error.code,
@@ -158,6 +178,7 @@ export async function agentController(
strictConstrainToURLs: req.body.strictConstrainToURLs ?? undefined,
webhook: req.body.webhook ?? undefined,
model: req.body.model,
+ auditMetadata: req.body.auditMetadata,
}),
},
);
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index d393df0e5..e4ac42355 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -45,6 +45,8 @@ import {
import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
+import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
+import { CrawlDenialError } from "../../lib/error";
export async function batchScrapeController(
req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -129,6 +131,7 @@ export async function batchScrapeController(
let urls: string[] = req.body.urls;
let unnormalizedURLs = preNormalizedBody.urls;
let invalidURLs: string[] | undefined = undefined;
+ const locallyBlockedURLs: string[] = [];
if (req.body.ignoreInvalidURLs) {
invalidURLs = [];
@@ -150,21 +153,39 @@ export async function batchScrapeController(
unnormalizedURLs.push(u);
} else {
invalidURLs.push(u);
+ locallyBlockedURLs.push(nu);
}
} catch (_) {
invalidURLs.push(u);
}
}
} else {
- if (
- req.body.urls?.some((url: string) =>
+ const blockedURLs =
+ req.body.urls?.filter((url: string) =>
isUrlBlocked(url, req.acuc?.flags ?? null, {
team_id: req.auth.team_id,
org_id: req.acuc?.org_id ?? null,
origin: req.body.origin ?? null,
}),
- )
- ) {
+ ) ?? [];
+ if (blockedURLs.length > 0) {
+ locallyBlockedURLs.push(...blockedURLs);
+ emitRejectedScrapeActivityEvents(
+ locallyBlockedURLs.map(url => ({
+ scrapeId: uuidv7(),
+ requestId: req.body.__agentInterop?.requestId ?? id,
+ endpoint: req.body.__agentInterop ? "agent" : "batch_scrape",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.auditMetadata,
+ url,
+ error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention,
+ })),
+ );
+ locallyBlockedURLs.length = 0;
if (!res.headersSent) {
return res.status(403).json({
success: false,
@@ -174,6 +195,22 @@ export async function batchScrapeController(
}
}
+ emitRejectedScrapeActivityEvents(
+ locallyBlockedURLs.map(url => ({
+ scrapeId: uuidv7(),
+ requestId: req.body.__agentInterop?.requestId ?? id,
+ endpoint: req.body.__agentInterop ? "agent" : "batch_scrape",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.auditMetadata,
+ url,
+ error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention,
+ })),
+ );
+
// Threat protection: reject/report blocked URLs at enqueue time so they
// never consume scrape slots. Mirrors the isUrlBlocked handling above:
// with ignoreInvalidURLs they are reported in invalidURLs; without it the
@@ -212,6 +249,25 @@ export async function batchScrapeController(
);
});
}
+ emitRejectedScrapeActivityEvents(
+ blocked.map(blockedUrl => ({
+ scrapeId: uuidv7(),
+ requestId: req.body.__agentInterop?.requestId ?? id,
+ endpoint: req.body.__agentInterop ? "agent" : "batch_scrape",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.auditMetadata,
+ url: blockedUrl.url,
+ error: new UnsafeDomainBlockedError(
+ blockedUrl.url,
+ blockedUrl.decision,
+ ),
+ threatDecisions: [blockedUrl.decision],
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention,
+ })),
+ );
if (req.body.ignoreInvalidURLs) {
const blockedSet = new Set(blocked.map(x => x.url));
const keptUnnormalized: string[] = [];
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 0ed8a5029..090aa94b2 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -37,6 +37,7 @@ import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";
+import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -44,6 +45,9 @@ export async function crawlController(
) {
const preNormalizedBody = req.body;
req.body = crawlRequestSchema.parse(req.body);
+ const id = uuidv7();
+ const zeroDataRetention =
+ getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;
const threatProtection = await resolveThreatProtection({
teamId: req.auth.team_id,
@@ -95,6 +99,20 @@ export async function crawlController(
});
}
const error = new UnsafeDomainBlockedError(req.body.url, decision);
+ emitRejectedScrapeActivityEvent({
+ scrapeId: uuidv7(),
+ requestId: id,
+ endpoint: "crawl",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.scrapeOptions?.auditMetadata,
+ url: req.body.url,
+ error,
+ threatDecisions: [decision],
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention: zeroDataRetention ?? false,
+ });
return res.status(403).json({
success: false,
code: error.code,
@@ -116,10 +134,6 @@ export async function crawlController(
});
}
- const zeroDataRetention =
- getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;
-
- const id = uuidv7();
const logger = _logger.child({
crawlId: id,
module: "api/v2",
diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.ts
index fb21af95f..5691cf498 100644
--- a/apps/api/src/controllers/v2/extract.ts
+++ b/apps/api/src/controllers/v2/extract.ts
@@ -21,6 +21,8 @@ import {
import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
+import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
+import { CrawlDenialError } from "../../lib/error";
/**
* Extracts data from the provided URLs based on the request parameters.
@@ -72,6 +74,22 @@ export async function extractController(
}),
) ?? [];
+ emitRejectedScrapeActivityEvents(
+ invalidURLs.map(url => ({
+ scrapeId: uuidv7(),
+ requestId: extractId,
+ endpoint: "extract",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.scrapeOptions?.auditMetadata,
+ url,
+ error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention: false,
+ })),
+ );
+
if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) {
if (!res.headersSent) {
return res.status(403).json({
@@ -122,6 +140,27 @@ export async function extractController(
});
}
if (blocked.length > 0) {
+ emitRejectedScrapeActivityEvents(
+ blocked
+ .filter(blockedUrl => !invalidURLs.includes(blockedUrl.url))
+ .map(blockedUrl => ({
+ scrapeId: uuidv7(),
+ requestId: extractId,
+ endpoint: "extract",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ auditMetadata: req.body.scrapeOptions?.auditMetadata,
+ url: blockedUrl.url,
+ error: new UnsafeDomainBlockedError(
+ blockedUrl.url,
+ blockedUrl.decision,
+ ),
+ threatDecisions: [blockedUrl.decision],
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention: false,
+ })),
+ );
if (req.body.ignoreInvalidURLs) {
invalidURLs.push(...blocked.map(x => x.url));
} else {
diff --git a/apps/api/src/controllers/v2/siem-logging-schema.test.ts b/apps/api/src/controllers/v2/siem-logging-schema.test.ts
new file mode 100644
index 000000000..cc5fe808e
--- /dev/null
+++ b/apps/api/src/controllers/v2/siem-logging-schema.test.ts
@@ -0,0 +1,171 @@
+import { describe, expect, it } from "vitest";
+import {
+ agentRequestSchema,
+ batchScrapeRequestSchema,
+ crawlRequestSchema,
+ extractRequestSchema,
+ mapRequestSchema,
+ scrapeRequestSchema,
+ searchRequestSchema,
+} from "./types";
+import {
+ batchScrapeRequestSchema as batchScrapeRequestSchemaV1,
+ crawlRequestSchema as crawlRequestSchemaV1,
+ extractRequestSchema as extractRequestSchemaV1,
+ mapRequestSchema as mapRequestSchemaV1,
+ scrapeRequestSchema as scrapeRequestSchemaV1,
+ searchRequestSchema as searchRequestSchemaV1,
+} from "../v1/types";
+import { siemLoggingConfigInputSchema } from "../../lib/siem-logging/types";
+
+const auditMetadata = {
+ username: "[email protected]",
+ session: "session-123",
+};
+
+describe("auditMetadata request validation", () => {
+ it("accepts audit metadata across every scrape-producing request schema", () => {
+ expect(
+ scrapeRequestSchema.parse({
+ url: "https://example.com",
+ auditMetadata,
+ }).auditMetadata,
+ ).toEqual(auditMetadata);
+ expect(
+ batchScrapeRequestSchema.parse({
+ urls: ["https://example.com"],
+ auditMetadata,
+ }).auditMetadata,
+ ).toEqual(auditMetadata);
+ expect(
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ scrapeOptions: { auditMetadata },
+ }).scrapeOptions.auditMetadata,
+ ).toEqual(auditMetadata);
+ expect(
+ searchRequestSchema.parse({
+ query: "example",
+ scrapeOptions: { auditMetadata },
+ }).scrapeOptions?.auditMetadata,
+ ).toEqual(auditMetadata);
+ expect(
+ extractRequestSchema.parse({
+ urls: ["https://example.com"],
+ scrapeOptions: { auditMetadata },
+ }).scrapeOptions?.auditMetadata,
+ ).toEqual(auditMetadata);
+ expect(
+ mapRequestSchema.parse({
+ url: "https://example.com",
+ auditMetadata,
+ }).auditMetadata,
+ ).toEqual(auditMetadata);
+ expect(
+ agentRequestSchema.parse({
+ prompt: "Find the pricing page",
+ auditMetadata,
+ }).auditMetadata,
+ ).toEqual(auditMetadata);
+ });
+
+ it("rejects oversized metadata", () => {
+ const tooManyFields = Object.fromEntries(
+ Array.from({ length: 33 }, (_, index) => [`field-${index}`, "value"]),
+ );
+ expect(() =>
+ scrapeRequestSchema.parse({
+ url: "https://example.com",
+ auditMetadata: tooManyFields,
+ }),
+ ).toThrow("at most 32 fields");
+ });
+
+ it("accepts audit metadata on the corresponding v1 request schemas", () => {
+ expect(
+ scrapeRequestSchemaV1.parse({
+ url: "https://example.com",
+ auditMetadata,
+ }).auditMetadata,
+ ).toEqual(auditMetadata);
+ expect(
+ batchScrapeRequestSchemaV1.parse({
+ urls: ["https://example.com"],
+ auditMetadata,
+ }).auditMetadata,
+ ).toEqual(auditMetadata);
+ expect(
+ crawlRequestSchemaV1.parse({
+ url: "https://example.com",
+ scrapeOptions: { auditMetadata },
+ }).scrapeOptions.auditMetadata,
+ ).toEqual(auditMetadata);
+ expect(
+ searchRequestSchemaV1.parse({
+ query: "example",
+ scrapeOptions: { auditMetadata },
+ }).scrapeOptions.auditMetadata,
+ ).toEqual(auditMetadata);
+ expect(
+ extractRequestSchemaV1.parse({
+ urls: ["https://example.com"],
+ scrapeOptions: { auditMetadata },
+ }).scrapeOptions?.auditMetadata,
+ ).toEqual(auditMetadata);
+ expect(
+ mapRequestSchemaV1.parse({
+ url: "https://example.com",
+ auditMetadata,
+ }).auditMetadata,
+ ).toEqual(auditMetadata);
+ });
+});
+
+describe("SIEM destination validation", () => {
+ const destination = {
+ type: "azure_sentinel" as const,
+ tenantId: "tenant",
+ clientId: "client",
+ clientSecret: "secret",
+ dceUrl: "https://example.eastus.ingest.monitor.azure.com",
+ dcrImmutableId: "dcr-id",
+ streamName: "Custom-FirecrawlScrapeActivity",
+ };
+
+ it("trims credential and destination fields", () => {
+ const parsed = siemLoggingConfigInputSchema.parse({
+ enabled: true,
+ destination: {
+ ...destination,
+ tenantId: " tenant ",
+ clientId: " client ",
+ clientSecret: " secret ",
+ },
+ });
+
+ expect(parsed.destination).toMatchObject({
+ tenantId: "tenant",
+ clientId: "client",
+ clientSecret: "secret",
+ });
+ });
+
+ it("rejects whitespace-only credentials and endpoint userinfo", () => {
+ expect(() =>
+ siemLoggingConfigInputSchema.parse({
+ enabled: true,
+ destination: { ...destination, clientSecret: " " },
+ }),
+ ).toThrow();
+ expect(() =>
+ siemLoggingConfigInputSchema.parse({
+ enabled: true,
+ destination: {
+ ...destination,
+ dceUrl:
+ "https://user:[email protected]",
+ },
+ }),
+ ).toThrow("credential-free");
+ });
+});
diff --git a/apps/api/src/controllers/v2/team-siem-logging.ts b/apps/api/src/controllers/v2/team-siem-logging.ts
new file mode 100644
index 000000000..b69a141e2
--- /dev/null
+++ b/apps/api/src/controllers/v2/team-siem-logging.ts
@@ -0,0 +1,176 @@
+import { randomUUID } from "crypto";
+import type { Response } from "express";
+import type { RequestWithAuth } from "./types";
+import { logger as _logger } from "../../lib/logger";
+import { sendAzureSentinelEvents } from "../../lib/siem-logging/azure-sentinel";
+import {
+ getOrgSiemLoggingConfig,
+ upsertOrgSiemLoggingConfig,
+} from "../../lib/siem-logging/store";
+import {
+ SiemDeliveryError,
+ siemLoggingConfigInputSchema,
+ type OrgSiemLoggingConfig,
+ type ScrapeActivityEvent,
+} from "../../lib/siem-logging/types";
+import { getOrgIdForTeam } from "../../lib/threat-protection/store";
+
+const logger = _logger.child({ module: "team-siem-logging" });
+const SUPPORT_EMAIL = "[email protected]";
+
+function rejectWithoutFlag(
+ req: RequestWithAuth<any, any, any>,
+ res: Response,
+): boolean {
+ if (req.acuc?.flags?.siemLogging !== true) {
+ res.status(403).json({
+ success: false,
+ error: `SIEM Logging is an enterprise feature and is not enabled for your team. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,
+ });
+ return true;
+ }
+ return false;
+}
+
+async function resolveOrgId(
+ req: RequestWithAuth<any, any, any>,
+ res: Response,
+): Promise<string | null> {
+ const orgId = req.auth.org_id ?? (await getOrgIdForTeam(req.auth.team_id));
+ if (orgId) return orgId;
+ logger.error("Failed to resolve org for SIEM configuration", {
+ teamId: req.auth.team_id,
+ });
+ res.status(500).json({
+ success: false,
+ error: "Failed to resolve the organization for this team.",
+ });
+ return null;
+}
+
+function serializeConfig(config: OrgSiemLoggingConfig | null) {
+ if (!config) return { configured: false, enabled: false, destination: null };
+ const { clientSecret: _clientSecret, ...destination } = config.destination;
+ return {
+ configured: true,
+ enabled: config.enabled,
+ destination: {
+ ...destination,
+ clientSecretConfigured: true,
+ },
+ updatedAt: config.updatedAt,
+ };
+}
+
+export async function getTeamSiemLoggingController(
+ req: RequestWithAuth,
+ res: Response,
+): Promise<void> {
+ if (rejectWithoutFlag(req, res)) return;
+ const orgId = await resolveOrgId(req, res);
+ if (!orgId) return;
+ const config = await getOrgSiemLoggingConfig(orgId);
+ res.status(200).json({ success: true, data: serializeConfig(config) });
+}
+
+export async function putTeamSiemLoggingController(
+ req: RequestWithAuth<{}, any, unknown>,
+ res: Response,
+): Promise<void> {
+ if (rejectWithoutFlag(req, res)) return;
+ const input = siemLoggingConfigInputSchema.parse(req.body);
+ const orgId = await resolveOrgId(req, res);
+ if (!orgId) return;
+
+ try {
+ const updated = await upsertOrgSiemLoggingConfig(orgId, input);
+ logger.info("SIEM logging configuration updated", {
+ teamId: req.auth.team_id,
+ orgId,
+ enabled: updated.enabled,
+ destinationType: updated.destination.type,
+ streamName: updated.destination.streamName,
+ });
+ res.status(200).json({
+ success: true,
+ data: serializeConfig(updated),
+ });
+ } catch (error) {
+ if (
+ error instanceof Error &&
+ error.message === "clientSecret is required when SIEM is first configured"
+ ) {
+ res.status(400).json({ success: false, error: error.message });
+ return;
+ }
+ throw error;
+ }
+}
+
+export async function testTeamSiemLoggingController(
+ req: RequestWithAuth,
+ res: Response,
+): Promise<void> {
+ if (rejectWithoutFlag(req, res)) return;
+ const orgId = await resolveOrgId(req, res);
+ if (!orgId) return;
+ const siemConfig = await getOrgSiemLoggingConfig(orgId);
+ if (!siemConfig?.enabled) {
+ res.status(409).json({
+ success: false,
+ error: "SIEM Logging is not configured and enabled.",
+ });
+ return;
+ }
+
+ const now = new Date().toISOString();
+ const id = randomUUID();
+ const event: ScrapeActivityEvent = {
+ schema_version: 1,
+ event_type: "scrape_activity",
+ scrape_id: id,
+ request_id: id,
+ endpoint: "scrape",
+ team_id: req.auth.team_id,
+ org_id: orgId,
+ api_key: {
+ id: req.acuc?.api_key_id == null ? null : String(req.acuc.api_key_id),
+ name: null,
+ },
+ audit_metadata: { synthetic: "true" },
+ started_at: now,
+ completed_at: now,
+ url: "https://example.com/siem-test",
+ domain: "example.com",
+ http_method: "GET",
+ http_status: 200,
+ result: "success",
+ error: null,
+ origin: "siem-test",
+ integration: null,
+ zero_data_retention: true,
+ };
+
+ try {
+ await sendAzureSentinelEvents(siemConfig.destination, [event]);
+ res.status(200).json({ success: true, data: { delivered: true } });
+ } catch (error) {
+ if (error instanceof SiemDeliveryError) {
+ const status =
+ error.kind === "invalid_credentials" ||
+ error.kind === "schema_rejection"
+ ? 400
+ : 502;
+ res.status(status).json({
+ success: false,
+ error: {
+ type: error.kind,
+ message: error.message,
+ statusCode: error.statusCode ?? null,
+ },
+ });
+ return;
+ }
+ throw error;
+ }
+}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index e73875a29..fe104e09d 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -29,6 +29,7 @@ import { BrandingProfile } from "../../types/branding";
import { ProductProfile } from "../../types/product";
import { MenuProfile } from "../../types/menu";
import { threatProtectionOverrideSchema } from "../../lib/threat-protection/config";
+import { auditMetadataSchema } from "../../lib/siem-logging/types";
// Base URL schema with common validation logic
export const URL = z.preprocess(
@@ -696,6 +697,7 @@ const baseScrapeOptions = z.strictObject({
// Enterprise: per-request field-level override of the org's threat
// protection policy. Gated on the team flag + org config (checkPermissions).
threatProtection: threatProtectionOverrideSchema.optional(),
+ auditMetadata: auditMetadataSchema.optional(),
profile: z
.object({
@@ -946,6 +948,7 @@ export const agentRequestSchema = z.strictObject({
overrideWhitelist: z.string().optional(),
model: z.enum(["spark-1-pro", "spark-1-mini"]).default("spark-1-pro"),
threatProtection: threatProtectionOverrideSchema.optional(),
+ auditMetadata: auditMetadataSchema.optional(),
});
export type AgentRequest = z.infer<typeof agentRequestSchema>;
@@ -1198,6 +1201,7 @@ const mapRequestSchemaBase = crawlerOptions
location: locationSchema,
headers: z.record(z.string(), z.string()).optional(),
threatProtection: threatProtectionOverrideSchema.optional(),
+ auditMetadata: auditMetadataSchema.optional(),
});
export const mapRequestSchema = strictWithMessage(mapRequestSchemaBase);
@@ -1543,6 +1547,7 @@ export type TeamFlags = {
ignoreRobots?: "disabled" | "allowed" | "forced";
customRobotsAgent?: "disabled" | "allowed";
threatProtection?: "disabled" | "allowed" | "forced";
+ siemLogging?: boolean;
unblockedDomains?: string[];
forceZDR?: boolean;
allowZDR?: boolean;
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
index 5acb8afb4..af0485a39 100644
--- a/apps/api/src/db/schema/public.ts
+++ b/apps/api/src/db/schema/public.ts
@@ -703,6 +703,16 @@ export const threat_protection_config = pgTable("threat_protection_config", {
updated_at: ts("updated_at").notNull().defaultNow(),
});
+export const siem_logging_config = pgTable("siem_logging_config", {
+ id: uuid("id").notNull().defaultRandom(),
+ org_id: uuid("org_id").notNull().unique(),
+ enabled: boolean("enabled").notNull().default(false),
+ destination: jsonb("destination").notNull().default({}),
+ secret_ciphertext: text("secret_ciphertext").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+});
+
export const user_notifications = pgTable("user_notifications", {
id: uuid("id").notNull().defaultRandom(),
team_id: uuid("team_id"),
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index ed9b5d50a..177b7f7af 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -11,6 +11,7 @@ import {
getDeepResearchQueue,
getBillingQueue,
getPrecrawlQueue,
+ getSiemLoggingQueue,
} from "./services/queue-service";
import { v0Router } from "./routes/v0";
import os from "os";
@@ -103,6 +104,7 @@ const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
new BullMQAdapter(getDeepResearchQueue()),
new BullMQAdapter(getBillingQueue()),
new BullMQAdapter(getPrecrawlQueue()),
+ new BullMQAdapter(getSiemLoggingQueue()),
],
serverAdapter: serverAdapter,
});
diff --git a/apps/api/src/lib/siem-logging/azure-sentinel.test.ts b/apps/api/src/lib/siem-logging/azure-sentinel.test.ts
new file mode 100644
index 000000000..63e6145d3
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/azure-sentinel.test.ts
@@ -0,0 +1,331 @@
+import { gunzipSync } from "zlib";
+import { Response } from "undici";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { AzureSentinelDestination, ScrapeActivityEvent } from "./types";
+import {
+ buildCompressedBatches,
+ clearAzureTokenCache,
+ sendAzureSentinelEvents,
+ type SenderDependencies,
+} from "./azure-sentinel";
+
+const destination: AzureSentinelDestination = {
+ type: "azure_sentinel",
+ tenantId: "tenant",
+ clientId: "client",
+ clientSecret: "secret",
+ dceUrl: "https://example.eastus.ingest.monitor.azure.com",
+ dcrImmutableId: "dcr-immutable",
+ streamName: "Custom-FirecrawlScrapeActivity",
+};
+
+function event(id = "scrape-id"): ScrapeActivityEvent {
+ return {
+ schema_version: 1,
+ event_type: "scrape_activity",
+ scrape_id: id,
+ request_id: "request-id",
+ endpoint: "scrape",
+ team_id: "team-id",
+ org_id: "org-id",
+ api_key: { id: "1", name: "default" },
+ audit_metadata: { username: "[email protected]" },
+ started_at: "2026-07-24T00:00:00.000Z",
+ completed_at: "2026-07-24T00:00:01.000Z",
+ url: "https://example.com",
+ domain: "example.com",
+ http_method: "GET",
+ http_status: 200,
+ result: "success",
+ error: null,
+ origin: "api",
+ integration: null,
+ zero_data_retention: true,
+ };
+}
+
+afterEach(() => clearAzureTokenCache());
+
+describe("Azure Sentinel sender", () => {
+ it("uses client credentials and sends a gzip JSON array", async () => {
+ const calls: Array<{ url: string; init: any }> = [];
+ const fetchImpl: SenderDependencies["fetchImpl"] = async (url, init) => {
+ calls.push({ url, init });
+ if (url.includes("/token")) {
+ return new Response(
+ JSON.stringify({ access_token: "access-token", expires_in: 3600 }),
+ { status: 200, headers: { "content-type": "application/json" } },
+ );
+ }
+ return new Response(null, { status: 204 });
+ };
+
+ await sendAzureSentinelEvents(destination, [event()], {
+ fetchImpl,
+ tokenUrl: "https://login.example.test/token",
+ });
+
+ expect(calls).toHaveLength(2);
+ expect(calls[0].init.body).toContain("client_secret=secret");
+ expect(calls[0].init.body).toContain(
+ "scope=https%3A%2F%2Fmonitor.azure.com%2F%2F.default",
+ );
+ expect(calls[1].url).toContain(
+ "/dataCollectionRules/dcr-immutable/streams/Custom-FirecrawlScrapeActivity?api-version=2023-01-01",
+ );
+ expect(calls[1].init.headers["content-encoding"]).toBe("gzip");
+ expect(calls[1].init.headers.authorization).toBe("Bearer access-token");
+ expect(JSON.parse(gunzipSync(calls[1].init.body).toString("utf8"))).toEqual(
+ [event()],
+ );
+ });
+
+ it("distinguishes invalid credentials from a DCR schema rejection", async () => {
+ const invalidCredentials: SenderDependencies["fetchImpl"] = async () =>
+ new Response("invalid_client", { status: 400 });
+ await expect(
+ sendAzureSentinelEvents(destination, [event()], {
+ fetchImpl: invalidCredentials,
+ tokenUrl: "https://login.example.test/token",
+ }),
+ ).rejects.toMatchObject({
+ kind: "invalid_credentials",
+ statusCode: 400,
+ });
+
+ clearAzureTokenCache();
+ const schemaRejection: SenderDependencies["fetchImpl"] = async url =>
+ url.includes("/token")
+ ? new Response(
+ JSON.stringify({ access_token: "token", expires_in: 3600 }),
+ { status: 200, headers: { "content-type": "application/json" } },
+ )
+ : new Response("InvalidOutputTable", { status: 400 });
+ await expect(
+ sendAzureSentinelEvents(destination, [event()], {
+ fetchImpl: schemaRejection,
+ tokenUrl: "https://login.example.test/token",
+ }),
+ ).rejects.toMatchObject({
+ kind: "schema_rejection",
+ statusCode: 400,
+ });
+ });
+
+ it("respects Retry-After and makes three total attempts on 429", async () => {
+ let ingestionAttempts = 0;
+ const sleep = vi.fn(async () => {});
+ const fetchImpl: SenderDependencies["fetchImpl"] = async url => {
+ if (url.includes("/token")) {
+ return new Response(
+ JSON.stringify({ access_token: "token", expires_in: 3600 }),
+ { status: 200, headers: { "content-type": "application/json" } },
+ );
+ }
+ ingestionAttempts++;
+ return ingestionAttempts < 3
+ ? new Response("slow down", {
+ status: 429,
+ headers: { "retry-after": "2" },
+ })
+ : new Response(null, { status: 204 });
+ };
+
+ await sendAzureSentinelEvents(destination, [event()], {
+ fetchImpl,
+ sleep,
+ tokenUrl: "https://login.example.test/token",
+ });
+
+ expect(ingestionAttempts).toBe(3);
+ expect(sleep).toHaveBeenNthCalledWith(1, 2000);
+ expect(sleep).toHaveBeenNthCalledWith(2, 2000);
+ });
+
+ it("caps Retry-After and releases retry response bodies", async () => {
+ const sleep = vi.fn(async () => {});
+ const retryResponses: Response[] = [];
+ let ingestionAttempts = 0;
+ const fetchImpl: SenderDependencies["fetchImpl"] = async url => {
+ if (url.includes("/token")) {
+ return new Response(
+ JSON.stringify({ access_token: "token", expires_in: 3600 }),
+ { status: 200, headers: { "content-type": "application/json" } },
+ );
+ }
+ ingestionAttempts++;
+ if (ingestionAttempts < 3) {
+ const response = new Response("try later", {
+ status: 503,
+ headers: { "retry-after": "86400" },
+ });
+ retryResponses.push(response);
+ return response;
+ }
+ return new Response(null, { status: 204 });
+ };
+
+ await sendAzureSentinelEvents(destination, [event()], {
+ fetchImpl,
+ sleep,
+ tokenUrl: "https://login.example.test/token",
+ });
+
+ expect(sleep).toHaveBeenNthCalledWith(1, 30_000);
+ expect(sleep).toHaveBeenNthCalledWith(2, 30_000);
+ expect(retryResponses.every(response => response.bodyUsed)).toBe(true);
+ });
+
+ it("exposes the capped Retry-After delay after the final attempt", async () => {
+ const sleep = vi.fn(async () => {});
+ const fetchImpl: SenderDependencies["fetchImpl"] = async url => {
+ if (url.includes("/token")) {
+ return new Response(
+ JSON.stringify({ access_token: "token", expires_in: 3600 }),
+ { status: 200 },
+ );
+ }
+ return new Response("try later", {
+ status: 429,
+ headers: { "retry-after": "86400" },
+ });
+ };
+
+ await expect(
+ sendAzureSentinelEvents(destination, [event()], {
+ fetchImpl,
+ sleep,
+ tokenUrl: "https://login.example.test/token",
+ }),
+ ).rejects.toMatchObject({
+ kind: "rate_limited",
+ retryAfterMs: 30_000,
+ });
+ });
+
+ it("retries transient network failures three times", async () => {
+ let ingestionAttempts = 0;
+ const sleep = vi.fn(async () => {});
+ const fetchImpl: SenderDependencies["fetchImpl"] = async url => {
+ if (url.includes("/token")) {
+ return new Response(
+ JSON.stringify({ access_token: "token", expires_in: 3600 }),
+ { status: 200, headers: { "content-type": "application/json" } },
+ );
+ }
+ ingestionAttempts++;
+ throw new Error("connection reset");
+ };
+
+ await expect(
+ sendAzureSentinelEvents(destination, [event()], {
+ fetchImpl,
+ sleep,
+ tokenUrl: "https://login.example.test/token",
+ }),
+ ).rejects.toMatchObject({ kind: "delivery_error" });
+
+ expect(ingestionAttempts).toBe(3);
+ expect(sleep).toHaveBeenNthCalledWith(1, 250);
+ expect(sleep).toHaveBeenNthCalledWith(2, 500);
+ });
+
+ it("classifies ingestion authorization failures as invalid credentials", async () => {
+ const fetchImpl: SenderDependencies["fetchImpl"] = async url =>
+ url.includes("/token")
+ ? new Response(
+ JSON.stringify({ access_token: "token", expires_in: 3600 }),
+ { status: 200, headers: { "content-type": "application/json" } },
+ )
+ : new Response("Forbidden", { status: 403 });
+
+ await expect(
+ sendAzureSentinelEvents(destination, [event()], {
+ fetchImpl,
+ tokenUrl: "https://login.example.test/token",
+ }),
+ ).rejects.toMatchObject({
+ kind: "invalid_credentials",
+ statusCode: 403,
+ });
+ });
+
+ it("refreshes a rejected cached token once", async () => {
+ let tokenRequests = 0;
+ const ingestionTokens: string[] = [];
+ const fetchImpl: SenderDependencies["fetchImpl"] = async (url, init) => {
+ if (url.includes("/token")) {
+ tokenRequests++;
+ return new Response(
+ JSON.stringify({
+ access_token: `token-${tokenRequests}`,
+ expires_in: 3600,
+ }),
+ { status: 200, headers: { "content-type": "application/json" } },
+ );
+ }
+ ingestionTokens.push(init.headers.authorization);
+ return init.headers.authorization === "Bearer token-1"
+ ? new Response("revoked", { status: 401 })
+ : new Response(null, { status: 204 });
+ };
+
+ await sendAzureSentinelEvents(destination, [event()], {
+ fetchImpl,
+ tokenUrl: "https://login.example.test/token",
+ });
+
+ expect(tokenRequests).toBe(2);
+ expect(ingestionTokens).toEqual(["Bearer token-1", "Bearer token-2"]);
+ });
+
+ it("does not reuse a token after client-secret rotation", async () => {
+ let tokenRequests = 0;
+ const fetchImpl: SenderDependencies["fetchImpl"] = async url => {
+ if (url.includes("/token")) {
+ tokenRequests++;
+ return new Response(
+ JSON.stringify({
+ access_token: `token-${tokenRequests}`,
+ expires_in: 3600,
+ }),
+ { status: 200, headers: { "content-type": "application/json" } },
+ );
+ }
+ return new Response(null, { status: 204 });
+ };
+
+ await sendAzureSentinelEvents(destination, [event("first")], {
+ fetchImpl,
+ tokenUrl: "https://login.example.test/token",
+ });
+ await sendAzureSentinelEvents(
+ { ...destination, clientSecret: "rotated-secret" },
+ [event("second")],
+ { fetchImpl, tokenUrl: "https://login.example.test/token" },
+ );
+
+ expect(tokenRequests).toBe(2);
+ });
+
+ it("splits compressed payloads at the configured byte ceiling", () => {
+ const noisyEvents = Array.from({ length: 8 }, (_, index) =>
+ event(
+ `${index}-${Array.from({ length: 30 }, () => crypto.randomUUID()).join("")}`,
+ ),
+ );
+ const batches = buildCompressedBatches(noisyEvents, 1200);
+ expect(Array.isArray(batches)).toBe(false);
+ const compressedBodies = [...batches];
+
+ expect(compressedBodies.length).toBeGreaterThan(1);
+ expect(compressedBodies.every(batch => batch.byteLength <= 1200)).toBe(
+ true,
+ );
+ expect(
+ compressedBodies.flatMap(batch =>
+ JSON.parse(gunzipSync(batch).toString("utf8")),
+ ),
+ ).toHaveLength(noisyEvents.length);
+ });
+});
diff --git a/apps/api/src/lib/siem-logging/azure-sentinel.ts b/apps/api/src/lib/siem-logging/azure-sentinel.ts
new file mode 100644
index 000000000..8bef81489
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/azure-sentinel.ts
@@ -0,0 +1,364 @@
+import { gzipSync } from "zlib";
+import { createHash, randomUUID } from "crypto";
+import {
+ fetch as undiciFetch,
+ ProxyAgent,
+ type Dispatcher,
+ type Response,
+} from "undici";
+import { config } from "../../config";
+import { siemLoggingDeliveryBatchesTotal } from "./metrics";
+import type { AzureSentinelDestination, ScrapeActivityEvent } from "./types";
+import { SiemDeliveryError } from "./types";
+
+const MAX_COMPRESSED_BODY_BYTES = 1_000_000;
+const MAX_ATTEMPTS = 3;
+const MAX_RETRY_DELAY_MS = 30_000;
+const MAX_TOKEN_CACHE_ENTRIES = 1000;
+const REQUEST_TIMEOUT_MS = 15_000;
+
+type FetchImpl = (
+ input: string,
+ init: {
+ method: string;
+ headers: Record<string, string>;
+ body?: string | Buffer;
+ dispatcher?: Dispatcher;
+ signal?: AbortSignal;
+ },
+) => Promise<Response>;
+
+export interface SenderDependencies {
+ fetchImpl?: FetchImpl;
+ sleep?: (ms: number) => Promise<void>;
+ dispatcher?: Dispatcher;
+ tokenUrl?: string;
+}
+
+interface CachedToken {
+ token: string;
+ expiresAt: number;
+}
+
+const tokenCache = new Map<string, CachedToken>();
+let proxyDispatcher: ProxyAgent | undefined;
+let proxyDispatcherUrl: string | undefined;
+
+function getDispatcher(deps: SenderDependencies): Dispatcher | undefined {
+ if (deps.dispatcher) return deps.dispatcher;
+ if (!config.PARTNER_EGRESS_PROXY_URL) return undefined;
+ if (
+ !proxyDispatcher ||
+ proxyDispatcherUrl !== config.PARTNER_EGRESS_PROXY_URL
+ ) {
+ proxyDispatcher = new ProxyAgent(config.PARTNER_EGRESS_PROXY_URL);
+ proxyDispatcherUrl = config.PARTNER_EGRESS_PROXY_URL;
+ }
+ return proxyDispatcher;
+}
+
+function tokenCacheKey(destination: AzureSentinelDestination): string {
+ const credentialDigest = createHash("sha256")
+ .update(destination.clientSecret)
+ .digest("base64url");
+ return `${destination.tenantId}:${destination.clientId}:${credentialDigest}`;
+}
+
+function cacheToken(cacheKey: string, token: CachedToken): void {
+ const now = Date.now();
+ for (const [key, cached] of tokenCache) {
+ if (cached.expiresAt <= now) tokenCache.delete(key);
+ }
+ tokenCache.delete(cacheKey);
+ while (tokenCache.size >= MAX_TOKEN_CACHE_ENTRIES) {
+ const oldestKey = tokenCache.keys().next().value;
+ if (oldestKey === undefined) break;
+ tokenCache.delete(oldestKey);
+ }
+ tokenCache.set(cacheKey, token);
+}
+
+function invalidateToken(destination: AzureSentinelDestination): void {
+ tokenCache.delete(tokenCacheKey(destination));
+}
+
+function retryAfterMs(response: Response): number | undefined {
+ const value = response.headers.get("retry-after");
+ if (!value) return undefined;
+ const seconds = Number(value);
+ if (Number.isFinite(seconds)) {
+ return Math.min(Math.max(0, seconds * 1000), MAX_RETRY_DELAY_MS);
+ }
+ const date = Date.parse(value);
+ if (Number.isNaN(date)) return undefined;
+ return Math.min(Math.max(0, date - Date.now()), MAX_RETRY_DELAY_MS);
+}
+
+async function errorText(response: Response): Promise<string> {
+ const text = await response.text();
+ return text.slice(0, 2048) || response.statusText || "Request failed";
+}
+
+async function waitBeforeRetry(
+ attempt: number,
+ response: Response,
+ sleep: (ms: number) => Promise<void>,
+): Promise<void> {
+ const delay =
+ retryAfterMs(response) ?? Math.min(250 * 2 ** (attempt - 1), 2000);
+ await response.body?.cancel();
+ await sleep(delay);
+}
+
+function networkErrorMessage(error: unknown): string {
+ return error instanceof Error ? error.message : "Network request failed";
+}
+
+async function getAccessToken(
+ destination: AzureSentinelDestination,
+ deps: SenderDependencies,
+): Promise<string> {
+ const cacheKey = tokenCacheKey(destination);
+ const cached = tokenCache.get(cacheKey);
+ if (cached && cached.expiresAt > Date.now() + 60_000) return cached.token;
+
+ const fetchImpl = deps.fetchImpl ?? (undiciFetch as FetchImpl);
+ const sleep =
+ deps.sleep ??
+ ((ms: number) => new Promise(resolve => setTimeout(resolve, ms)));
+ const dispatcher = getDispatcher(deps);
+ const tokenUrl =
+ deps.tokenUrl ??
+ `https://login.microsoftonline.com/${encodeURIComponent(destination.tenantId)}/oauth2/v2.0/token`;
+ const body = new URLSearchParams({
+ client_id: destination.clientId,
+ client_secret: destination.clientSecret,
+ scope: "https://monitor.azure.com//.default",
+ grant_type: "client_credentials",
+ }).toString();
+
+ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+ let response: Response;
+ try {
+ response = await fetchImpl(tokenUrl, {
+ method: "POST",
+ headers: { "content-type": "application/x-www-form-urlencoded" },
+ body,
+ dispatcher,
+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+ });
+ } catch (error) {
+ if (attempt < MAX_ATTEMPTS) {
+ await sleep(Math.min(250 * 2 ** (attempt - 1), 2000));
+ continue;
+ }
+ throw new SiemDeliveryError(
+ "delivery_error",
+ `Failed to obtain an Entra token: ${networkErrorMessage(error)}`,
+ );
+ }
+
+ if (response.ok) {
+ const payload = (await response.json()) as {
+ access_token?: unknown;
+ expires_in?: unknown;
+ };
+ if (typeof payload.access_token !== "string") {
+ throw new SiemDeliveryError(
+ "invalid_credentials",
+ "Entra token response did not include an access token",
+ response.status,
+ );
+ }
+ const expiresIn =
+ typeof payload.expires_in === "number" ? payload.expires_in : 3600;
+ cacheToken(cacheKey, {
+ token: payload.access_token,
+ expiresAt: Date.now() + expiresIn * 1000,
+ });
+ return payload.access_token;
+ }
+
+ if (
+ (response.status === 429 || response.status >= 500) &&
+ attempt < MAX_ATTEMPTS
+ ) {
+ await waitBeforeRetry(attempt, response, sleep);
+ continue;
+ }
+
+ const message = await errorText(response);
+ if (
+ response.status === 400 ||
+ response.status === 401 ||
+ response.status === 403
+ ) {
+ throw new SiemDeliveryError(
+ "invalid_credentials",
+ `Entra rejected the SIEM credentials: ${message}`,
+ response.status,
+ );
+ }
+ throw new SiemDeliveryError(
+ response.status === 429 ? "rate_limited" : "delivery_error",
+ `Failed to obtain an Entra token: ${message}`,
+ response.status,
+ retryAfterMs(response),
+ );
+ }
+
+ throw new SiemDeliveryError(
+ "delivery_error",
+ "Failed to obtain an Entra token",
+ );
+}
+
+export function* buildCompressedBatches(
+ events: ScrapeActivityEvent[],
+ maxBytes = MAX_COMPRESSED_BODY_BYTES,
+): Generator<Buffer> {
+ let pending: ScrapeActivityEvent[] = [];
+
+ for (const event of events) {
+ const candidate = [...pending, event];
+ const compressed = gzipSync(Buffer.from(JSON.stringify(candidate), "utf8"));
+ if (compressed.byteLength <= maxBytes) {
+ pending = candidate;
+ continue;
+ }
+
+ if (pending.length === 0) {
+ throw new SiemDeliveryError(
+ "payload_too_large",
+ `A single SIEM event exceeds the ${maxBytes}-byte compressed payload limit`,
+ );
+ }
+ yield gzipSync(Buffer.from(JSON.stringify(pending), "utf8"));
+ pending = [event];
+ const single = gzipSync(Buffer.from(JSON.stringify(pending), "utf8"));
+ if (single.byteLength > maxBytes) {
+ throw new SiemDeliveryError(
+ "payload_too_large",
+ `A single SIEM event exceeds the ${maxBytes}-byte compressed payload limit`,
+ );
+ }
+ }
+
+ if (pending.length > 0) {
+ yield gzipSync(Buffer.from(JSON.stringify(pending), "utf8"));
+ }
+}
+
+async function sendBatch(
+ destination: AzureSentinelDestination,
+ token: string,
+ body: Buffer,
+ deps: SenderDependencies,
+): Promise<void> {
+ const fetchImpl = deps.fetchImpl ?? (undiciFetch as FetchImpl);
+ const sleep =
+ deps.sleep ??
+ ((ms: number) => new Promise(resolve => setTimeout(resolve, ms)));
+ const dispatcher = getDispatcher(deps);
+ const url =
+ `${destination.dceUrl.replace(/\/+$/, "")}/dataCollectionRules/` +
+ `${encodeURIComponent(destination.dcrImmutableId)}/streams/` +
+ `${encodeURIComponent(destination.streamName)}?api-version=2023-01-01`;
+
+ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+ let response: Response;
+ try {
+ response = await fetchImpl(url, {
+ method: "POST",
+ headers: {
+ authorization: `Bearer ${token}`,
+ "content-type": "application/json",
+ "content-encoding": "gzip",
+ "x-ms-client-request-id": randomUUID(),
+ },
+ body,
+ dispatcher,
+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+ });
+ } catch (error) {
+ if (attempt < MAX_ATTEMPTS) {
+ await sleep(Math.min(250 * 2 ** (attempt - 1), 2000));
+ continue;
+ }
+ throw new SiemDeliveryError(
+ "delivery_error",
+ `Azure SIEM delivery failed: ${networkErrorMessage(error)}`,
+ );
+ }
+ if (response.ok) return;
+
+ if (
+ (response.status === 429 || response.status >= 500) &&
+ attempt < MAX_ATTEMPTS
+ ) {
+ await waitBeforeRetry(attempt, response, sleep);
+ continue;
+ }
+
+ const message = await errorText(response);
+ if (response.status >= 400 && response.status < 500) {
+ throw new SiemDeliveryError(
+ response.status === 401 || response.status === 403
+ ? "invalid_credentials"
+ : response.status === 429
+ ? "rate_limited"
+ : "schema_rejection",
+ `Azure rejected the SIEM event batch: ${message}`,
+ response.status,
+ retryAfterMs(response),
+ );
+ }
+ throw new SiemDeliveryError(
+ "delivery_error",
+ `Azure SIEM delivery failed: ${message}`,
+ response.status,
+ );
+ }
+}
+
+export async function sendAzureSentinelEvents(
+ destination: AzureSentinelDestination,
+ events: ScrapeActivityEvent[],
+ deps: SenderDependencies = {},
+): Promise<void> {
+ if (events.length === 0) return;
+ let token: string | undefined;
+ for (const body of buildCompressedBatches(events)) {
+ try {
+ token ??= await getAccessToken(destination, deps);
+ try {
+ await sendBatch(destination, token, body, deps);
+ } catch (error) {
+ if (
+ error instanceof SiemDeliveryError &&
+ (error.statusCode === 401 || error.statusCode === 403)
+ ) {
+ invalidateToken(destination);
+ token = await getAccessToken(destination, deps);
+ await sendBatch(destination, token, body, deps);
+ } else {
+ throw error;
+ }
+ }
+ siemLoggingDeliveryBatchesTotal.inc({ result: "success" });
+ } catch (error) {
+ if (
+ error instanceof SiemDeliveryError &&
+ (error.statusCode === 401 || error.statusCode === 403)
+ ) {
+ invalidateToken(destination);
+ }
+ siemLoggingDeliveryBatchesTotal.inc({ result: "failure" });
+ throw error;
+ }
+ }
+}
+
+export function clearAzureTokenCache(): void {
+ tokenCache.clear();
+}
diff --git a/apps/api/src/lib/siem-logging/crypto.test.ts b/apps/api/src/lib/siem-logging/crypto.test.ts
new file mode 100644
index 000000000..ea0c98a14
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/crypto.test.ts
@@ -0,0 +1,34 @@
+import crypto from "crypto";
+import { afterEach, describe, expect, it } from "vitest";
+import { config } from "../../config";
+import { decryptSiemSecret, encryptSiemSecret } from "./crypto";
+
+const originalKey = config.SIEM_LOGGING_ENCRYPTION_KEY;
+
+afterEach(() => {
+ config.SIEM_LOGGING_ENCRYPTION_KEY = originalKey;
+});
+
+describe("SIEM secret encryption", () => {
+ it("round-trips with AES-256-GCM without storing plaintext", () => {
+ config.SIEM_LOGGING_ENCRYPTION_KEY = crypto.randomBytes(32).toString("hex");
+ const encrypted = encryptSiemSecret("client-secret-value", "org-one");
+
+ expect(encrypted).toMatch(/^gcm:/);
+ expect(encrypted).not.toContain("client-secret-value");
+ expect(decryptSiemSecret(encrypted, "org-one")).toBe("client-secret-value");
+ expect(() => decryptSiemSecret(encrypted, "org-two")).toThrow();
+ });
+
+ it("refuses to store a secret without a valid external key", () => {
+ config.SIEM_LOGGING_ENCRYPTION_KEY = undefined;
+ expect(() => encryptSiemSecret("secret", "org-one")).toThrow(
+ "SIEM_LOGGING_ENCRYPTION_KEY is not configured",
+ );
+
+ config.SIEM_LOGGING_ENCRYPTION_KEY = "too-short";
+ expect(() => encryptSiemSecret("secret", "org-one")).toThrow(
+ "must be a 32-byte",
+ );
+ });
+});
diff --git a/apps/api/src/lib/siem-logging/crypto.ts b/apps/api/src/lib/siem-logging/crypto.ts
new file mode 100644
index 000000000..04cd3fc4e
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/crypto.ts
@@ -0,0 +1,64 @@
+import crypto from "crypto";
+import { config } from "../../config";
+
+const GCM_PREFIX = "gcm:";
+
+function getEncryptionKey(): Buffer {
+ const raw = config.SIEM_LOGGING_ENCRYPTION_KEY?.trim();
+ if (!raw) {
+ throw new Error("SIEM_LOGGING_ENCRYPTION_KEY is not configured");
+ }
+
+ let key: Buffer | null = null;
+ if (/^[0-9a-fA-F]{64}$/.test(raw)) {
+ key = Buffer.from(raw, "hex");
+ } else {
+ const decoded = Buffer.from(raw, "base64");
+ if (decoded.length === 32) key = decoded;
+ }
+
+ if (!key || key.length !== 32) {
+ throw new Error(
+ "SIEM_LOGGING_ENCRYPTION_KEY must be a 32-byte hex or base64 value",
+ );
+ }
+ return key;
+}
+
+export function encryptSiemSecret(secret: string, orgId: string): string {
+ const key = getEncryptionKey();
+ const iv = crypto.randomBytes(12);
+ const cipher = crypto.createCipheriv("aes-256-gcm", key, iv);
+ cipher.setAAD(Buffer.from(orgId, "utf8"));
+ const ciphertext = Buffer.concat([
+ cipher.update(secret, "utf8"),
+ cipher.final(),
+ ]);
+ const tag = cipher.getAuthTag();
+ return `${GCM_PREFIX}${iv.toString("base64")}:${tag.toString("base64")}:${ciphertext.toString("base64")}`;
+}
+
+export function decryptSiemSecret(stored: string, orgId: string): string {
+ if (!stored.startsWith(GCM_PREFIX)) {
+ throw new Error("SIEM client secret is not encrypted with AES-256-GCM");
+ }
+
+ const [ivB64, tagB64, ciphertextB64] = stored
+ .slice(GCM_PREFIX.length)
+ .split(":");
+ if (!ivB64 || !tagB64 || !ciphertextB64) {
+ throw new Error("Malformed encrypted SIEM client secret");
+ }
+
+ const decipher = crypto.createDecipheriv(
+ "aes-256-gcm",
+ getEncryptionKey(),
+ Buffer.from(ivB64, "base64"),
+ );
+ decipher.setAAD(Buffer.from(orgId, "utf8"));
+ decipher.setAuthTag(Buffer.from(tagB64, "base64"));
+ return Buffer.concat([
+ decipher.update(Buffer.from(ciphertextB64, "base64")),
+ decipher.final(),
+ ]).toString("utf8");
+}
diff --git a/apps/api/src/lib/siem-logging/event.test.ts b/apps/api/src/lib/siem-logging/event.test.ts
new file mode 100644
index 000000000..55d622597
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/event.test.ts
@@ -0,0 +1,202 @@
+import { describe, expect, it } from "vitest";
+import type { ScrapeJobSingleUrls } from "../../types";
+import { scrapeOptions } from "../../controllers/v2/types";
+import { UnsafeDomainBlockedError } from "../threat-protection/error";
+import type { ThreatDecision } from "../threat-protection/types";
+import { CrawlDenialError } from "../error";
+import {
+ buildRejectedScrapeActivityEvent,
+ buildScrapeActivityEvent,
+} from "./event";
+
+const allowedDecision: ThreatDecision = {
+ allowed: true,
+ rule: "default-allow",
+ url: "https://example.com/",
+ domain: "example.com",
+ providerConsulted: true,
+ verdict: {
+ provider: "google-web-risk",
+ riskScore: 0,
+ categories: [],
+ fromCache: false,
+ raw: { mustNotLeak: true },
+ },
+ mode: "normal",
+};
+
+function job(
+ overrides: Partial<ScrapeJobSingleUrls> = {},
+): ScrapeJobSingleUrls {
+ return {
+ mode: "single_urls",
+ url: "https://example.com/private?q=1",
+ team_id: "00000000-0000-0000-0000-000000000001",
+ scrapeOptions: scrapeOptions.parse({
+ auditMetadata: { username: "[email protected]" },
+ }),
+ internalOptions: {
+ teamId: "00000000-0000-0000-0000-000000000001",
+ orgId: "00000000-0000-0000-0000-000000000002",
+ },
+ origin: "api",
+ integration: "sdk",
+ zeroDataRetention: true,
+ apiKeyId: 42,
+ requestId: "00000000-0000-0000-0000-000000000003",
+ billing: { endpoint: "search" },
+ ...overrides,
+ };
+}
+
+describe("ScrapeActivityEvent", () => {
+ it("builds an attributable success event without raw classifier data", () => {
+ const event = buildScrapeActivityEvent(
+ "00000000-0000-0000-0000-000000000004",
+ job(),
+ "00000000-0000-0000-0000-000000000002",
+ "production",
+ {
+ success: true,
+ document: {
+ markdown: "not included",
+ metadata: { statusCode: 200, proxyUsed: "basic" },
+ },
+ threatDecisions: [allowedDecision],
+ startedAt: 1_000,
+ completedAt: 2_500,
+ },
+ );
+
+ expect(event).toMatchObject({
+ endpoint: "search",
+ api_key: { id: "42", name: "production" },
+ audit_metadata: { username: "[email protected]" },
+ domain: "example.com",
+ result: "success",
+ zero_data_retention: true,
+ threat: {
+ decision: "allow",
+ provider: "google-web-risk",
+ categories: [],
+ },
+ });
+ expect(JSON.stringify(event)).not.toContain("mustNotLeak");
+ expect(JSON.stringify(event)).not.toContain("not included");
+ });
+
+ it("normalizes a blocked threat decision and error", () => {
+ const denied: ThreatDecision = {
+ ...allowedDecision,
+ allowed: false,
+ rule: "risk-score",
+ verdict: {
+ ...allowedDecision.verdict!,
+ riskScore: 100,
+ categories: ["MALWARE"],
+ },
+ };
+ const error = new UnsafeDomainBlockedError(
+ "https://example.com/private",
+ denied,
+ );
+ const event = buildScrapeActivityEvent("scrape-id", job(), "org-id", null, {
+ success: false,
+ error,
+ threatDecisions: [denied],
+ startedAt: 1_000,
+ completedAt: 2_000,
+ });
+
+ expect(event.result).toBe("blocked");
+ expect(event.error?.code).toBe("unsafe_domain_blocked");
+ expect(event.threat).toMatchObject({
+ decision: "deny",
+ rule: "risk-score",
+ categories: ["MALWARE"],
+ security_alert: { detected: true, category: "MALWARE" },
+ });
+ });
+
+ it("serializes circular and bigint non-Error failures", () => {
+ const thrown: { count: bigint; self?: unknown } = { count: 42n };
+ thrown.self = thrown;
+ const event = buildScrapeActivityEvent("scrape-id", job(), "org-id", null, {
+ success: false,
+ error: thrown,
+ startedAt: 1_000,
+ completedAt: 2_000,
+ });
+
+ expect(event.result).toBe("failure");
+ expect(event.error?.message).toContain('"count":"42"');
+ expect(event.error?.message).toContain("[Circular]");
+ });
+
+ it("builds blocked events for requests rejected before job creation", () => {
+ const denied: ThreatDecision = {
+ ...allowedDecision,
+ allowed: false,
+ rule: "risk-score",
+ verdict: {
+ ...allowedDecision.verdict!,
+ categories: ["SOCIAL_ENGINEERING"],
+ },
+ };
+ const error = new UnsafeDomainBlockedError(denied.url, denied);
+ const event = buildRejectedScrapeActivityEvent(
+ {
+ scrapeId: "blocked-scrape-id",
+ requestId: "parent-request-id",
+ endpoint: "batch_scrape",
+ teamId: "team-id",
+ apiKeyId: 42,
+ auditMetadata: { username: "[email protected]" },
+ startedAt: 1_000,
+ completedAt: 1_500,
+ url: denied.url,
+ error,
+ threatDecisions: [denied],
+ origin: "api",
+ integration: "sdk",
+ zeroDataRetention: true,
+ },
+ "org-id",
+ "production",
+ );
+
+ expect(event).toMatchObject({
+ scrape_id: "blocked-scrape-id",
+ request_id: "parent-request-id",
+ endpoint: "batch_scrape",
+ result: "blocked",
+ http_status: null,
+ api_key: { id: "42", name: "production" },
+ threat: {
+ decision: "deny",
+ categories: ["SOCIAL_ENGINEERING"],
+ },
+ });
+ });
+
+ it("classifies non-threat policy denials as blocked", () => {
+ const event = buildRejectedScrapeActivityEvent(
+ {
+ scrapeId: "blocked-scrape-id",
+ requestId: "parent-request-id",
+ endpoint: "extract",
+ teamId: "team-id",
+ url: "https://example.com",
+ error: new CrawlDenialError("URL blocked by policy"),
+ origin: "api",
+ zeroDataRetention: false,
+ },
+ "org-id",
+ null,
+ );
+
+ expect(event.result).toBe("blocked");
+ expect(event.error?.code).toBe("CRAWL_DENIAL");
+ expect(event.threat).toBeUndefined();
+ });
+});
diff --git a/apps/api/src/lib/siem-logging/event.ts b/apps/api/src/lib/siem-logging/event.ts
new file mode 100644
index 000000000..8fa713dcd
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/event.ts
@@ -0,0 +1,250 @@
+import type { Document } from "../../controllers/v2/types";
+import type { ScrapeJobSingleUrls } from "../../types";
+import type { ThreatDecision } from "../threat-protection/types";
+import {
+ CrawlDenialError,
+ JobCancelledError,
+ TransportableError,
+} from "../error";
+import { UnsafeDomainBlockedError } from "../threat-protection/error";
+import type {
+ AuditMetadata,
+ ScrapeActivityEvent,
+ ScrapeActivityResult,
+ ScrapeActivityThreat,
+} from "./types";
+
+export interface ScrapeActivityOutcome {
+ success: boolean;
+ document?: Document | null;
+ error?: unknown;
+ threatDecisions?: ThreatDecision[];
+ startedAt: number;
+ completedAt: number;
+}
+
+export interface RejectedScrapeActivity {
+ scrapeId: string;
+ requestId: string;
+ endpoint: ScrapeActivityEvent["endpoint"];
+ teamId: string;
+ apiKeyId?: string | number | null;
+ auditMetadata?: AuditMetadata;
+ startedAt?: number;
+ completedAt?: number;
+ url: string;
+ error: unknown;
+ threatDecisions?: ThreatDecision[];
+ origin: string;
+ integration?: string | null;
+ zeroDataRetention: boolean;
+}
+
+function endpointForJob(
+ job: ScrapeJobSingleUrls,
+): ScrapeActivityEvent["endpoint"] {
+ const endpoint = job.billing?.endpoint;
+ if (
+ endpoint === "scrape" ||
+ endpoint === "crawl" ||
+ endpoint === "batch_scrape" ||
+ endpoint === "search" ||
+ endpoint === "extract" ||
+ endpoint === "agent" ||
+ endpoint === "parse"
+ ) {
+ return endpoint;
+ }
+ if (job.internalOptions?.isParse) return "parse";
+ if (job.from_extract) return "extract";
+ if (job.crawl_id) {
+ return job.crawlerOptions === null ? "batch_scrape" : "crawl";
+ }
+ return "scrape";
+}
+
+function resultForOutcome(
+ outcome: ScrapeActivityOutcome,
+): ScrapeActivityResult {
+ if (outcome.success) return "success";
+ if (
+ outcome.error instanceof CrawlDenialError ||
+ outcome.error instanceof UnsafeDomainBlockedError ||
+ outcome.threatDecisions?.some(decision => !decision.allowed)
+ ) {
+ return "blocked";
+ }
+ if (outcome.error instanceof JobCancelledError) return "cancelled";
+ return "failure";
+}
+
+function threatForDecisions(
+ decisions: ThreatDecision[] | undefined,
+): ScrapeActivityThreat | undefined {
+ if (!decisions || decisions.length === 0) return undefined;
+ const decision =
+ decisions.find(candidate => !candidate.allowed) ??
+ decisions[decisions.length - 1];
+ const categories = [
+ ...new Set(
+ decisions.flatMap(candidate => candidate.verdict?.categories ?? []),
+ ),
+ ];
+ return {
+ decision: decision.allowed ? "allow" : "deny",
+ rule: decision.rule,
+ provider: decision.verdict?.provider ?? null,
+ categories,
+ security_alert: {
+ detected: categories.length > 0,
+ category: categories[0] ?? null,
+ },
+ };
+}
+
+function errorForOutcome(
+ outcome: ScrapeActivityOutcome,
+): ScrapeActivityEvent["error"] {
+ if (outcome.success || outcome.error === undefined) return null;
+ const error =
+ outcome.error instanceof Error
+ ? outcome.error
+ : new Error(serializeThrownValue(outcome.error));
+ return {
+ code: error instanceof TransportableError ? error.code : null,
+ message: error.message.slice(0, 2048),
+ };
+}
+
+function serializeThrownValue(value: unknown): string {
+ if (typeof value === "string") return value;
+ try {
+ const seen = new WeakSet<object>();
+ const serialized = JSON.stringify(value, (_key, candidate: unknown) => {
+ if (typeof candidate === "bigint") return candidate.toString();
+ if (candidate && typeof candidate === "object") {
+ if (seen.has(candidate)) return "[Circular]";
+ seen.add(candidate);
+ }
+ return candidate;
+ });
+ return serialized ?? String(value);
+ } catch {
+ return "Non-Error value thrown";
+ }
+}
+
+interface ScrapeActivityEventSource {
+ scrapeId: string;
+ requestId: string;
+ endpoint: ScrapeActivityEvent["endpoint"];
+ teamId: string;
+ apiKeyId: string | number | null | undefined;
+ auditMetadata: AuditMetadata | undefined;
+ url: string;
+ origin: string;
+ integration: string | null | undefined;
+ zeroDataRetention: boolean;
+ httpStatus: number | null;
+}
+
+function buildEvent(
+ source: ScrapeActivityEventSource,
+ orgId: string,
+ apiKeyName: string | null,
+ outcome: ScrapeActivityOutcome,
+): ScrapeActivityEvent {
+ let domain = "";
+ try {
+ domain = new URL(source.url).hostname.toLowerCase();
+ } catch {}
+
+ const event: ScrapeActivityEvent = {
+ schema_version: 1,
+ event_type: "scrape_activity",
+ scrape_id: source.scrapeId,
+ request_id: source.requestId,
+ endpoint: source.endpoint,
+ team_id: source.teamId,
+ org_id: orgId,
+ api_key: {
+ id: source.apiKeyId == null ? null : String(source.apiKeyId),
+ name: apiKeyName,
+ },
+ audit_metadata: source.auditMetadata ?? {},
+ started_at: new Date(outcome.startedAt).toISOString(),
+ completed_at: new Date(outcome.completedAt).toISOString(),
+ url: source.url,
+ domain,
+ http_method: "GET",
+ http_status: source.httpStatus,
+ result: resultForOutcome(outcome),
+ error: errorForOutcome(outcome),
+ origin: source.origin,
+ integration: source.integration ?? null,
+ zero_data_retention: source.zeroDataRetention,
+ };
+ const threat = threatForDecisions(outcome.threatDecisions);
+ if (threat) event.threat = threat;
+ return event;
+}
+
+export function buildScrapeActivityEvent(
+ jobId: string,
+ job: ScrapeJobSingleUrls,
+ orgId: string,
+ apiKeyName: string | null,
+ outcome: ScrapeActivityOutcome,
+): ScrapeActivityEvent {
+ return buildEvent(
+ {
+ scrapeId: jobId,
+ requestId: job.requestId ?? job.crawl_id ?? jobId,
+ endpoint: endpointForJob(job),
+ teamId: job.team_id,
+ apiKeyId: job.apiKeyId,
+ auditMetadata: job.scrapeOptions.auditMetadata,
+ url: job.url,
+ origin: job.origin,
+ integration: job.integration,
+ zeroDataRetention: job.zeroDataRetention,
+ httpStatus: outcome.document?.metadata.statusCode ?? null,
+ },
+ orgId,
+ apiKeyName,
+ outcome,
+ );
+}
+
+export function buildRejectedScrapeActivityEvent(
+ input: RejectedScrapeActivity,
+ orgId: string,
+ apiKeyName: string | null,
+): ScrapeActivityEvent {
+ const startedAt = input.startedAt ?? Date.now();
+ const outcome: ScrapeActivityOutcome = {
+ success: false,
+ error: input.error,
+ threatDecisions: input.threatDecisions,
+ startedAt,
+ completedAt: input.completedAt ?? startedAt,
+ };
+ return buildEvent(
+ {
+ scrapeId: input.scrapeId,
+ requestId: input.requestId,
+ endpoint: input.endpoint,
+ teamId: input.teamId,
+ apiKeyId: input.apiKeyId,
+ auditMetadata: input.auditMetadata,
+ url: input.url,
+ origin: input.origin,
+ integration: input.integration,
+ zeroDataRetention: input.zeroDataRetention,
+ httpStatus: null,
+ },
+ orgId,
+ apiKeyName,
+ outcome,
+ );
+}
diff --git a/apps/api/src/lib/siem-logging/index.test.ts b/apps/api/src/lib/siem-logging/index.test.ts
new file mode 100644
index 000000000..78631474a
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/index.test.ts
@@ -0,0 +1,62 @@
+import { describe, expect, it, vi } from "vitest";
+import type { ScrapeJobSingleUrls } from "../../types";
+
+const mocks = vi.hoisted(() => ({
+ getTeam: vi.fn(),
+ isEnabled: vi.fn().mockResolvedValue(true),
+ getApiKeyName: vi.fn().mockResolvedValue("production"),
+ enqueue: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("../../controllers/auth", () => ({
+ getACUCTeam: mocks.getTeam,
+}));
+
+vi.mock("./store", () => ({
+ isOrgSiemLoggingEnabled: mocks.isEnabled,
+ getApiKeyName: mocks.getApiKeyName,
+}));
+
+vi.mock("./queue", () => ({
+ enqueueSiemLoggingEvent: mocks.enqueue,
+ enqueueSiemLoggingEvents: vi.fn().mockResolvedValue(undefined),
+}));
+
+import { emitScrapeActivityEvent } from "./index";
+
+const job = {
+ mode: "single_urls",
+ url: "https://example.com",
+ team_id: "team-id",
+ scrapeOptions: {},
+ internalOptions: { teamId: "team-id", orgId: "org-id" },
+ origin: "api",
+ zeroDataRetention: false,
+ apiKeyId: 42,
+} as ScrapeJobSingleUrls;
+
+describe("SIEM logging emission", () => {
+ it("does not wait for lookups or queue insertion", async () => {
+ let resolveTeam!: (value: unknown) => void;
+ mocks.getTeam.mockReturnValueOnce(
+ new Promise(resolve => {
+ resolveTeam = resolve;
+ }),
+ );
+
+ expect(
+ emitScrapeActivityEvent("scrape-id", job, {
+ success: true,
+ startedAt: 1_000,
+ completedAt: 2_000,
+ }),
+ ).toBeUndefined();
+ expect(mocks.enqueue).not.toHaveBeenCalled();
+
+ resolveTeam({
+ flags: { siemLogging: true },
+ org_id: "org-id",
+ });
+ await vi.waitFor(() => expect(mocks.enqueue).toHaveBeenCalledOnce());
+ });
+});
diff --git a/apps/api/src/lib/siem-logging/index.ts b/apps/api/src/lib/siem-logging/index.ts
new file mode 100644
index 000000000..e400fdae9
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/index.ts
@@ -0,0 +1,112 @@
+import type { ScrapeJobSingleUrls } from "../../types";
+import { getACUCTeam } from "../../controllers/auth";
+import { logger as _logger } from "../logger";
+import {
+ buildRejectedScrapeActivityEvent,
+ buildScrapeActivityEvent,
+ type RejectedScrapeActivity,
+ type ScrapeActivityOutcome,
+} from "./event";
+import { enqueueSiemLoggingEvent, enqueueSiemLoggingEvents } from "./queue";
+import { getApiKeyName, isOrgSiemLoggingEnabled } from "./store";
+
+const logger = _logger.child({ module: "siem-logging" });
+
+export function emitScrapeActivityEvent(
+ jobId: string,
+ job: ScrapeJobSingleUrls,
+ outcome: ScrapeActivityOutcome,
+): void {
+ void emitScrapeActivityEventAsync(jobId, job, outcome);
+}
+
+async function emitScrapeActivityEventAsync(
+ jobId: string,
+ job: ScrapeJobSingleUrls,
+ outcome: ScrapeActivityOutcome,
+): Promise<void> {
+ try {
+ const team = await getACUCTeam(job.team_id);
+ if (team?.flags?.siemLogging !== true) return;
+ const orgId = job.internalOptions?.orgId ?? team.org_id;
+ if (!orgId) return;
+
+ if (!(await isOrgSiemLoggingEnabled(orgId))) return;
+
+ const apiKeyName = await getApiKeyName(job.team_id, job.apiKeyId);
+ const event = buildScrapeActivityEvent(
+ jobId,
+ job,
+ orgId,
+ apiKeyName,
+ outcome,
+ );
+ await enqueueSiemLoggingEvent(orgId, event);
+ } catch (error) {
+ logger.error("Failed to enqueue scrape activity event", {
+ error,
+ jobId,
+ teamId: job.team_id,
+ });
+ }
+}
+
+export function emitRejectedScrapeActivityEvent(
+ input: RejectedScrapeActivity,
+): void {
+ emitRejectedScrapeActivityEvents([input]);
+}
+
+export function emitRejectedScrapeActivityEvents(
+ inputs: RejectedScrapeActivity[],
+): void {
+ if (inputs.length === 0) return;
+ void emitRejectedScrapeActivityEventsAsync(inputs);
+}
+
+async function emitRejectedScrapeActivityEventsAsync(
+ inputs: RejectedScrapeActivity[],
+): Promise<void> {
+ try {
+ const byTeam = new Map<string, RejectedScrapeActivity[]>();
+ for (const input of inputs) {
+ const teamInputs = byTeam.get(input.teamId) ?? [];
+ teamInputs.push(input);
+ byTeam.set(input.teamId, teamInputs);
+ }
+
+ for (const [teamId, teamInputs] of byTeam) {
+ const team = await getACUCTeam(teamId);
+ if (team?.flags?.siemLogging !== true) continue;
+ const orgId = team.org_id;
+ if (!orgId) continue;
+
+ if (!(await isOrgSiemLoggingEnabled(orgId))) continue;
+
+ const apiKeyNames = new Map<string | null, string | null>();
+ const apiKeyIds = teamInputs.map(input =>
+ input.apiKeyId == null ? null : String(input.apiKeyId),
+ );
+ await Promise.all(
+ [...new Set(apiKeyIds)].map(async apiKeyId => {
+ apiKeyNames.set(apiKeyId, await getApiKeyName(teamId, apiKeyId));
+ }),
+ );
+ const events = teamInputs.map(input => {
+ const apiKeyId = input.apiKeyId == null ? null : String(input.apiKeyId);
+ return buildRejectedScrapeActivityEvent(
+ input,
+ orgId,
+ apiKeyNames.get(apiKeyId) ?? null,
+ );
+ });
+ await enqueueSiemLoggingEvents(orgId, events);
+ }
+ } catch (error) {
+ logger.error("Failed to enqueue rejected scrape activity events", {
+ error,
+ eventCount: inputs.length,
+ teamIds: [...new Set(inputs.map(input => input.teamId))],
+ });
+ }
+}
diff --git a/apps/api/src/lib/siem-logging/metrics.ts b/apps/api/src/lib/siem-logging/metrics.ts
new file mode 100644
index 000000000..f8837e310
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/metrics.ts
@@ -0,0 +1,19 @@
+import { Counter } from "prom-client";
+
+export const siemLoggingEventsTotal = new Counter({
+ name: "firecrawl_siem_logging_events_total",
+ help: "SIEM logging event lifecycle outcomes",
+ labelNames: ["result"] as const,
+});
+
+export const siemLoggingDeliveryBatchesTotal = new Counter({
+ name: "firecrawl_siem_logging_delivery_batches_total",
+ help: "SIEM delivery batches by outcome",
+ labelNames: ["result"] as const,
+});
+
+export const siemLoggingDeliveryFailuresTotal = new Counter({
+ name: "firecrawl_siem_logging_delivery_failures_total",
+ help: "SIEM delivery failures by normalized reason",
+ labelNames: ["reason"] as const,
+});
diff --git a/apps/api/src/lib/siem-logging/queue.test.ts b/apps/api/src/lib/siem-logging/queue.test.ts
new file mode 100644
index 000000000..6fe00ee56
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/queue.test.ts
@@ -0,0 +1,58 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { ScrapeActivityEvent } from "./types";
+
+const mocks = vi.hoisted(() => ({
+ addBulk: vi.fn().mockResolvedValue([]),
+ inc: vi.fn(),
+}));
+
+vi.mock("../../services/queue-service", () => ({
+ getSiemLoggingQueue: () => ({ addBulk: mocks.addBulk }),
+}));
+
+vi.mock("./metrics", () => ({
+ siemLoggingEventsTotal: { inc: mocks.inc },
+}));
+
+import { enqueueSiemLoggingEvent } from "./queue";
+
+const event: ScrapeActivityEvent = {
+ schema_version: 1,
+ event_type: "scrape_activity",
+ scrape_id: "scrape-id",
+ request_id: "request-id",
+ endpoint: "scrape",
+ team_id: "team-id",
+ org_id: "org-id",
+ api_key: { id: null, name: null },
+ audit_metadata: {},
+ started_at: "2026-07-27T00:00:00.000Z",
+ completed_at: "2026-07-27T00:00:01.000Z",
+ url: "https://example.com",
+ domain: "example.com",
+ http_method: "GET",
+ http_status: 200,
+ result: "success",
+ error: null,
+ origin: "api",
+ integration: null,
+ zero_data_retention: false,
+};
+
+describe("SIEM logging queue", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("deduplicates repeat terminal events regardless of completion time", async () => {
+ await enqueueSiemLoggingEvent("org-id", event);
+ await enqueueSiemLoggingEvent("org-id", {
+ ...event,
+ completed_at: "2026-07-27T00:01:00.000Z",
+ });
+
+ const firstJobId = mocks.addBulk.mock.calls[0][0][0].opts.jobId;
+ const secondJobId = mocks.addBulk.mock.calls[1][0][0].opts.jobId;
+ expect(firstJobId).toBe(secondJobId);
+ });
+});
diff --git a/apps/api/src/lib/siem-logging/queue.ts b/apps/api/src/lib/siem-logging/queue.ts
new file mode 100644
index 000000000..45e343400
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/queue.ts
@@ -0,0 +1,46 @@
+import { createHash } from "node:crypto";
+import { getSiemLoggingQueue } from "../../services/queue-service";
+import { siemLoggingEventsTotal } from "./metrics";
+import type { ScrapeActivityEvent, SiemLoggingJobData } from "./types";
+
+function jobIdFor(orgId: string, event: ScrapeActivityEvent): string {
+ return createHash("sha256")
+ .update(
+ JSON.stringify([
+ orgId,
+ event.schema_version,
+ event.event_type,
+ event.scrape_id,
+ event.request_id,
+ event.result,
+ ]),
+ )
+ .digest("hex");
+}
+
+export async function enqueueSiemLoggingEvent(
+ orgId: string,
+ event: ScrapeActivityEvent,
+): Promise<void> {
+ await enqueueSiemLoggingEvents(orgId, [event]);
+}
+
+export async function enqueueSiemLoggingEvents(
+ orgId: string,
+ events: ScrapeActivityEvent[],
+): Promise<void> {
+ if (events.length === 0) return;
+ try {
+ await getSiemLoggingQueue().addBulk(
+ events.map(event => ({
+ name: "deliver",
+ data: { orgId, event } satisfies SiemLoggingJobData,
+ opts: { jobId: jobIdFor(orgId, event) },
+ })),
+ );
+ siemLoggingEventsTotal.inc({ result: "queued" }, events.length);
+ } catch (error) {
+ siemLoggingEventsTotal.inc({ result: "enqueue_failed" }, events.length);
+ throw error;
+ }
+}
diff --git a/apps/api/src/lib/siem-logging/store.test.ts b/apps/api/src/lib/siem-logging/store.test.ts
new file mode 100644
index 000000000..003cab309
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/store.test.ts
@@ -0,0 +1,64 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ redisGet: vi.fn(),
+ redisSet: vi.fn().mockResolvedValue("OK"),
+ limit: vi.fn(),
+}));
+
+const query = {
+ from: vi.fn(() => query),
+ where: vi.fn(() => query),
+ limit: mocks.limit,
+};
+
+vi.mock("../../db/connection", () => ({
+ db: { select: vi.fn(() => query) },
+ dbRr: { select: vi.fn(() => query) },
+}));
+
+vi.mock("../../services/queue-service", () => ({
+ getRedisConnection: () => ({
+ get: mocks.redisGet,
+ set: mocks.redisSet,
+ }),
+}));
+
+import { isOrgSiemLoggingEnabled } from "./store";
+
+describe("SIEM logging enabled cache", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.redisGet.mockResolvedValue(null);
+ });
+
+ it("uses an organization-specific Redis Cluster hash tag", async () => {
+ mocks.limit.mockResolvedValue([{ enabled: true }]);
+
+ await expect(isOrgSiemLoggingEnabled("org-one")).resolves.toBe(true);
+
+ expect(mocks.redisGet).toHaveBeenCalledWith(
+ "{siemLoggingConfig:org-one}:enabled",
+ );
+ expect(mocks.redisSet).toHaveBeenCalledWith(
+ "{siemLoggingConfig:org-one}:enabled",
+ "1",
+ "EX",
+ 60,
+ );
+ });
+
+ it("treats a missing configuration table as disabled", async () => {
+ mocks.limit.mockRejectedValue({
+ cause: { code: "42P01" },
+ });
+
+ await expect(isOrgSiemLoggingEnabled("org-two")).resolves.toBe(false);
+ expect(mocks.redisSet).toHaveBeenCalledWith(
+ "{siemLoggingConfig:org-two}:enabled",
+ "0",
+ "EX",
+ 60,
+ );
+ });
+});
diff --git a/apps/api/src/lib/siem-logging/store.ts b/apps/api/src/lib/siem-logging/store.ts
new file mode 100644
index 000000000..4c69885f3
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/store.ts
@@ -0,0 +1,227 @@
+import { and, eq, sql } from "drizzle-orm";
+import { z } from "zod";
+import { db, dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
+import { getRedisConnection } from "../../services/queue-service";
+import { logger as _logger } from "../logger";
+import { decryptSiemSecret, encryptSiemSecret } from "./crypto";
+import type { OrgSiemLoggingConfig, SiemLoggingConfigInput } from "./types";
+
+const CACHE_TTL_MS = 60_000;
+const CACHE_MAX_ENTRIES = 1000;
+const ENABLED_CACHE_TTL_SECONDS = 60;
+const logger = _logger.child({ module: "siem-logging-store" });
+type SiemConfigRow = typeof schema.siem_logging_config.$inferSelect;
+
+const configCache = new Map<
+ string,
+ { expiresAt: number; value: SiemConfigRow | null }
+>();
+
+function enabledCacheKey(orgId: string): string {
+ return `{siemLoggingConfig:${orgId}}:enabled`;
+}
+
+const storedDestinationSchema = z.strictObject({
+ type: z.literal("azure_sentinel"),
+ tenantId: z.string(),
+ clientId: z.string(),
+ dceUrl: z.string(),
+ dcrImmutableId: z.string(),
+ streamName: z.string(),
+});
+
+function isMissingTableError(error: unknown): boolean {
+ let current: unknown = error;
+ for (let depth = 0; depth < 5 && current; depth++) {
+ if ((current as { code?: unknown }).code === "42P01") return true;
+ current = (current as { cause?: unknown }).cause;
+ }
+ return false;
+}
+
+function rowToConfig(row: SiemConfigRow): OrgSiemLoggingConfig {
+ const destination = storedDestinationSchema.parse(row.destination);
+ return {
+ orgId: row.org_id,
+ enabled: row.enabled,
+ destination: {
+ ...destination,
+ clientSecret: decryptSiemSecret(row.secret_ciphertext, row.org_id),
+ },
+ createdAt: row.created_at,
+ updatedAt: row.updated_at,
+ };
+}
+
+function getCachedRow(orgId: string): SiemConfigRow | null | undefined {
+ const cached = configCache.get(orgId);
+ if (!cached) return undefined;
+ if (cached.expiresAt <= Date.now()) {
+ configCache.delete(orgId);
+ return undefined;
+ }
+ configCache.delete(orgId);
+ configCache.set(orgId, cached);
+ return cached.value;
+}
+
+function cacheRow(orgId: string, value: SiemConfigRow | null): void {
+ const now = Date.now();
+ for (const [cachedOrgId, cached] of configCache) {
+ if (cached.expiresAt <= now) configCache.delete(cachedOrgId);
+ }
+ configCache.delete(orgId);
+ while (configCache.size >= CACHE_MAX_ENTRIES) {
+ const oldestOrgId = configCache.keys().next().value;
+ if (oldestOrgId === undefined) break;
+ configCache.delete(oldestOrgId);
+ }
+ configCache.set(orgId, {
+ expiresAt: now + CACHE_TTL_MS,
+ value,
+ });
+}
+
+export async function getOrgSiemLoggingConfig(
+ orgId: string,
+ options: { fresh?: boolean; primary?: boolean } = {},
+): Promise<OrgSiemLoggingConfig | null> {
+ const row = await getOrgSiemLoggingConfigRow(orgId, options);
+ return row ? rowToConfig(row) : null;
+}
+
+async function getOrgSiemLoggingConfigRow(
+ orgId: string,
+ options: { fresh?: boolean; primary?: boolean } = {},
+): Promise<SiemConfigRow | null> {
+ if (!options.fresh) {
+ const cached = getCachedRow(orgId);
+ if (cached !== undefined) return cached;
+ }
+
+ let rows: SiemConfigRow[];
+ try {
+ rows = await (options.primary ? db : dbRr)
+ .select()
+ .from(schema.siem_logging_config)
+ .where(eq(schema.siem_logging_config.org_id, orgId))
+ .limit(1);
+ } catch (error) {
+ if (!isMissingTableError(error)) throw error;
+ rows = [];
+ }
+
+ const row = rows[0] ?? null;
+ cacheRow(orgId, row);
+ return row;
+}
+
+export async function isOrgSiemLoggingEnabled(orgId: string): Promise<boolean> {
+ const key = enabledCacheKey(orgId);
+ const redis = getRedisConnection();
+ const cached = await redis.get(key);
+ if (cached !== null) return cached === "1";
+
+ let rows: { enabled: boolean }[] = [];
+ try {
+ rows = await db
+ .select({ enabled: schema.siem_logging_config.enabled })
+ .from(schema.siem_logging_config)
+ .where(eq(schema.siem_logging_config.org_id, orgId))
+ .limit(1);
+ } catch (error) {
+ if (!isMissingTableError(error)) throw error;
+ }
+ const enabled = rows[0]?.enabled === true;
+ await redis.set(key, enabled ? "1" : "0", "EX", ENABLED_CACHE_TTL_SECONDS);
+ return enabled;
+}
+
+export async function upsertOrgSiemLoggingConfig(
+ orgId: string,
+ input: SiemLoggingConfigInput,
+): Promise<OrgSiemLoggingConfig> {
+ const destination = {
+ type: input.destination.type,
+ tenantId: input.destination.tenantId,
+ clientId: input.destination.clientId,
+ dceUrl: input.destination.dceUrl.replace(/\/+$/, ""),
+ dcrImmutableId: input.destination.dcrImmutableId,
+ streamName: input.destination.streamName,
+ };
+ const existingRows = await db
+ .select({
+ secret_ciphertext: schema.siem_logging_config.secret_ciphertext,
+ })
+ .from(schema.siem_logging_config)
+ .where(eq(schema.siem_logging_config.org_id, orgId))
+ .limit(1);
+
+ const existingCiphertext = existingRows[0]?.secret_ciphertext;
+ if (!input.destination.clientSecret && !existingCiphertext) {
+ throw new Error("clientSecret is required when SIEM is first configured");
+ }
+
+ const secretCiphertext = input.destination.clientSecret
+ ? encryptSiemSecret(input.destination.clientSecret, orgId)
+ : existingCiphertext!;
+ const values = {
+ org_id: orgId,
+ enabled: input.enabled,
+ destination,
+ secret_ciphertext: secretCiphertext,
+ };
+ const [row] = await db
+ .insert(schema.siem_logging_config)
+ .values(values)
+ .onConflictDoUpdate({
+ target: schema.siem_logging_config.org_id,
+ set: {
+ enabled: values.enabled,
+ destination: values.destination,
+ secret_ciphertext: input.destination.clientSecret
+ ? values.secret_ciphertext
+ : sql`${schema.siem_logging_config.secret_ciphertext}`,
+ updated_at: new Date().toISOString(),
+ },
+ })
+ .returning();
+
+ const updated = rowToConfig(row);
+ cacheRow(orgId, row);
+ await getRedisConnection()
+ .set(
+ enabledCacheKey(orgId),
+ row.enabled ? "1" : "0",
+ "EX",
+ ENABLED_CACHE_TTL_SECONDS,
+ )
+ .catch(error => {
+ logger.warn("Failed to update shared SIEM logging config cache", {
+ error,
+ orgId,
+ });
+ });
+ return updated;
+}
+
+export async function getApiKeyName(
+ teamId: string,
+ apiKeyId: string | number | null | undefined,
+): Promise<string | null> {
+ if (apiKeyId == null) return null;
+ const normalizedApiKeyId = String(apiKeyId);
+ if (!/^\d+$/.test(normalizedApiKeyId)) return null;
+ const rows = await dbRr
+ .select({ name: schema.api_keys.name })
+ .from(schema.api_keys)
+ .where(
+ and(
+ sql`${schema.api_keys.id} = cast(${normalizedApiKeyId} as bigint)`,
+ eq(schema.api_keys.team_id, teamId),
+ ),
+ )
+ .limit(1);
+ return rows[0]?.name ?? null;
+}
diff --git a/apps/api/src/lib/siem-logging/types.ts b/apps/api/src/lib/siem-logging/types.ts
new file mode 100644
index 000000000..c6f4b3af1
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/types.ts
@@ -0,0 +1,150 @@
+import { z } from "zod";
+import type {
+ ThreatDecisionRule,
+ ThreatProvider,
+} from "../threat-protection/types";
+
+export const auditMetadataSchema = z
+ .record(z.string().min(1).max(64), z.string().max(1024))
+ .refine(value => Object.keys(value).length <= 32, {
+ message: "auditMetadata may contain at most 32 fields",
+ });
+
+export type AuditMetadata = z.infer<typeof auditMetadataSchema>;
+
+const azureSentinelDestinationInputSchema = z.strictObject({
+ type: z.literal("azure_sentinel"),
+ tenantId: z.string().trim().min(1).max(128),
+ clientId: z.string().trim().min(1).max(128),
+ clientSecret: z.string().trim().min(1).max(4096).optional(),
+ dceUrl: z
+ .string()
+ .trim()
+ .pipe(
+ z.url().refine(value => {
+ const url = new URL(value);
+ return (
+ url.protocol === "https:" &&
+ url.username === "" &&
+ url.password === "" &&
+ (url.hostname === "ingest.monitor.azure.com" ||
+ url.hostname.endsWith(".ingest.monitor.azure.com"))
+ );
+ }, "dceUrl must be a credential-free Azure Monitor ingestion endpoint"),
+ ),
+ dcrImmutableId: z.string().trim().min(1).max(256),
+ streamName: z
+ .string()
+ .trim()
+ .min(1)
+ .max(256)
+ .refine(value => value.startsWith("Custom-"), {
+ message: "streamName must start with Custom-",
+ }),
+});
+
+export const siemLoggingConfigInputSchema = z.strictObject({
+ enabled: z.boolean(),
+ destination: azureSentinelDestinationInputSchema,
+});
+
+export type SiemLoggingConfigInput = z.infer<
+ typeof siemLoggingConfigInputSchema
+>;
+
+export interface AzureSentinelDestination {
+ type: "azure_sentinel";
+ tenantId: string;
+ clientId: string;
+ clientSecret: string;
+ dceUrl: string;
+ dcrImmutableId: string;
+ streamName: string;
+}
+
+export interface OrgSiemLoggingConfig {
+ orgId: string;
+ enabled: boolean;
+ destination: AzureSentinelDestination;
+ createdAt: string | null;
+ updatedAt: string | null;
+}
+
+export type ScrapeActivityResult =
+ | "success"
+ | "failure"
+ | "blocked"
+ | "cancelled";
+
+export interface ScrapeActivityThreat {
+ decision: "allow" | "deny";
+ rule: ThreatDecisionRule;
+ provider: ThreatProvider | null;
+ categories: string[];
+ security_alert: {
+ detected: boolean;
+ category: string | null;
+ };
+}
+
+export interface ScrapeActivityEvent {
+ schema_version: 1;
+ event_type: "scrape_activity";
+ scrape_id: string;
+ request_id: string;
+ endpoint:
+ | "scrape"
+ | "crawl"
+ | "batch_scrape"
+ | "search"
+ | "extract"
+ | "agent"
+ | "parse"
+ | "unknown";
+ team_id: string;
+ org_id: string;
+ api_key: {
+ id: string | null;
+ name: string | null;
+ };
+ audit_metadata: AuditMetadata;
+ started_at: string;
+ completed_at: string;
+ url: string;
+ domain: string;
+ http_method: "GET";
+ http_status: number | null;
+ result: ScrapeActivityResult;
+ error: {
+ code: string | null;
+ message: string;
+ } | null;
+ origin: string;
+ integration: string | null;
+ zero_data_retention: boolean;
+ threat?: ScrapeActivityThreat;
+}
+
+export interface SiemLoggingJobData {
+ orgId: string;
+ event: ScrapeActivityEvent;
+}
+
+type SiemDeliveryErrorKind =
+ | "invalid_credentials"
+ | "schema_rejection"
+ | "rate_limited"
+ | "delivery_error"
+ | "payload_too_large";
+
+export class SiemDeliveryError extends Error {
+ constructor(
+ public readonly kind: SiemDeliveryErrorKind,
+ message: string,
+ public readonly statusCode?: number,
+ public readonly retryAfterMs?: number,
+ ) {
+ super(message);
+ this.name = "SiemDeliveryError";
+ }
+}
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 3cc6ef36e..a2767c5f2 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -70,6 +70,11 @@ import {
getTeamThreatProtectionController,
putTeamThreatProtectionController,
} from "../controllers/v2/team-threat-protection";
+import {
+ getTeamSiemLoggingController,
+ putTeamSiemLoggingController,
+ testTeamSiemLoggingController,
+} from "../controllers/v2/team-siem-logging";
import { supportProxyController } from "../controllers/v2/support-proxy";
import { createResearchRouter } from "../controllers/v2/research-proxy";
import {
@@ -439,6 +444,24 @@ v2Router.put(
wrap(putTeamThreatProtectionController),
);
+v2Router.get(
+ "/team/siem",
+ authMiddleware(RateLimiterMode.Account),
+ wrap(getTeamSiemLoggingController),
+);
+
+v2Router.put(
+ "/team/siem",
+ authMiddleware(RateLimiterMode.Account),
+ wrap(putTeamSiemLoggingController),
+);
+
+v2Router.post(
+ "/team/siem/test",
+ authMiddleware(RateLimiterMode.Account),
+ wrap(testTeamSiemLoggingController),
+);
+
v2Router.post(
"/monitor",
authMiddleware(RateLimiterMode.Crawl),
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 4858328ab..5bb2efa9a 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -9,6 +9,7 @@ import {
getRedisConnection,
getBillingQueue,
getPrecrawlQueue,
+ getSiemLoggingQueue,
precrawlQueueName,
} from "../queue-service";
import {
@@ -42,6 +43,7 @@ import { crawlGroup, resolveNewGroupBackend } from "../worker/nuq-router";
import { getACUCTeam } from "../../controllers/auth";
import { processEngpickerJob } from "../../lib/engpicker";
import { logRequest } from "../logging/log_job";
+import { processSiemLoggingJob } from "../siem-logging/worker";
const workerLockDuration = config.WORKER_LOCK_DURATION;
const workerStalledCheckInterval = config.WORKER_STALLED_CHECK_INTERVAL;
@@ -698,6 +700,10 @@ const BROWSER_ACTIVITY_INSERT_INTERVAL = 10000;
: (async () => {
logger.warn("PRECRAWL_TEAM_ID not set, skipping precrawl worker");
})();
+ const siemLoggingWorkerPromise = workerFun(
+ getSiemLoggingQueue(),
+ processSiemLoggingJob,
+ );
const indexInserterInterval = setInterval(async () => {
if (isShuttingDown) {
@@ -779,6 +785,7 @@ const BROWSER_ACTIVITY_INSERT_INTERVAL = 10000;
await Promise.all([
billingWorkerPromise,
precrawlWorkerPromise,
+ siemLoggingWorkerPromise,
engpickerPromise,
]);
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index 04dc0b263..1c5bf0ccd 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -3,6 +3,7 @@ import { config } from "../config";
import { logger } from "../lib/logger";
import IORedis from "ioredis";
import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";
+import type { SiemLoggingJobData } from "../lib/siem-logging/types";
import { addExtractJob, ExtractJobData } from "./extract-queue";
let loggingQueue: Queue;
@@ -11,6 +12,7 @@ let deepResearchQueue: Queue;
let generateLlmsTxtQueue: Queue;
let billingQueue: Queue;
let precrawlQueue: Queue;
+let siemLoggingQueue: Queue<SiemLoggingJobData>;
let redisConnection: IORedis;
export function getRedisConnection(): IORedis {
@@ -29,6 +31,7 @@ const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";
const deepResearchQueueName = "{deepResearchQueue}";
const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
+const siemLoggingQueueName = "{siemLoggingQueue}";
export async function addExtractJobToQueue(
extractId: string,
@@ -107,3 +110,25 @@ export function getPrecrawlQueue() {
}
return precrawlQueue;
}
+
+export function getSiemLoggingQueue(): Queue<SiemLoggingJobData> {
+ if (!siemLoggingQueue) {
+ siemLoggingQueue = new Queue<SiemLoggingJobData>(siemLoggingQueueName, {
+ connection: getRedisConnection(),
+ defaultJobOptions: {
+ attempts: 8,
+ backoff: {
+ type: "exponential",
+ delay: 1000,
+ },
+ removeOnComplete: {
+ age: 60,
+ },
+ removeOnFail: {
+ age: 24 * 60 * 60,
+ },
+ },
+ });
+ }
+ return siemLoggingQueue;
+}
diff --git a/apps/api/src/services/siem-logging/worker.test.ts b/apps/api/src/services/siem-logging/worker.test.ts
new file mode 100644
index 000000000..e83e2a57b
--- /dev/null
+++ b/apps/api/src/services/siem-logging/worker.test.ts
@@ -0,0 +1,147 @@
+import type { Job } from "bullmq";
+import { describe, expect, it, vi } from "vitest";
+import {
+ SiemDeliveryError,
+ type OrgSiemLoggingConfig,
+ type ScrapeActivityEvent,
+ type SiemLoggingJobData,
+} from "../../lib/siem-logging/types";
+import { processSiemLoggingJob } from "./worker";
+
+const event: ScrapeActivityEvent = {
+ schema_version: 1,
+ event_type: "scrape_activity",
+ scrape_id: "scrape-id",
+ request_id: "request-id",
+ endpoint: "scrape",
+ team_id: "team-id",
+ org_id: "org-id",
+ api_key: { id: null, name: null },
+ audit_metadata: {},
+ started_at: "2026-07-27T00:00:00.000Z",
+ completed_at: "2026-07-27T00:00:01.000Z",
+ url: "https://example.com",
+ domain: "example.com",
+ http_method: "GET",
+ http_status: 200,
+ result: "success",
+ error: null,
+ origin: "api",
+ integration: null,
+ zero_data_retention: false,
+};
+
+const config: OrgSiemLoggingConfig = {
+ orgId: "org-id",
+ enabled: true,
+ destination: {
+ type: "azure_sentinel",
+ tenantId: "tenant",
+ clientId: "client",
+ clientSecret: "secret",
+ dceUrl: "https://example.eastus.ingest.monitor.azure.com",
+ dcrImmutableId: "dcr-id",
+ streamName: "Custom-FirecrawlScrapeActivity",
+ },
+ createdAt: null,
+ updatedAt: null,
+};
+
+function createJob() {
+ return {
+ id: "job-id",
+ data: { orgId: "org-id", event },
+ extendLock: vi.fn().mockResolvedValue(1),
+ moveToCompleted: vi.fn().mockResolvedValue(undefined),
+ moveToFailed: vi.fn().mockResolvedValue(undefined),
+ discard: vi.fn(),
+ attemptsMade: 0,
+ opts: {
+ attempts: 8,
+ backoff: { type: "exponential", delay: 1000 },
+ },
+ } as unknown as Job<SiemLoggingJobData>;
+}
+
+describe("SIEM logging worker", () => {
+ it("loads the current secret at delivery time and completes the job", async () => {
+ const job = createJob();
+ const deliver = vi.fn().mockResolvedValue(undefined);
+
+ await processSiemLoggingJob("token", job, {
+ getConfig: vi.fn().mockResolvedValue(config),
+ deliver,
+ });
+
+ expect(deliver).toHaveBeenCalledWith(config.destination, [event]);
+ expect(job.moveToCompleted).toHaveBeenCalledWith(
+ { delivered: true },
+ "token",
+ false,
+ );
+ expect(job.moveToFailed).not.toHaveBeenCalled();
+ });
+
+ it("skips queued events after logging is disabled", async () => {
+ const job = createJob();
+ const deliver = vi.fn();
+
+ await processSiemLoggingJob("token", job, {
+ getConfig: vi.fn().mockResolvedValue({ ...config, enabled: false }),
+ deliver,
+ });
+
+ expect(deliver).not.toHaveBeenCalled();
+ expect(job.moveToCompleted).toHaveBeenCalledWith(
+ { skipped: true },
+ "token",
+ false,
+ );
+ });
+
+ it("leaves transient delivery failures retryable", async () => {
+ const job = createJob();
+
+ await processSiemLoggingJob("token", job, {
+ getConfig: vi.fn().mockResolvedValue(config),
+ deliver: vi
+ .fn()
+ .mockRejectedValue(new SiemDeliveryError("rate_limited", "retry")),
+ });
+
+ expect(job.discard).not.toHaveBeenCalled();
+ expect(job.moveToFailed).toHaveBeenCalledOnce();
+ });
+
+ it("does not retry before the provider's Retry-After delay", async () => {
+ const job = createJob();
+
+ await processSiemLoggingJob("token", job, {
+ getConfig: vi.fn().mockResolvedValue(config),
+ deliver: vi
+ .fn()
+ .mockRejectedValue(
+ new SiemDeliveryError("rate_limited", "retry", 429, 15_000),
+ ),
+ });
+
+ expect(job.opts.backoff).toEqual({ type: "fixed", delay: 15_000 });
+ expect(job.moveToFailed).toHaveBeenCalledOnce();
+ });
+
+ it("discards permanent destination failures", async () => {
+ const job = createJob();
+
+ await processSiemLoggingJob("token", job, {
+ getConfig: vi.fn().mockResolvedValue(config),
+ deliver: vi
+ .fn()
+ .mockRejectedValue(
+ new SiemDeliveryError("invalid_credentials", "invalid"),
+ ),
+ });
+
+ expect(job.discard).toHaveBeenCalledOnce();
+ expect(job.moveToFailed).toHaveBeenCalledOnce();
+ });
+});
diff --git a/apps/api/src/services/siem-logging/worker.ts b/apps/api/src/services/siem-logging/worker.ts
new file mode 100644
index 000000000..c3075779e
--- /dev/null
+++ b/apps/api/src/services/siem-logging/worker.ts
@@ -0,0 +1,121 @@
+import type { Job } from "bullmq";
+import { config } from "../../config";
+import { sendAzureSentinelEvents } from "../../lib/siem-logging/azure-sentinel";
+import {
+ siemLoggingDeliveryFailuresTotal,
+ siemLoggingEventsTotal,
+} from "../../lib/siem-logging/metrics";
+import { getOrgSiemLoggingConfig } from "../../lib/siem-logging/store";
+import {
+ SiemDeliveryError,
+ type AzureSentinelDestination,
+ type OrgSiemLoggingConfig,
+ type ScrapeActivityEvent,
+ type SiemLoggingJobData,
+} from "../../lib/siem-logging/types";
+import { logger as _logger } from "../../lib/logger";
+
+interface WorkerDependencies {
+ getConfig: (orgId: string) => Promise<OrgSiemLoggingConfig | null>;
+ deliver: (
+ destination: AzureSentinelDestination,
+ events: ScrapeActivityEvent[],
+ ) => Promise<void>;
+}
+
+const defaultDependencies: WorkerDependencies = {
+ getConfig: orgId =>
+ getOrgSiemLoggingConfig(orgId, { fresh: true, primary: true }),
+ deliver: sendAzureSentinelEvents,
+};
+
+const permanentFailureKinds = new Set([
+ "invalid_credentials",
+ "schema_rejection",
+ "payload_too_large",
+]);
+
+function asError(error: unknown): Error {
+ return error instanceof Error ? error : new Error(String(error));
+}
+
+function configuredRetryDelay(job: Job<SiemLoggingJobData>): number {
+ const backoff = job.opts.backoff;
+ if (typeof backoff === "number") return backoff;
+ if (!backoff) return 0;
+ const delay = backoff.delay ?? 0;
+ if (backoff.type === "exponential") {
+ return delay * 2 ** job.attemptsMade;
+ }
+ return delay;
+}
+
+function applyRetryAfter(
+ job: Job<SiemLoggingJobData>,
+ error: SiemDeliveryError,
+): void {
+ if (
+ error.kind !== "rate_limited" ||
+ error.retryAfterMs === undefined ||
+ error.retryAfterMs <= 0
+ ) {
+ return;
+ }
+ job.opts.backoff = {
+ type: "fixed",
+ delay: Math.max(configuredRetryDelay(job), error.retryAfterMs),
+ };
+}
+
+export async function processSiemLoggingJob(
+ token: string,
+ job: Job<SiemLoggingJobData>,
+ deps: WorkerDependencies = defaultDependencies,
+): Promise<void> {
+ const logger = _logger.child({
+ module: "siem-logging-worker",
+ jobId: job.id,
+ orgId: job.data.orgId,
+ });
+ const extendLockInterval = setInterval(() => {
+ void job.extendLock(token, config.JOB_LOCK_EXTENSION_TIME).catch(error => {
+ logger.warn("Failed to extend SIEM logging job lock", { error });
+ });
+ }, config.JOB_LOCK_EXTEND_INTERVAL);
+ extendLockInterval.unref?.();
+
+ try {
+ const currentConfig = await deps.getConfig(job.data.orgId);
+ if (!currentConfig?.enabled) {
+ siemLoggingEventsTotal.inc({ result: "skipped_disabled" });
+ await job.moveToCompleted({ skipped: true }, token, false);
+ return;
+ }
+
+ await deps.deliver(currentConfig.destination, [job.data.event]);
+ siemLoggingEventsTotal.inc({ result: "delivered" });
+ await job.moveToCompleted({ delivered: true }, token, false);
+ } catch (error) {
+ const reason =
+ error instanceof SiemDeliveryError ? error.kind : "delivery_error";
+ siemLoggingEventsTotal.inc({ result: "delivery_failed" });
+ siemLoggingDeliveryFailuresTotal.inc({ reason });
+ logger.error("SIEM logging event delivery failed", {
+ error,
+ reason,
+ scrapeId: job.data.event.scrape_id,
+ });
+ if (
+ error instanceof SiemDeliveryError &&
+ permanentFailureKinds.has(error.kind)
+ ) {
+ job.discard();
+ }
+ if (error instanceof SiemDeliveryError) {
+ applyRetryAfter(job, error);
+ }
+ await job.moveToFailed(asError(error), token, false);
+ } finally {
+ clearInterval(extendLockInterval);
+ }
+}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index bfabe4ead..fb62e9d90 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -104,6 +104,7 @@ import {
warmExchangeCatalog,
type ExchangeScrapeMetadata,
} from "../../lib/exchange";
+import { emitScrapeActivityEvent } from "../../lib/siem-logging";
configDotenv();
@@ -836,9 +837,25 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
}
}
+ emitScrapeActivityEvent(job.id, job.data, {
+ success: true,
+ document: doc,
+ threatDecisions: pipeline.threatDecisions,
+ startedAt: start,
+ completedAt: Date.now(),
+ });
+
logger.info(`🐂 Job done ${job.id}`);
return data;
} catch (error) {
+ emitScrapeActivityEvent(job.id, job.data, {
+ success: false,
+ error,
+ threatDecisions: pipeline?.threatDecisions,
+ startedAt: start,
+ completedAt: Date.now(),
+ });
+
// Record top-level robots.txt rejections so crawl status can warn
try {
if (
diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
index 3ab23a069..3d80e6503 100644
--- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
+++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
@@ -17,7 +17,8 @@ public class ModelsTests
OnlyMainContent = true,
Timeout = 30000,
Mobile = false,
- RedactPII = true
+ RedactPII = true,
+ AuditMetadata = new Dictionary<string, string> { ["requestId"] = "req-123" }
};
var json = JsonSerializer.Serialize(options, JsonOptions);
@@ -28,6 +29,7 @@ public class ModelsTests
Assert.Contains("\"timeout\":30000", json);
Assert.Contains("\"mobile\":false", json);
Assert.Contains("\"redactPII\":true", json);
+ Assert.Contains("\"auditMetadata\":{\"requestId\":\"req-123\"}", json);
}
[Fact]
@@ -70,13 +72,15 @@ public class ModelsTests
{
Search = "pricing",
Limit = 10,
- IncludeSubdomains = true
+ IncludeSubdomains = true,
+ AuditMetadata = new Dictionary<string, string> { ["requestId"] = "req-123" }
};
var json = JsonSerializer.Serialize(options, JsonOptions);
Assert.Contains("\"search\":\"pricing\"", json);
Assert.Contains("\"limit\":10", json);
Assert.Contains("\"includeSubdomains\":true", json);
+ Assert.Contains("\"auditMetadata\":{\"requestId\":\"req-123\"}", json);
}
[Fact]
diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs
index 00f145294..8708c2b86 100644
--- a/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs
+++ b/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs
@@ -57,6 +57,7 @@ public class ParseTests
Timeout = 30000,
Proxy = "auto",
RedactPII = true,
+ AuditMetadata = new Dictionary<string, string> { ["requestId"] = "req-123" },
};
var json = JsonSerializer.Serialize(options, FirecrawlHttpClient.JsonOptions);
@@ -65,6 +66,7 @@ public class ParseTests
Assert.Contains("\"timeout\":30000", json);
Assert.Contains("\"proxy\":\"auto\"", json);
Assert.Contains("\"redactPII\":true", json);
+ Assert.Contains("\"auditMetadata\":{\"requestId\":\"req-123\"}", json);
}
[Fact]
diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
index b84ed222a..56c8588de 100644
--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
@@ -8,7 +8,7 @@
<!-- NuGet package metadata -->
<PackageId>firecrawl-sdk</PackageId>
- <Version>1.9.1</Version>
+ <Version>1.10.0</Version>
<Authors>Firecrawl</Authors>
<Company>Firecrawl</Company>
<Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>
diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
index c7210428e..0fa4f9729 100644
--- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
@@ -714,7 +714,7 @@ public class FirecrawlClient
// INTERNAL UTILITIES
// ================================================================
- private const string SdkOrigin = "[email protected]";
+ private const string SdkOrigin = "[email protected]";
private static Dictionary<string, object> BuildBody(object? options)
{
diff --git a/apps/dot-net-sdk/Firecrawl/Models/MapOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/MapOptions.cs
index 912c628b7..8b33ec39c 100644
--- a/apps/dot-net-sdk/Firecrawl/Models/MapOptions.cs
+++ b/apps/dot-net-sdk/Firecrawl/Models/MapOptions.cs
@@ -42,4 +42,8 @@ public class MapOptions
[JsonPropertyName("ignoreCache")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? IgnoreCache { get; set; }
+
+ [JsonPropertyName("auditMetadata")]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public Dictionary<string, string>? AuditMetadata { get; set; }
}
diff --git a/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs
index a77fb9dc7..b1c92bc6f 100644
--- a/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs
+++ b/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs
@@ -83,6 +83,10 @@ public class ParseOptions
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? RedactPII { get; set; }
+ [JsonPropertyName("auditMetadata")]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public Dictionary<string, string>? AuditMetadata { get; set; }
+
/// <summary>
/// Validate the options against /v2/parse's supported surface.
/// </summary>
diff --git a/apps/dot-net-sdk/Firecrawl/Models/ScrapeOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/ScrapeOptions.cs
index 6dbc2612f..4496de609 100644
--- a/apps/dot-net-sdk/Firecrawl/Models/ScrapeOptions.cs
+++ b/apps/dot-net-sdk/Firecrawl/Models/ScrapeOptions.cs
@@ -79,6 +79,10 @@ public class ScrapeOptions
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? RedactPII { get; set; }
+ [JsonPropertyName("auditMetadata")]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public Dictionary<string, string>? AuditMetadata { get; set; }
+
[JsonPropertyName("integration")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Integration { get; set; }
diff --git a/apps/elixir-sdk/generate.exs b/apps/elixir-sdk/generate.exs
index ab781bdc5..54ba3f095 100644
--- a/apps/elixir-sdk/generate.exs
+++ b/apps/elixir-sdk/generate.exs
@@ -145,7 +145,7 @@ defmodule Firecrawl.Generator do
@type response :: {:ok, Req.Response.t()} | {:error, Exception.t() | Firecrawl.Error.t()}
@base_url "#{base_url}"
- @sdk_origin "[email protected]"
+ @sdk_origin "[email protected]"
defp client(opts) do
api_key =
diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.ex
index 760957ce5..e390b6cc9 100644
--- a/apps/elixir-sdk/lib/firecrawl.ex
+++ b/apps/elixir-sdk/lib/firecrawl.ex
@@ -41,7 +41,7 @@ defmodule Firecrawl do
@type response :: {:ok, Req.Response.t()} | {:error, Exception.t() | Firecrawl.Error.t()}
@base_url "https://api.firecrawl.dev/v2"
- @sdk_origin "[email protected]"
+ @sdk_origin "[email protected]"
defp client(opts) do
api_key =
@@ -867,6 +867,7 @@ defmodule Firecrawl do
@map_urls_schema NimbleOptions.new!([
+ audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],
ignore_cache: [type: :boolean, doc: "Bypass the sitemap cache to retrieve fresh URLs. Sitemap data is cached for up to 7 days; use this parameter when your sitemap has been recently updated."],
ignore_query_parameters: [type: :boolean, doc: "Do not return URLs with query parameters"],
include_subdomains: [type: :boolean, doc: "Include subdomains of the website"],
@@ -878,7 +879,7 @@ defmodule Firecrawl do
url: [type: :string, required: true, doc: "The base URL to start crawling from"]
])
- @map_urls_key_mapping %{ignore_cache: "ignoreCache", ignore_query_parameters: "ignoreQueryParameters", include_subdomains: "includeSubdomains", limit: "limit", location: "location", search: "search", sitemap: "sitemap", timeout: "timeout", url: "url"}
+ @map_urls_key_mapping %{audit_metadata: "auditMetadata", ignore_cache: "ignoreCache", ignore_query_parameters: "ignoreQueryParameters", include_subdomains: "includeSubdomains", limit: "limit", location: "location", search: "search", sitemap: "sitemap", timeout: "timeout", url: "url"}
@doc """
Map multiple URLs based on options
@@ -916,6 +917,7 @@ defmodule Firecrawl do
@parse_file_schema NimbleOptions.new!([
+ audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],
block_ads: [type: :boolean, doc: "Enable ad and cookie popup blocking."],
exclude_tags: [type: {:list, :string}, doc: "Tags to exclude from the output."],
formats: [type: {:list, :any}, doc: "Output formats supported for `/parse` uploads. Browser-rendering formats and change tracking are not supported."],
@@ -933,7 +935,7 @@ defmodule Firecrawl do
zero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this parse. To enable this feature, please contact [email protected]"]
])
- @parse_file_key_mapping %{block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", integration: "integration", only_main_content: "onlyMainContent", origin: "origin", parsers: "parsers", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", timeout: "timeout", zero_data_retention: "zeroDataRetention"}
+ @parse_file_key_mapping %{audit_metadata: "auditMetadata", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", integration: "integration", only_main_content: "onlyMainContent", origin: "origin", parsers: "parsers", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", timeout: "timeout", zero_data_retention: "zeroDataRetention"}
@doc """
Upload and parse a file
@@ -1017,6 +1019,7 @@ defmodule Firecrawl do
@scrape_and_extract_from_url_schema NimbleOptions.new!([
url: [type: :string, required: true, doc: "The URL to scrape"],
+ audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],
actions: [type: {:list, :any}, doc: "Actions to perform on the page before grabbing the content"],
block_ads: [type: :boolean, doc: "Enables ad-blocking and cookie popup blocking."],
exclude_tags: [type: {:list, :string}, doc: "Tags to exclude from the output."],
@@ -1041,7 +1044,7 @@ defmodule Firecrawl do
zero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this scrape. To enable this feature, please contact [email protected]"]
])
- @scrape_and_extract_from_url_key_mapping %{url: "url", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", lockdown: "lockdown", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}
+ @scrape_and_extract_from_url_key_mapping %{url: "url", audit_metadata: "auditMetadata", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", lockdown: "lockdown", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}
@doc """
Scrape a single URL and optionally extract information using an LLM
@@ -1079,6 +1082,7 @@ defmodule Firecrawl do
@scrape_and_extract_from_urls_schema NimbleOptions.new!([
+ audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],
ignore_invalid_urls: [type: :boolean, doc: "If invalid URLs are specified in the urls array, they will be ignored. Instead of them failing the entire request, a batch scrape using the remaining valid URLs will be created, and the invalid URLs will be returned in the invalidURLs field of the response."],
max_concurrency: [type: :integer, doc: "Maximum number of concurrent scrapes. This parameter allows you to set a concurrency limit for this batch scrape. If not specified, the batch scrape adheres to your team's concurrency limit."],
urls: [type: {:list, :string}, required: true],
@@ -1107,7 +1111,7 @@ defmodule Firecrawl do
zero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this batch scrape. To enable this feature, please contact [email protected]"]
])
- @scrape_and_extract_from_urls_key_mapping %{ignore_invalid_urls: "ignoreInvalidURLs", max_concurrency: "maxConcurrency", urls: "urls", webhook: "webhook", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", lockdown: "lockdown", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}
+ @scrape_and_extract_from_urls_key_mapping %{audit_metadata: "auditMetadata", ignore_invalid_urls: "ignoreInvalidURLs", max_concurrency: "maxConcurrency", urls: "urls", webhook: "webhook", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", lockdown: "lockdown", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}
@doc """
Scrape multiple URLs and optionally extract information using an LLM
@@ -1342,6 +1346,7 @@ defmodule Firecrawl do
@start_agent_schema NimbleOptions.new!([
+ audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],
max_credits: [type: {:or, [:integer, :float]}, doc: "Maximum credits to spend on this agent task. Defaults to 2500 if not set. Values above 2,500 are always billed as paid requests."],
model: [type: {:or, [{:in, [:"spark-1-mini", :"spark-1-pro"]}, :string]}, doc: "The model to use for the agent task. spark-1-mini (default) is 60% cheaper, spark-1-pro offers higher accuracy for complex tasks"],
prompt: [type: :string, required: true, doc: "The prompt describing what data to extract"],
@@ -1350,7 +1355,7 @@ defmodule Firecrawl do
urls: [type: {:list, :string}, doc: "Optional list of URLs to constrain the agent to"]
])
- @start_agent_key_mapping %{max_credits: "maxCredits", model: "model", prompt: "prompt", schema: "schema", strict_constrain_to_urls: "strictConstrainToURLs", urls: "urls"}
+ @start_agent_key_mapping %{audit_metadata: "auditMetadata", max_credits: "maxCredits", model: "model", prompt: "prompt", schema: "schema", strict_constrain_to_urls: "strictConstrainToURLs", urls: "urls"}
@doc """
Start an agent task for agentic data extraction
diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs
index 163203b4b..5013beee7 100644
--- a/apps/elixir-sdk/mix.exs
+++ b/apps/elixir-sdk/mix.exs
@@ -1,7 +1,7 @@
defmodule Firecrawl.MixProject do
use Mix.Project
- @version "1.8.1"
+ @version "1.9.0"
@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"
def project do
diff --git a/apps/elixir-sdk/test/firecrawl_test.exs b/apps/elixir-sdk/test/firecrawl_test.exs
index be8c4400e..4dcc3f4d3 100644
--- a/apps/elixir-sdk/test/firecrawl_test.exs
+++ b/apps/elixir-sdk/test/firecrawl_test.exs
@@ -317,6 +317,62 @@ defmodule FirecrawlTest do
assert body["urls"] == ["https://example.com"]
end
+ test "request endpoints map audit_metadata to auditMetadata" do
+ parent = self()
+
+ adapter = fn request ->
+ send(parent, {:request, request})
+
+ resp = Req.Response.new(
+ status: 200,
+ headers: %{"content-type" => ["application/json"]},
+ body: Jason.encode!(%{"success" => true, "id" => "job", "links" => []})
+ )
+
+ {request, resp}
+ end
+
+ metadata = %{"requestId" => "req-123"}
+
+ requests = [
+ fn ->
+ Firecrawl.scrape_and_extract_from_url(
+ [url: "https://example.com", audit_metadata: metadata],
+ api_key: "test-key",
+ adapter: adapter
+ )
+ end,
+ fn ->
+ Firecrawl.map_urls(
+ [url: "https://example.com", audit_metadata: metadata],
+ api_key: "test-key",
+ adapter: adapter
+ )
+ end,
+ fn ->
+ Firecrawl.start_agent(
+ [prompt: "find pricing", audit_metadata: metadata],
+ api_key: "test-key",
+ adapter: adapter
+ )
+ end
+ ]
+
+ Enum.each(requests, fn make_request ->
+ assert {:ok, %Req.Response{status: 200}} = make_request.()
+ assert_receive {:request, request}
+
+ body =
+ cond do
+ is_binary(request.body) -> Jason.decode!(request.body)
+ is_map(request.body) -> request.body
+ true -> request.options[:json]
+ end
+
+ assert body["auditMetadata"] == metadata
+ end)
+ end
+
test "search maps highlights to highlights" do
parent = self()
diff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.go
index c7022f57e..9cb22232f 100644
--- a/apps/go-sdk/options.go
+++ b/apps/go-sdk/options.go
@@ -91,6 +91,7 @@ type ScrapeOptions struct {
StoreInCache *bool `json:"storeInCache,omitempty"`
Lockdown *bool `json:"lockdown,omitempty"`
RedactPII *bool `json:"redactPII,omitempty"`
+ AuditMetadata map[string]string `json:"auditMetadata,omitempty"`
Integration *string `json:"integration,omitempty"`
JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`
}
@@ -150,14 +151,15 @@ type BatchScrapeOptions struct {
// MapOptions configures a map (URL discovery) request.
type MapOptions struct {
- Search *string `json:"search,omitempty"`
- Sitemap *string `json:"sitemap,omitempty"`
- IncludeSubdomains *bool `json:"includeSubdomains,omitempty"`
- IgnoreQueryParameters *bool `json:"ignoreQueryParameters,omitempty"`
- Limit *int `json:"limit,omitempty"`
- Timeout *int `json:"timeout,omitempty"`
- Integration *string `json:"integration,omitempty"`
- Location *LocationConfig `json:"location,omitempty"`
+ Search *string `json:"search,omitempty"`
+ Sitemap *string `json:"sitemap,omitempty"`
+ IncludeSubdomains *bool `json:"includeSubdomains,omitempty"`
+ IgnoreQueryParameters *bool `json:"ignoreQueryParameters,omitempty"`
+ Limit *int `json:"limit,omitempty"`
+ Timeout *int `json:"timeout,omitempty"`
+ Integration *string `json:"integration,omitempty"`
+ Location *LocationConfig `json:"location,omitempty"`
+ AuditMetadata map[string]string `json:"auditMetadata,omitempty"`
}
// SearchOptions configures a search request.
@@ -186,6 +188,7 @@ type AgentOptions struct {
StrictConstrainToURLs *bool `json:"strictConstrainToURLs,omitempty"`
Model *string `json:"model,omitempty"`
Webhook *WebhookConfig `json:"webhook,omitempty"`
+ AuditMetadata map[string]string `json:"auditMetadata,omitempty"`
}
// LocationConfig specifies geolocation for requests.
diff --git a/apps/go-sdk/options_test.go b/apps/go-sdk/options_test.go
index d41057c2d..4afbb89a7 100644
--- a/apps/go-sdk/options_test.go
+++ b/apps/go-sdk/options_test.go
@@ -87,3 +87,23 @@ func TestSearchOptionsSerializesHighlights(t *testing.T) {
t.Fatalf("serialized search options = %s", payload)
}
}
+
+func TestAuditMetadataSerializesAcrossRequestOptions(t *testing.T) {
+ metadata := map[string]string{"requestId": "req-123"}
+ for name, options := range map[string]interface{}{
+ "scrape": ScrapeOptions{AuditMetadata: metadata},
+ "map": MapOptions{AuditMetadata: metadata},
+ "agent": AgentOptions{Prompt: "find pricing", AuditMetadata: metadata},
+ "parse": ParseOptions{AuditMetadata: metadata},
+ } {
+ t.Run(name, func(t *testing.T) {
+ payload, err := json.Marshal(options)
+ if err != nil {
+ t.Fatalf("Marshal options: %v", err)
+ }
+ if !strings.Contains(string(payload), `"auditMetadata":{"requestId":"req-123"}`) {
+ t.Fatalf("serialized options = %s", payload)
+ }
+ })
+ }
+}
diff --git a/apps/go-sdk/parse.go b/apps/go-sdk/parse.go
index 57ca32806..6aacae6d3 100644
--- a/apps/go-sdk/parse.go
+++ b/apps/go-sdk/parse.go
@@ -71,6 +71,7 @@ type ParseOptions struct {
Proxy *string `json:"proxy,omitempty"`
Integration *string `json:"integration,omitempty"`
RedactPII *bool `json:"redactPII,omitempty"`
+ AuditMetadata map[string]string `json:"auditMetadata,omitempty"`
JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`
}
diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go
index 79775e6d8..31f90dbfe 100644
--- a/apps/go-sdk/version.go
+++ b/apps/go-sdk/version.go
@@ -9,4 +9,4 @@ package firecrawl
// Bump this when preparing a new release. The publish-go-sdk GitHub workflow
// reads this value and creates the corresponding monorepo-prefixed tag on
// merge to main.
-const Version = "1.8.1"
+const Version = "1.9.0"
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts
index 0310669b8..b30daeb42 100644
--- a/apps/java-sdk/build.gradle.kts
+++ b/apps/java-sdk/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
}
group = "com.firecrawl"
-version = "1.11.1"
+version = "1.12.0"
java {
sourceCompatibility = JavaVersion.VERSION_11
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
index a2b43e53b..c605b3da4 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
@@ -37,7 +37,7 @@ import java.util.concurrent.ForkJoinPool;
public class FirecrawlClient {
private static final String DEFAULT_API_URL = "https://api.firecrawl.dev";
- private static final String SDK_ORIGIN = "[email protected]";
+ private static final String SDK_ORIGIN = "[email protected]";
private static final long DEFAULT_TIMEOUT_MS = 300_000; // 5 minutes
private static final int DEFAULT_MAX_RETRIES = 3;
private static final double DEFAULT_BACKOFF_FACTOR = 0.5;
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/AgentOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/AgentOptions.java
index e0e25fd62..c2873512b 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/AgentOptions.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/AgentOptions.java
@@ -1,6 +1,9 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -18,6 +21,7 @@ public class AgentOptions {
private Boolean strictConstrainToURLs;
private String model;
private WebhookConfig webhook;
+ private Map<String, String> auditMetadata;
private AgentOptions() {}
@@ -29,6 +33,8 @@ public class AgentOptions {
public Boolean getStrictConstrainToURLs() { return strictConstrainToURLs; }
public String getModel() { return model; }
public WebhookConfig getWebhook() { return webhook; }
+ @JsonProperty("auditMetadata")
+ public Map<String, String> getAuditMetadata() { return auditMetadata; }
public static Builder builder() { return new Builder(); }
@@ -41,6 +47,7 @@ public class AgentOptions {
private Boolean strictConstrainToURLs;
private String model;
private WebhookConfig webhook;
+ private Map<String, String> auditMetadata;
private Builder() {}
@@ -60,6 +67,8 @@ public class AgentOptions {
public Builder model(String model) { this.model = model; return this; }
/** Webhook configuration. */
public Builder webhook(WebhookConfig webhook) { this.webhook = webhook; return this; }
+ /** Metadata to include with SIEM logging events. */
+ public Builder auditMetadata(Map<String, String> auditMetadata) { this.auditMetadata = auditMetadata; return this; }
public AgentOptions build() {
if (prompt == null || prompt.isEmpty()) {
@@ -74,6 +83,9 @@ public class AgentOptions {
o.strictConstrainToURLs = this.strictConstrainToURLs;
o.model = this.model;
o.webhook = this.webhook;
+ o.auditMetadata = this.auditMetadata != null
+ ? Collections.unmodifiableMap(new HashMap<>(this.auditMetadata))
+ : null;
return o;
}
}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MapOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MapOptions.java
index 06940939d..b4c14ce47 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/MapOptions.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MapOptions.java
@@ -1,6 +1,10 @@
package com.firecrawl.models;
import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
/**
* Options for mapping (discovering URLs on) a website.
@@ -16,6 +20,7 @@ public class MapOptions {
private Integer timeout;
private String integration;
private LocationConfig location;
+ private Map<String, String> auditMetadata;
private MapOptions() {}
@@ -28,6 +33,8 @@ public class MapOptions {
public Integer getTimeout() { return timeout; }
public String getIntegration() { return integration; }
public LocationConfig getLocation() { return location; }
+ @JsonProperty("auditMetadata")
+ public Map<String, String> getAuditMetadata() { return auditMetadata; }
public static Builder builder() { return new Builder(); }
@@ -40,6 +47,7 @@ public class MapOptions {
private Integer timeout;
private String integration;
private LocationConfig location;
+ private Map<String, String> auditMetadata;
private Builder() {}
@@ -59,6 +67,8 @@ public class MapOptions {
public Builder integration(String integration) { this.integration = integration; return this; }
/** Geolocation configuration. */
public Builder location(LocationConfig location) { this.location = location; return this; }
+ /** Metadata to include with SIEM logging events. */
+ public Builder auditMetadata(Map<String, String> auditMetadata) { this.auditMetadata = auditMetadata; return this; }
public MapOptions build() {
MapOptions o = new MapOptions();
@@ -70,6 +80,9 @@ public class MapOptions {
o.timeout = this.timeout;
o.integration = this.integration;
o.location = this.location;
+ o.auditMetadata = this.auditMetadata != null
+ ? Collections.unmodifiableMap(new HashMap<>(this.auditMetadata))
+ : null;
return o;
}
}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java
index 079bc29d4..c8094b264 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java
@@ -31,6 +31,7 @@ public class ParseOptions {
private String integration;
@JsonProperty("redactPII")
private Boolean redactPII;
+ private Map<String, String> auditMetadata;
private ParseOptions() {}
@@ -48,6 +49,8 @@ public class ParseOptions {
public String getIntegration() { return integration; }
@JsonProperty("redactPII")
public Boolean getRedactPII() { return redactPII; }
+ @JsonProperty("auditMetadata")
+ public Map<String, String> getAuditMetadata() { return auditMetadata; }
public static Builder builder() { return new Builder(); }
@@ -66,6 +69,7 @@ public class ParseOptions {
b.proxy = this.proxy;
b.integration = this.integration;
b.redactPII = this.redactPII;
+ b.auditMetadata = this.auditMetadata != null ? new HashMap<>(this.auditMetadata) : null;
return b;
}
@@ -113,6 +117,7 @@ public class ParseOptions {
private String proxy;
private String integration;
private Boolean redactPII;
+ private Map<String, String> auditMetadata;
private Builder() {}
@@ -129,6 +134,7 @@ public class ParseOptions {
public Builder proxy(String proxy) { this.proxy = proxy; return this; }
public Builder integration(String integration) { this.integration = integration; return this; }
public Builder redactPII(Boolean redactPII) { this.redactPII = redactPII; return this; }
+ public Builder auditMetadata(Map<String, String> auditMetadata) { this.auditMetadata = auditMetadata; return this; }
public ParseOptions build() {
if (timeout != null && timeout <= 0) {
@@ -162,6 +168,7 @@ public class ParseOptions {
o.proxy = this.proxy;
o.integration = this.integration;
o.redactPII = this.redactPII;
+ o.auditMetadata = this.auditMetadata != null ? Collections.unmodifiableMap(new HashMap<>(this.auditMetadata)) : null;
return o;
}
}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
index a1889f50e..e36050148 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
@@ -35,6 +35,7 @@ public class ScrapeOptions {
private Boolean lockdown;
@JsonProperty("redactPII")
private Boolean redactPII;
+ private Map<String, String> auditMetadata;
private String integration;
private ScrapeOptions() {}
@@ -59,6 +60,8 @@ public class ScrapeOptions {
public Boolean getLockdown() { return lockdown; }
@JsonProperty("redactPII")
public Boolean getRedactPII() { return redactPII; }
+ @JsonProperty("auditMetadata")
+ public Map<String, String> getAuditMetadata() { return auditMetadata; }
public String getIntegration() { return integration; }
public static Builder builder() { return new Builder(); }
@@ -84,6 +87,7 @@ public class ScrapeOptions {
b.storeInCache = this.storeInCache;
b.lockdown = this.lockdown;
b.redactPII = this.redactPII;
+ b.auditMetadata = this.auditMetadata != null ? new HashMap<>(this.auditMetadata) : null;
b.integration = this.integration;
return b;
}
@@ -108,6 +112,7 @@ public class ScrapeOptions {
private Boolean storeInCache;
private Boolean lockdown;
private Boolean redactPII;
+ private Map<String, String> auditMetadata;
private String integration;
private Builder() {}
@@ -173,6 +178,9 @@ public class ScrapeOptions {
/** Redact personally identifiable information from returned content. */
public Builder redactPII(Boolean redactPII) { this.redactPII = redactPII; return this; }
+ /** Metadata to include with SIEM logging events. */
+ public Builder auditMetadata(Map<String, String> auditMetadata) { this.auditMetadata = auditMetadata; return this; }
+
/** Integration identifier. */
public Builder integration(String integration) { this.integration = integration; return this; }
@@ -197,6 +205,7 @@ public class ScrapeOptions {
o.storeInCache = this.storeInCache;
o.lockdown = this.lockdown;
o.redactPII = this.redactPII;
+ o.auditMetadata = this.auditMetadata != null ? Collections.unmodifiableMap(new HashMap<>(this.auditMetadata)) : null;
o.integration = this.integration;
return o;
}
diff --git a/apps/java-sdk/src/test/java/com/firecrawl/AuditMetadataTest.java b/apps/java-sdk/src/test/java/com/firecrawl/AuditMetadataTest.java
new file mode 100644
index 000000000..354ed6b61
--- /dev/null
+++ b/apps/java-sdk/src/test/java/com/firecrawl/AuditMetadataTest.java
@@ -0,0 +1,54 @@
+package com.firecrawl;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.firecrawl.models.AgentOptions;
+import com.firecrawl.models.MapOptions;
+import com.firecrawl.models.ParseOptions;
+import com.firecrawl.models.ScrapeOptions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class AuditMetadataTest {
+
+ private final ObjectMapper mapper = new ObjectMapper();
+
+ @Test
+ void serializesAuditMetadataAcrossRequestOptions() {
+ Map<String, String> metadata = Map.of("requestId", "req-123");
+
+ assertAuditMetadata(ScrapeOptions.builder().auditMetadata(metadata).build());
+ assertAuditMetadata(MapOptions.builder().auditMetadata(metadata).build());
+ assertAuditMetadata(AgentOptions.builder().prompt("find pricing").auditMetadata(metadata).build());
+ assertAuditMetadata(ParseOptions.builder().auditMetadata(metadata).build());
+ }
+
+ @Test
+ void mapAndAgentOptionsDefensivelyCopyAuditMetadata() {
+ Map<String, String> metadata = new HashMap<>();
+ metadata.put("requestId", "req-123");
+
+ MapOptions map = MapOptions.builder().auditMetadata(metadata).build();
+ AgentOptions agent = AgentOptions.builder()
+ .prompt("find pricing")
+ .auditMetadata(metadata)
+ .build();
+ metadata.put("requestId", "mutated");
+
+ assertEquals("req-123", map.getAuditMetadata().get("requestId"));
+ assertEquals("req-123", agent.getAuditMetadata().get("requestId"));
+ assertThrows(UnsupportedOperationException.class,
+ () -> map.getAuditMetadata().put("requestId", "mutated"));
+ assertThrows(UnsupportedOperationException.class,
+ () -> agent.getAuditMetadata().put("requestId", "mutated"));
+ }
+
+ private void assertAuditMetadata(Object options) {
+ Map<?, ?> body = mapper.convertValue(options, Map.class);
+ assertEquals(Map.of("requestId", "req-123"), body.get("auditMetadata"));
+ }
+}
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 82128f2ec..c3fbe5b6c 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.30.1",
+ "version": "4.31.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/audit-metadata.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/audit-metadata.unit.test.ts
new file mode 100644
index 000000000..620fde00a
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/audit-metadata.unit.test.ts
@@ -0,0 +1,79 @@
+import { describe, test, expect, jest } from "@jest/globals";
+import { scrape } from "../../../v2/methods/scrape";
+import { startBatchScrape } from "../../../v2/methods/batch";
+import { startCrawl } from "../../../v2/methods/crawl";
+import { search } from "../../../v2/methods/search";
+import { map } from "../../../v2/methods/map";
+import { startExtract } from "../../../v2/methods/extract";
+import { startAgent } from "../../../v2/methods/agent";
+
+const auditMetadata = { requestId: "req-123" };
+
+function makeHttp(data: Record<string, unknown>) {
+ const post = jest.fn(async () => ({ status: 200, data }));
+ return {
+ post,
+ prepareHeaders: jest.fn(() => undefined),
+ } as any;
+}
+
+describe("v2 auditMetadata request serialization", () => {
+ test("scrape and batch scrape send metadata at the top level", async () => {
+ const scrapeHttp = makeHttp({ success: true, data: {} });
+ await scrape(scrapeHttp, "https://example.com", { auditMetadata });
+ expect(scrapeHttp.post.mock.calls[0][1]).toEqual(
+ expect.objectContaining({ auditMetadata }),
+ );
+
+ const batchHttp = makeHttp({ success: true, id: "job", url: "u" });
+ await startBatchScrape(batchHttp, ["https://example.com"], {
+ options: { auditMetadata },
+ });
+ expect(batchHttp.post.mock.calls[0][1]).toEqual(
+ expect.objectContaining({ auditMetadata }),
+ );
+ });
+
+ test("crawl, search, and extract send metadata under scrapeOptions", async () => {
+ const crawlHttp = makeHttp({ success: true, id: "job", url: "u" });
+ await startCrawl(crawlHttp, {
+ url: "https://example.com",
+ scrapeOptions: { auditMetadata },
+ });
+ expect(crawlHttp.post.mock.calls[0][1]).toEqual(
+ expect.objectContaining({ scrapeOptions: expect.objectContaining({ auditMetadata }) }),
+ );
+
+ const searchHttp = makeHttp({ success: true, data: {} });
+ await search(searchHttp, {
+ query: "example",
+ scrapeOptions: { auditMetadata },
+ });
+ expect(searchHttp.post.mock.calls[0][1]).toEqual(
+ expect.objectContaining({ scrapeOptions: expect.objectContaining({ auditMetadata }) }),
+ );
+
+ const extractHttp = makeHttp({ success: true, id: "job" });
+ await startExtract(extractHttp, {
+ urls: ["https://example.com"],
+ scrapeOptions: { auditMetadata },
+ });
+ expect(extractHttp.post.mock.calls[0][1]).toEqual(
+ expect.objectContaining({ scrapeOptions: expect.objectContaining({ auditMetadata }) }),
+ );
+ });
+
+ test("map and agent send metadata at the top level", async () => {
+ const mapHttp = makeHttp({ success: true, links: [] });
+ await map(mapHttp, "https://example.com", { auditMetadata });
+ expect(mapHttp.post.mock.calls[0][1]).toEqual(
+ expect.objectContaining({ auditMetadata }),
+ );
+
+ const agentHttp = makeHttp({ success: true, id: "job" });
+ await startAgent(agentHttp, { prompt: "find pricing", auditMetadata });
+ expect(agentHttp.post.mock.calls[0][1]).toEqual(
+ expect.objectContaining({ auditMetadata }),
+ );
+ });
+});
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts
index b293d10e4..665130d9b 100644
--- a/apps/js-sdk/firecrawl/src/v1/index.ts
+++ b/apps/js-sdk/firecrawl/src/v1/index.ts
@@ -135,6 +135,7 @@ export interface CrawlScrapeOptions {
storeInCache?: boolean;
maxAge?: number;
parsePDF?: boolean;
+ auditMetadata?: Record<string, string>;
}
export type Action = {
@@ -310,6 +311,7 @@ export interface MapParams {
timeout?: number;
useIndex?: boolean;
location?: LocationConfig;
+ auditMetadata?: Record<string, string>;
}
/**
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/agent.ts b/apps/js-sdk/firecrawl/src/v2/methods/agent.ts
index 08ae78563..16cdcb388 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/agent.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/agent.ts
@@ -15,6 +15,7 @@ function prepareAgentPayload(args: {
model?: "spark-1-pro" | "spark-1-mini";
webhook?: string | AgentWebhookConfig;
threatProtection?: ThreatProtectionOptions;
+ auditMetadata?: Record<string, string>;
}): Record<string, unknown> {
const body: Record<string, unknown> = {};
if (args.urls) body.urls = args.urls;
@@ -30,6 +31,8 @@ function prepareAgentPayload(args: {
if (args.webhook != null) body.webhook = args.webhook;
if (args.threatProtection != null)
body.threatProtection = args.threatProtection;
+ if (args.auditMetadata != null)
+ body.auditMetadata = args.auditMetadata;
return body;
}
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/map.ts b/apps/js-sdk/firecrawl/src/v2/methods/map.ts
index eab0b6d06..2bc493e2d 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/map.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/map.ts
@@ -26,6 +26,8 @@ function prepareMapPayload(
if (options.location != null) payload.location = options.location;
if (options.threatProtection != null)
payload.threatProtection = options.threatProtection;
+ if (options.auditMetadata != null)
+ payload.auditMetadata = options.auditMetadata;
}
return payload;
}
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index da2ea3208..24583af32 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -209,6 +209,7 @@ export interface ScrapeOptions {
lockdown?: boolean;
redactPII?: boolean | RedactPIIOptions;
threatProtection?: ThreatProtectionOptions;
+ auditMetadata?: Record<string, string>;
profile?: {
name: string;
saveChanges?: boolean;
@@ -796,6 +797,7 @@ export interface MapOptions {
origin?: string;
location?: LocationConfig;
threatProtection?: ThreatProtectionOptions;
+ auditMetadata?: Record<string, string>;
}
export type FeedbackRating = "good" | "partial" | "bad";
diff --git a/apps/php-sdk/src/Models/AgentOptions.php b/apps/php-sdk/src/Models/AgentOptions.php
index 49061561a..381268549 100644
--- a/apps/php-sdk/src/Models/AgentOptions.php
+++ b/apps/php-sdk/src/Models/AgentOptions.php
@@ -9,6 +9,7 @@ final class AgentOptions
/**
* @param list<string>|null $urls
* @param array<string, mixed>|null $schema
+ * @param array<string, string>|null $auditMetadata
*/
private function __construct(
private readonly ?array $urls = null,
@@ -19,11 +20,14 @@ final class AgentOptions
private readonly ?bool $strictConstrainToURLs = null,
private readonly ?string $model = null,
private readonly ?WebhookConfig $webhook = null,
+ /** @var array<string, string>|null */
+ private readonly ?array $auditMetadata = null,
) {}
/**
* @param list<string>|null $urls
* @param array<string, mixed>|null $schema
+ * @param array<string, string>|null $auditMetadata
*/
public static function with(
?array $urls = null,
@@ -34,10 +38,11 @@ final class AgentOptions
?bool $strictConstrainToURLs = null,
?string $model = null,
?WebhookConfig $webhook = null,
+ ?array $auditMetadata = null,
): self {
return new self(
$urls, $prompt, $schema, $integration,
- $maxCredits, $strictConstrainToURLs, $model, $webhook,
+ $maxCredits, $strictConstrainToURLs, $model, $webhook, $auditMetadata,
);
}
@@ -53,6 +58,7 @@ final class AgentOptions
'strictConstrainToURLs' => $this->strictConstrainToURLs,
'model' => $this->model,
'webhook' => $this->webhook?->toArray(),
+ 'auditMetadata' => $this->auditMetadata,
];
return array_filter($fields, fn (mixed $v): bool => $v !== null);
diff --git a/apps/php-sdk/src/Models/MapOptions.php b/apps/php-sdk/src/Models/MapOptions.php
index 6eebd94ef..397847898 100644
--- a/apps/php-sdk/src/Models/MapOptions.php
+++ b/apps/php-sdk/src/Models/MapOptions.php
@@ -6,6 +6,7 @@ namespace Firecrawl\Models;
final class MapOptions
{
+ /** @param array<string, string>|null $auditMetadata */
private function __construct(
private readonly ?string $search = null,
private readonly ?string $sitemap = null,
@@ -15,8 +16,11 @@ final class MapOptions
private readonly ?int $timeout = null,
private readonly ?string $integration = null,
private readonly ?LocationConfig $location = null,
+ /** @var array<string, string>|null */
+ private readonly ?array $auditMetadata = null,
) {}
+ /** @param array<string, string>|null $auditMetadata */
public static function with(
?string $search = null,
?string $sitemap = null,
@@ -26,10 +30,11 @@ final class MapOptions
?int $timeout = null,
?string $integration = null,
?LocationConfig $location = null,
+ ?array $auditMetadata = null,
): self {
return new self(
$search, $sitemap, $includeSubdomains, $ignoreQueryParameters,
- $limit, $timeout, $integration, $location,
+ $limit, $timeout, $integration, $location, $auditMetadata,
);
}
@@ -45,6 +50,7 @@ final class MapOptions
'timeout' => $this->timeout,
'integration' => $this->integration,
'location' => $this->location?->toArray(),
+ 'auditMetadata' => $this->auditMetadata,
];
return array_filter($fields, fn (mixed $v): bool => $v !== null);
diff --git a/apps/php-sdk/src/Models/ParseOptions.php b/apps/php-sdk/src/Models/ParseOptions.php
index 08db06878..c8a3f0fc8 100644
--- a/apps/php-sdk/src/Models/ParseOptions.php
+++ b/apps/php-sdk/src/Models/ParseOptions.php
@@ -32,6 +32,7 @@ final class ParseOptions
* @param list<string>|null $includeTags
* @param list<string>|null $excludeTags
* @param list<mixed>|null $parsers
+ * @param array<string, string>|null $auditMetadata
*/
private function __construct(
private readonly ?array $formats = null,
@@ -47,6 +48,8 @@ final class ParseOptions
private readonly ?string $proxy = null,
private readonly ?string $integration = null,
private readonly ?bool $redactPII = null,
+ /** @var array<string, string>|null */
+ private readonly ?array $auditMetadata = null,
) {}
/**
@@ -55,6 +58,7 @@ final class ParseOptions
* @param list<string>|null $includeTags
* @param list<string>|null $excludeTags
* @param list<mixed>|null $parsers
+ * @param array<string, string>|null $auditMetadata
*/
public static function with(
?array $formats = null,
@@ -70,6 +74,7 @@ final class ParseOptions
?string $proxy = null,
?string $integration = null,
?bool $redactPII = null,
+ ?array $auditMetadata = null,
): self {
if ($timeout !== null && $timeout <= 0) {
throw new FirecrawlException('timeout must be positive');
@@ -102,6 +107,7 @@ final class ParseOptions
$proxy,
$integration,
$redactPII,
+ $auditMetadata,
);
}
@@ -136,6 +142,7 @@ final class ParseOptions
'proxy' => $this->proxy,
'integration' => $this->integration,
'redactPII' => $this->redactPII,
+ 'auditMetadata' => $this->auditMetadata,
];
foreach ($fields as $key => $value) {
@@ -234,4 +241,10 @@ final class ParseOptions
{
return $this->integration;
}
+
+ /** @return array<string, string>|null */
+ public function getAuditMetadata(): ?array
+ {
+ return $this->auditMetadata;
+ }
}
diff --git a/apps/php-sdk/src/Models/ScrapeOptions.php b/apps/php-sdk/src/Models/ScrapeOptions.php
index 94c7f0d09..d264cd162 100644
--- a/apps/php-sdk/src/Models/ScrapeOptions.php
+++ b/apps/php-sdk/src/Models/ScrapeOptions.php
@@ -13,6 +13,7 @@ final class ScrapeOptions
* @param list<string>|null $excludeTags
* @param list<mixed>|null $parsers
* @param list<array<string, mixed>>|null $actions
+ * @param array<string, string>|null $auditMetadata
*/
private function __construct(
private readonly ?array $formats = null,
@@ -39,6 +40,8 @@ final class ScrapeOptions
private readonly ?array $profile = null,
private readonly ?bool $changeTracking = null,
private readonly ?bool $redactPII = null,
+ /** @var array<string, string>|null */
+ private readonly ?array $auditMetadata = null,
) {}
/**
@@ -49,6 +52,7 @@ final class ScrapeOptions
* @param list<mixed>|null $parsers
* @param list<array<string, mixed>>|null $actions
* @param array<string, string>|null $profile
+ * @param array<string, string>|null $auditMetadata
*/
public static function with(
?array $formats = null,
@@ -74,13 +78,14 @@ final class ScrapeOptions
?array $profile = null,
?bool $changeTracking = null,
?bool $redactPII = null,
+ ?array $auditMetadata = null,
): self {
return new self(
$formats, $headers, $includeTags, $excludeTags, $onlyMainContent,
$timeout, $waitFor, $mobile, $parsers, $actions, $location,
$skipTlsVerification, $removeBase64Images, $blockAds, $proxy,
$maxAge, $minAge, $storeInCache, $lockdown, $integration, $profile,
- $changeTracking, $redactPII,
+ $changeTracking, $redactPII, $auditMetadata,
);
}
@@ -126,6 +131,7 @@ final class ScrapeOptions
'profile' => $this->profile,
'changeTracking' => $this->changeTracking,
'redactPII' => $this->redactPII,
+ 'auditMetadata' => $this->auditMetadata,
];
foreach ($fields as $key => $value) {
@@ -258,4 +264,10 @@ final class ScrapeOptions
{
return $this->changeTracking;
}
+
+ /** @return array<string, string>|null */
+ public function getAuditMetadata(): ?array
+ {
+ return $this->auditMetadata;
+ }
}
diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.php
index 4e52a3fcf..d8f95f0bc 100644
--- a/apps/php-sdk/src/Version.php
+++ b/apps/php-sdk/src/Version.php
@@ -6,5 +6,5 @@ namespace Firecrawl;
final class Version
{
- public const SDK_VERSION = '1.9.1';
+ public const SDK_VERSION = '1.10.0';
}
diff --git a/apps/php-sdk/tests/Unit/ModelsTest.php b/apps/php-sdk/tests/Unit/ModelsTest.php
index e55ac7a9b..c78dbc863 100644
--- a/apps/php-sdk/tests/Unit/ModelsTest.php
+++ b/apps/php-sdk/tests/Unit/ModelsTest.php
@@ -10,6 +10,9 @@ use Firecrawl\Models\MapData;
use Firecrawl\Models\BatchScrapeJob;
use Firecrawl\Models\CrawlJob;
use Firecrawl\Models\HighlightsFormat;
+use Firecrawl\Models\AgentOptions;
+use Firecrawl\Models\MapOptions;
+use Firecrawl\Models\ParseOptions;
use Firecrawl\Models\QueryFormat;
use Firecrawl\Models\QuestionFormat;
use Firecrawl\Models\ScrapeOptions;
@@ -340,6 +343,21 @@ it('serializes redactPII in ScrapeOptions', function (): void {
expect(array_key_exists('formats', $options->toArray()))->toBeFalse();
});
+it('serializes audit metadata across request options', function (): void {
+ $metadata = ['requestId' => 'req-123'];
+
+ $scrape = ScrapeOptions::with(auditMetadata: $metadata);
+ expect($scrape->toArray())->toMatchArray(['auditMetadata' => $metadata]);
+ expect($scrape->getAuditMetadata())->toBe($metadata);
+ expect(MapOptions::with(auditMetadata: $metadata)->toArray())
+ ->toMatchArray(['auditMetadata' => $metadata]);
+ expect(AgentOptions::with(prompt: 'find pricing', auditMetadata: $metadata)->toArray())
+ ->toMatchArray(['auditMetadata' => $metadata]);
+ $parse = ParseOptions::with(auditMetadata: $metadata);
+ expect($parse->toArray())->toMatchArray(['auditMetadata' => $metadata]);
+ expect($parse->getAuditMetadata())->toBe($metadata);
+});
+
it('serializes query format mode in ScrapeOptions', function (): void {
$options = ScrapeOptions::with(
formats: [QueryFormat::with('What is Firecrawl?', QueryFormat::MODE_DIRECT_QUOTE)],
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index fc8b578bb..5ac05814e 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.32.1"
+__version__ = "4.33.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_audit_metadata_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_audit_metadata_request_preparation.py
new file mode 100644
index 000000000..685591abe
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_audit_metadata_request_preparation.py
@@ -0,0 +1,61 @@
+import json
+
+from firecrawl.v2.methods.agent import _prepare_agent_request
+from firecrawl.v2.methods.aio.batch import _prepare as prepare_async_batch_request
+from firecrawl.v2.methods.batch import prepare_batch_scrape_request
+from firecrawl.v2.methods.map import _prepare_map_request
+from firecrawl.v2.methods.parse import _prepare_parse_request
+from firecrawl.v2.types import MapOptions, ParseOptions, ScrapeOptions
+from firecrawl.v2.utils.validation import prepare_scrape_options
+
+
+AUDIT_METADATA = {"requestId": "req-123", "actor": "sdk-test"}
+
+
+def test_scrape_and_batch_audit_metadata_request_preparation():
+ options = ScrapeOptions(audit_metadata=AUDIT_METADATA)
+
+ assert prepare_scrape_options(options)["auditMetadata"] == AUDIT_METADATA
+ assert (
+ prepare_batch_scrape_request(
+ ["https://example.com"],
+ audit_metadata=AUDIT_METADATA,
+ )["auditMetadata"]
+ == AUDIT_METADATA
+ )
+ assert (
+ prepare_async_batch_request(
+ ["https://example.com"],
+ audit_metadata=AUDIT_METADATA,
+ )["auditMetadata"]
+ == AUDIT_METADATA
+ )
+
+
+def test_map_and_agent_audit_metadata_request_preparation():
+ assert (
+ _prepare_map_request(
+ "https://example.com",
+ MapOptions(audit_metadata=AUDIT_METADATA),
+ )["auditMetadata"]
+ == AUDIT_METADATA
+ )
+ assert (
+ _prepare_agent_request(
+ None,
+ prompt="Extract the page",
+ audit_metadata=AUDIT_METADATA,
+ )["auditMetadata"]
+ == AUDIT_METADATA
+ )
+
+
+def test_parse_audit_metadata_request_preparation():
+ fields, _ = _prepare_parse_request(
+ b"<html></html>",
+ ParseOptions(audit_metadata=AUDIT_METADATA),
+ filename="upload.html",
+ content_type="text/html",
+ )
+
+ assert json.loads(fields["options"])["auditMetadata"] == AUDIT_METADATA
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index a8236e573..b0a21caba 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -78,7 +78,7 @@ _SCRAPE_OPTION_KEYS = frozenset({
"parsers", "actions", "location", "skip_tls_verification",
"remove_base64_images", "fast_mode", "use_mock", "block_ads",
"proxy", "max_age", "store_in_cache", "lockdown", "threat_protection",
- "profile",
+ "profile", "audit_metadata",
})
@@ -161,6 +161,7 @@ class FirecrawlClient:
lockdown: Optional[bool] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
profile: Optional[Dict[str, Any]] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
integration: Optional[str] = None,
) -> Document:
"""
@@ -189,6 +190,7 @@ class FirecrawlClient:
lockdown: Serve only previously cached results; never make outbound requests. Returns 404 SCRAPE_LOCKDOWN_CACHE_MISS on cache miss.
threat_protection: Enterprise per-request override of the team's threat protection policy
profile: Browser profile for persistent state (e.g. {"name": "my-profile", "saveChanges": True})
+ audit_metadata: Metadata to include in SIEM logging events
Returns:
Document
"""
@@ -216,9 +218,10 @@ class FirecrawlClient:
lockdown=lockdown,
threat_protection=threat_protection,
profile=profile,
+ audit_metadata=audit_metadata,
integration=integration,
).items() if v is not None}
- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection, profile, integration]) else None
+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection, profile, audit_metadata, integration]) else None
return scrape_module.scrape(self.http_client, url, options)
def search_papers(self, query: str, **kwargs):
@@ -483,6 +486,7 @@ class FirecrawlClient:
lockdown: Optional[bool] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
profile: Optional[Dict[str, Any]] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
regex_on_full_url: bool = False,
deduplicate_similar_urls: bool = True,
zero_data_retention: bool = False,
@@ -533,6 +537,7 @@ class FirecrawlClient:
lockdown: Serve only cached results (convenience kwarg)
threat_protection: Enterprise threat protection override (convenience kwarg)
profile: Browser profile (convenience kwarg)
+ audit_metadata: Metadata to include in SIEM logging events
regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathname
deduplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)
zero_data_retention: Whether to delete data after 24 hours
@@ -558,6 +563,7 @@ class FirecrawlClient:
use_mock=use_mock, block_ads=block_ads, proxy=proxy,
max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,
threat_protection=threat_protection, profile=profile,
+ audit_metadata=audit_metadata,
).items() if v is not None}
if scrape_kwargs:
scrape_options = ScrapeOptions(**scrape_kwargs)
@@ -642,6 +648,7 @@ class FirecrawlClient:
lockdown: Optional[bool] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
profile: Optional[Dict[str, Any]] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
regex_on_full_url: bool = False,
deduplicate_similar_urls: bool = True,
zero_data_retention: bool = False,
@@ -690,6 +697,7 @@ class FirecrawlClient:
lockdown: Serve only cached results (convenience kwarg)
threat_protection: Enterprise threat protection override (convenience kwarg)
profile: Browser profile (convenience kwarg)
+ audit_metadata: Metadata to include in SIEM logging events
regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathname
deduplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)
zero_data_retention: Whether to delete data after 24 hours
@@ -712,6 +720,7 @@ class FirecrawlClient:
use_mock=use_mock, block_ads=block_ads, proxy=proxy,
max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,
threat_protection=threat_protection, profile=profile,
+ audit_metadata=audit_metadata,
).items() if v is not None}
if scrape_kwargs:
scrape_options = ScrapeOptions(**scrape_kwargs)
@@ -843,6 +852,7 @@ class FirecrawlClient:
integration: Optional[str] = None,
location: Optional[Location] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
) -> MapData:
"""Map a URL and return discovered links.
@@ -856,6 +866,7 @@ class FirecrawlClient:
timeout: Request timeout in milliseconds
threat_protection: Enterprise per-request override of the team's
threat protection policy
+ audit_metadata: Metadata to include in SIEM logging events
Returns:
MapData containing the discovered links
@@ -869,8 +880,9 @@ class FirecrawlClient:
timeout=timeout,
integration=integration,
location=location,
- threat_protection=threat_protection
- ) if any(v is not None for v in [search, include_subdomains, ignore_query_parameters, limit, sitemap, timeout, integration, location, threat_protection]) else None
+ threat_protection=threat_protection,
+ audit_metadata=audit_metadata,
+ ) if any(v is not None for v in [search, include_subdomains, ignore_query_parameters, limit, sitemap, timeout, integration, location, threat_protection, audit_metadata]) else None
return map_module.map(self.http_client, url, options)
@@ -1156,6 +1168,7 @@ class FirecrawlClient:
store_in_cache: Optional[bool] = None,
lockdown: Optional[bool] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
webhook: Optional[Union[str, WebhookConfig]] = None,
append_to_id: Optional[str] = None,
ignore_invalid_urls: Optional[bool] = None,
@@ -1189,6 +1202,7 @@ class FirecrawlClient:
store_in_cache: Whether to store results in cache
lockdown: Serve only previously cached results; never make outbound requests.
threat_protection: Enterprise per-request override of the team's threat protection policy
+ audit_metadata: Metadata to include in SIEM logging events
webhook: Webhook configuration
append_to_id: Append to an existing batch job
ignore_invalid_urls: Skip invalid URLs without failing
@@ -1236,6 +1250,7 @@ class FirecrawlClient:
max_concurrency=max_concurrency,
zero_data_retention=zero_data_retention,
integration=integration,
+ audit_metadata=audit_metadata,
idempotency_key=idempotency_key,
)
@@ -1330,6 +1345,7 @@ class FirecrawlClient:
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
):
"""Start an agent job (non-blocking).
@@ -1343,6 +1359,7 @@ class FirecrawlClient:
webhook: Webhook URL or configuration for notifications
threat_protection: Enterprise per-request override of the team's
threat protection policy
+ audit_metadata: Metadata to include in SIEM logging events
Returns:
Response payload with job id/status (poll with get_agent_status)
"""
@@ -1357,6 +1374,7 @@ class FirecrawlClient:
model=model,
webhook=webhook,
threat_protection=threat_protection,
+ audit_metadata=audit_metadata,
)
def agent(
@@ -1373,6 +1391,7 @@ class FirecrawlClient:
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
):
"""Run an agent and wait until completion.
@@ -1388,6 +1407,7 @@ class FirecrawlClient:
webhook: Webhook URL or configuration for notifications
threat_protection: Enterprise per-request override of the team's
threat protection policy
+ audit_metadata: Metadata to include in SIEM logging events
Returns:
Final agent response when completed
"""
@@ -1404,6 +1424,7 @@ class FirecrawlClient:
model=model,
webhook=webhook,
threat_protection=threat_protection,
+ audit_metadata=audit_metadata,
)
def get_agent_status(self, job_id: str):
@@ -1583,6 +1604,7 @@ class FirecrawlClient:
store_in_cache: Optional[bool] = None,
lockdown: Optional[bool] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
webhook: Optional[Union[str, WebhookConfig]] = None,
append_to_id: Optional[str] = None,
ignore_invalid_urls: Optional[bool] = None,
@@ -1632,6 +1654,7 @@ class FirecrawlClient:
max_concurrency=max_concurrency,
zero_data_retention=zero_data_retention,
integration=integration,
+ audit_metadata=audit_metadata,
idempotency_key=idempotency_key,
poll_interval=poll_interval,
timeout=wait_timeout,
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index 00fa01757..9ea7eaa4a 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -399,6 +399,7 @@ class AsyncFirecrawlClient:
timeout: Optional[int] = None,
integration: Optional[str] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
) -> MapData:
options = MapOptions(
search=search,
@@ -408,7 +409,8 @@ class AsyncFirecrawlClient:
timeout=timeout,
integration=integration,
threat_protection=threat_protection,
- ) if any(v is not None for v in [search, include_subdomains, limit, sitemap, integration, timeout, threat_protection]) else None
+ audit_metadata=audit_metadata,
+ ) if any(v is not None for v in [search, include_subdomains, limit, sitemap, integration, timeout, threat_protection, audit_metadata]) else None
return await async_map.map(self.async_http_client, url, options)
async def create_monitor(
@@ -680,6 +682,7 @@ class AsyncFirecrawlClient:
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
):
return await async_agent.agent(
self.async_http_client,
@@ -694,6 +697,7 @@ class AsyncFirecrawlClient:
model=model,
webhook=webhook,
threat_protection=threat_protection,
+ audit_metadata=audit_metadata,
)
async def get_agent_status(self, job_id: str):
@@ -711,6 +715,7 @@ class AsyncFirecrawlClient:
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
):
return await async_agent.start_agent(
self.async_http_client,
@@ -723,6 +728,7 @@ class AsyncFirecrawlClient:
model=model,
webhook=webhook,
threat_protection=threat_protection,
+ audit_metadata=audit_metadata,
)
async def cancel_agent(self, job_id: str) -> bool:
diff --git a/apps/python-sdk/firecrawl/v2/methods/agent.py b/apps/python-sdk/firecrawl/v2/methods/agent.py
index e6bc6c412..6823ce3f6 100644
--- a/apps/python-sdk/firecrawl/v2/methods/agent.py
+++ b/apps/python-sdk/firecrawl/v2/methods/agent.py
@@ -18,6 +18,7 @@ def _prepare_agent_request(
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
) -> Dict[str, Any]:
body: Dict[str, Any] = {}
if urls is not None:
@@ -49,6 +50,8 @@ def _prepare_agent_request(
body["threatProtection"] = threat_protection.model_dump(
by_alias=True, exclude_none=True
)
+ if audit_metadata is not None:
+ body["auditMetadata"] = audit_metadata
return body
@@ -73,6 +76,7 @@ def start_agent(
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
) -> AgentResponse:
body = _prepare_agent_request(
urls,
@@ -84,6 +88,7 @@ def start_agent(
model=model,
webhook=webhook,
threat_protection=threat_protection,
+ audit_metadata=audit_metadata,
)
resp = client.post("/v2/agent", body)
if not resp.ok:
@@ -131,6 +136,7 @@ def agent(
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
) -> AgentResponse:
started = start_agent(
client,
@@ -143,6 +149,7 @@ def agent(
model=model,
webhook=webhook,
threat_protection=threat_protection,
+ audit_metadata=audit_metadata,
)
job_id = getattr(started, "id", None)
if not job_id:
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py b/apps/python-sdk/firecrawl/v2/methods/aio/agent.py
index 601399871..86eecc076 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/agent.py
@@ -18,6 +18,7 @@ def _prepare_agent_request(
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
) -> Dict[str, Any]:
body: Dict[str, Any] = {}
if urls is not None:
@@ -49,6 +50,8 @@ def _prepare_agent_request(
body["threatProtection"] = threat_protection.model_dump(
by_alias=True, exclude_none=True
)
+ if audit_metadata is not None:
+ body["auditMetadata"] = audit_metadata
return body
@@ -73,6 +76,7 @@ async def start_agent(
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
) -> AgentResponse:
body = _prepare_agent_request(
urls,
@@ -84,6 +88,7 @@ async def start_agent(
model=model,
webhook=webhook,
threat_protection=threat_protection,
+ audit_metadata=audit_metadata,
)
resp = await client.post("/v2/agent", body)
if not resp.ok:
@@ -131,6 +136,7 @@ async def agent(
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
threat_protection: Optional[ThreatProtectionOptions] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
) -> AgentResponse:
started = await start_agent(
client,
@@ -143,6 +149,7 @@ async def agent(
model=model,
webhook=webhook,
threat_protection=threat_protection,
+ audit_metadata=audit_metadata,
)
job_id = getattr(started, "id", None)
if not job_id:
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py
index 4585f4443..1cdd1a13a 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py
@@ -40,6 +40,8 @@ def _prepare(urls: List[str], *, options: Optional[ScrapeOptions] = None, **kwar
opts = prepare_scrape_options(options)
if opts:
payload.update(opts)
+ if (v := kwargs.get("audit_metadata")) is not None:
+ payload["auditMetadata"] = v
if (w := kwargs.get("webhook")) is not None:
payload["webhook"] = w if isinstance(w, str) else w.model_dump(exclude_none=True)
if (v := kwargs.get("append_to_id")) is not None:
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/map.py b/apps/python-sdk/firecrawl/v2/methods/aio/map.py
index 883b01cb5..4ef0e35ae 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/map.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/map.py
@@ -32,6 +32,8 @@ def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict
data["threatProtection"] = options.threat_protection.model_dump(
by_alias=True, exclude_none=True
)
+ if options.audit_metadata is not None:
+ data["auditMetadata"] = options.audit_metadata
payload.update(data)
return payload
@@ -68,4 +70,3 @@ async def map(client: AsyncHttpClient, url: str, options: Optional[MapOptions] =
result_links.append(LinkResult(url=item))
return MapData(links=result_links)
-
diff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.py
index 2b3477d66..f81768018 100644
--- a/apps/python-sdk/firecrawl/v2/methods/batch.py
+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py
@@ -53,6 +53,7 @@ def start_batch_scrape(
max_concurrency: Optional[int] = None,
zero_data_retention: Optional[bool] = None,
integration: Optional[str] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
idempotency_key: Optional[str] = None,
) -> BatchScrapeResponse:
"""
@@ -79,6 +80,7 @@ def start_batch_scrape(
max_concurrency=max_concurrency,
zero_data_retention=zero_data_retention,
integration=integration,
+ audit_metadata=audit_metadata,
)
# Make the API request
@@ -341,6 +343,7 @@ def batch_scrape(
max_concurrency: Optional[int] = None,
zero_data_retention: Optional[bool] = None,
integration: Optional[str] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
idempotency_key: Optional[str] = None,
poll_interval: int = 2,
timeout: Optional[int] = None
@@ -373,6 +376,7 @@ def batch_scrape(
max_concurrency=max_concurrency,
zero_data_retention=zero_data_retention,
integration=integration,
+ audit_metadata=audit_metadata,
idempotency_key=idempotency_key,
)
@@ -424,6 +428,7 @@ def prepare_batch_scrape_request(
max_concurrency: Optional[int] = None,
zero_data_retention: Optional[bool] = None,
integration: Optional[str] = None,
+ audit_metadata: Optional[Dict[str, str]] = None,
) -> dict:
"""
Prepare a batch scrape request payload.
@@ -443,6 +448,8 @@ def prepare_batch_scrape_request(
scrape_data = prepare_scrape_options(options)
if scrape_data:
request_data.update(scrape_data)
+ if audit_metadata is not None:
+ request_data["auditMetadata"] = audit_metadata
# Batch-specific fields
if webhook is not None:
diff --git a/apps/python-sdk/firecrawl/v2/methods/map.py b/apps/python-sdk/firecrawl/v2/methods/map.py
index 5181931cf..7da8073ba 100644
--- a/apps/python-sdk/firecrawl/v2/methods/map.py
+++ b/apps/python-sdk/firecrawl/v2/methods/map.py
@@ -39,6 +39,8 @@ def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict
data["threatProtection"] = options.threat_protection.model_dump(
by_alias=True, exclude_none=True
)
+ if options.audit_metadata is not None:
+ data["auditMetadata"] = options.audit_metadata
payload.update(data)
return payload
@@ -86,4 +88,3 @@ def map(client: HttpClient, url: str, options: Optional[MapOptions] = None) -> M
result_links.append(LinkResult(url=item))
return MapData(links=result_links)
-
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index a769e843a..f0012742f 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -822,6 +822,9 @@ class ScrapeOptions(BaseModel):
threat_protection: Optional[ThreatProtectionOptions] = Field(
default=None, alias="threatProtection"
)
+ audit_metadata: Optional[Dict[str, str]] = Field(
+ default=None, alias="auditMetadata"
+ )
profile: Optional[Dict[str, Any]] = None
integration: Optional[str] = None
@@ -1054,6 +1057,7 @@ class MapOptions(BaseModel):
integration: Optional[str] = None
location: Optional["Location"] = None
threat_protection: Optional[ThreatProtectionOptions] = None
+ audit_metadata: Optional[Dict[str, str]] = None
class MapRequest(BaseModel):
diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.py
index 1f16e06a7..ac87d8fb5 100644
--- a/apps/python-sdk/firecrawl/v2/utils/validation.py
+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py
@@ -569,6 +569,7 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
"min_age": "minAge",
"redact_pii": "redactPII",
"threat_protection": "threatProtection",
+ "audit_metadata": "auditMetadata",
}
# Apply field mappings
@@ -818,4 +819,4 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
# For fields that don't need conversion, use as-is
scrape_data[key] = value
- return scrape_data
\ No newline at end of file
+ return scrape_data
diff --git a/apps/ruby-sdk/lib/firecrawl/models/agent_options.rb b/apps/ruby-sdk/lib/firecrawl/models/agent_options.rb
index 17daba219..c6f3313be 100644
--- a/apps/ruby-sdk/lib/firecrawl/models/agent_options.rb
+++ b/apps/ruby-sdk/lib/firecrawl/models/agent_options.rb
@@ -6,7 +6,7 @@ module Firecrawl
class AgentOptions
FIELDS = %i[
urls prompt schema integration max_credits
- strict_constrain_to_urls model webhook
+ strict_constrain_to_urls model webhook audit_metadata
].freeze
attr_reader(*FIELDS)
@@ -26,6 +26,7 @@ module Firecrawl
"strictConstrainToURLs" => strict_constrain_to_urls,
"model" => model,
"webhook" => webhook.is_a?(Hash) ? webhook : webhook&.to_h,
+ "auditMetadata" => audit_metadata,
}.compact
end
end
diff --git a/apps/ruby-sdk/lib/firecrawl/models/map_options.rb b/apps/ruby-sdk/lib/firecrawl/models/map_options.rb
index 47c176ad7..1f3824f68 100644
--- a/apps/ruby-sdk/lib/firecrawl/models/map_options.rb
+++ b/apps/ruby-sdk/lib/firecrawl/models/map_options.rb
@@ -6,7 +6,7 @@ module Firecrawl
class MapOptions
FIELDS = %i[
search sitemap include_subdomains ignore_query_parameters
- limit timeout integration location
+ limit timeout integration location audit_metadata
].freeze
attr_reader(*FIELDS)
@@ -25,6 +25,7 @@ module Firecrawl
"timeout" => timeout,
"integration" => integration,
"location" => location.is_a?(Hash) ? location : location&.to_h,
+ "auditMetadata" => audit_metadata,
}.compact
end
end
diff --git a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb
index b29d4fa04..7ba9235af 100644
--- a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb
+++ b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb
@@ -13,7 +13,7 @@ module Firecrawl
FIELDS = %i[
formats headers include_tags exclude_tags only_main_content
timeout parsers skip_tls_verification remove_base64_images
- block_ads proxy integration redact_pii json_options
+ block_ads proxy integration redact_pii json_options audit_metadata
].freeze
attr_reader(*FIELDS)
@@ -40,6 +40,7 @@ module Firecrawl
"integration" => integration,
"redactPII" => redact_pii,
"jsonOptions" => json_options.is_a?(Hash) ? json_options : json_options&.to_h,
+ "auditMetadata" => audit_metadata,
}.compact
end
diff --git a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb
index 50d4d43d5..d860268b9 100644
--- a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb
+++ b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb
@@ -8,7 +8,7 @@ module Firecrawl
formats headers include_tags exclude_tags only_main_content
timeout wait_for mobile parsers actions location
skip_tls_verification remove_base64_images block_ads proxy
- max_age store_in_cache lockdown redact_pii integration
+ max_age store_in_cache lockdown redact_pii integration audit_metadata
].freeze
attr_reader(*FIELDS)
@@ -40,6 +40,7 @@ module Firecrawl
"lockdown" => lockdown,
"redactPII" => redact_pii,
"integration" => integration,
+ "auditMetadata" => audit_metadata,
}.compact
end
diff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rb
index fd14285d4..7706c5496 100644
--- a/apps/ruby-sdk/lib/firecrawl/version.rb
+++ b/apps/ruby-sdk/lib/firecrawl/version.rb
@@ -1,5 +1,5 @@
# frozen_string_literal: true
module Firecrawl
- VERSION = "1.10.1"
+ VERSION = "1.11.0"
end
diff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rb
index 8019993f9..0165206cc 100644
--- a/apps/ruby-sdk/test/firecrawl/client_test.rb
+++ b/apps/ruby-sdk/test/firecrawl/client_test.rb
@@ -769,6 +769,23 @@ class ClientTest < Minitest::Test
assert_equal "spark-1-pro", h["model"]
end
+ def test_audit_metadata_to_h
+ metadata = { "requestId" => "req-123" }
+
+ assert_equal metadata, Firecrawl::Models::ScrapeOptions.new(audit_metadata: metadata).to_h["auditMetadata"]
+ assert_equal metadata, Firecrawl::Models::MapOptions.new(audit_metadata: metadata).to_h["auditMetadata"]
+ assert_equal metadata,
+ Firecrawl::Models::AgentOptions.new(
+ prompt: "find pricing",
+ audit_metadata: metadata
+ ).to_h["auditMetadata"]
+ assert_equal metadata, Firecrawl::Models::ParseOptions.new(audit_metadata: metadata).to_h["auditMetadata"]
+ crawl = Firecrawl::Models::CrawlOptions.new(
+ scrape_options: Firecrawl::Models::ScrapeOptions.new(audit_metadata: metadata)
+ )
+ assert_equal metadata, crawl.to_h.dig("scrapeOptions", "auditMetadata")
+ end
+
def test_batch_scrape_options_to_h
scrape_opts = Firecrawl::Models::ScrapeOptions.new(formats: ["markdown"])
opts = Firecrawl::Models::BatchScrapeOptions.new(
diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock
index c5fda5043..1918d7a5e 100644
--- a/apps/rust-sdk/Cargo.lock
+++ b/apps/rust-sdk/Cargo.lock
@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "firecrawl"
-version = "2.11.1"
+version = "2.12.0"
dependencies = [
"mockito",
"reqwest",
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index 40611dcad..3bdfa51a4 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "firecrawl"
-version = "2.11.1"
+version = "2.12.0"
edition = "2021"
license = "MIT"
homepage = "https://www.firecrawl.dev/"
diff --git a/apps/rust-sdk/src/agent.rs b/apps/rust-sdk/src/agent.rs
index bd4e77796..a4c18b5a2 100644
--- a/apps/rust-sdk/src/agent.rs
+++ b/apps/rust-sdk/src/agent.rs
@@ -5,6 +5,7 @@
use serde::{Deserialize, Serialize};
use serde_json::Value;
+use std::collections::HashMap;
use crate::client::Client;
use crate::types::{AgentModel, AgentWebhookConfig};
@@ -39,6 +40,9 @@ pub struct AgentOptions {
/// Webhook configuration for agent notifications.
pub webhook: Option<AgentWebhookConfig>,
+ /// Metadata to include with SIEM logging events.
+ pub audit_metadata: Option<HashMap<String, String>>,
+
/// Poll interval for synchronous agent execution (milliseconds).
#[serde(skip)]
pub poll_interval: Option<u64>,
diff --git a/apps/rust-sdk/src/map.rs b/apps/rust-sdk/src/map.rs
index 724e8bad2..1d686ae78 100644
--- a/apps/rust-sdk/src/map.rs
+++ b/apps/rust-sdk/src/map.rs
@@ -1,6 +1,7 @@
//! Map endpoint for Firecrawl API v2.
use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
use crate::client::Client;
use crate::types::{LocationConfig, SearchResultWeb, SitemapMode};
@@ -34,6 +35,9 @@ pub struct MapOptions {
/// Location configuration for proxy routing.
pub location: Option<LocationConfig>,
+
+ /// Metadata to include with SIEM logging events.
+ pub audit_metadata: Option<HashMap<String, String>>,
}
/// Request body for map endpoint.
diff --git a/apps/rust-sdk/src/parse.rs b/apps/rust-sdk/src/parse.rs
index 843ef1dc7..a6a4643c2 100644
--- a/apps/rust-sdk/src/parse.rs
+++ b/apps/rust-sdk/src/parse.rs
@@ -132,6 +132,8 @@ pub struct ParseOptions {
/// Redact personally identifiable information from returned content.
#[serde(rename = "redactPII")]
pub redact_pii: Option<bool>,
+ /// Metadata to include with SIEM logging events.
+ pub audit_metadata: Option<HashMap<String, String>>,
/// Request origin identifier.
pub origin: Option<String>,
/// Zero data retention mode.
diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rs
index 94820b20b..f697a0a8b 100644
--- a/apps/rust-sdk/src/scrape.rs
+++ b/apps/rust-sdk/src/scrape.rs
@@ -83,6 +83,9 @@ pub struct ScrapeOptions {
#[serde(rename = "redactPII")]
pub redact_pii: Option<bool>,
+ /// Metadata to include with SIEM logging events.
+ pub audit_metadata: Option<HashMap<String, String>>,
+
/// Persistent browser profile for maintaining state across scrapes.
pub profile: Option<ProfileConfig>,
diff --git a/apps/rust-sdk/tests/audit_metadata.rs b/apps/rust-sdk/tests/audit_metadata.rs
new file mode 100644
index 000000000..55c46c242
--- /dev/null
+++ b/apps/rust-sdk/tests/audit_metadata.rs
@@ -0,0 +1,32 @@
+use std::collections::HashMap;
+
+use firecrawl::{AgentOptions, MapOptions, ParseOptions, ScrapeOptions};
+use serde::Serialize;
+
+fn assert_audit_metadata<T: Serialize>(options: T) {
+ let value = serde_json::to_value(options).expect("options should serialize");
+ assert_eq!(value["auditMetadata"]["requestId"], "req-123");
+}
+
+#[test]
+fn serializes_audit_metadata_across_request_options() {
+ let metadata = HashMap::from([("requestId".to_string(), "req-123".to_string())]);
+
+ assert_audit_metadata(ScrapeOptions {
+ audit_metadata: Some(metadata.clone()),
+ ..Default::default()
+ });
+ assert_audit_metadata(MapOptions {
+ audit_metadata: Some(metadata.clone()),
+ ..Default::default()
+ });
+ assert_audit_metadata(AgentOptions {
+ prompt: "find pricing".to_string(),
+ audit_metadata: Some(metadata.clone()),
+ ..Default::default()
+ });
+ assert_audit_metadata(ParseOptions {
+ audit_metadata: Some(metadata),
+ ..Default::default()
+ });
+}
diff --git a/apps/siem/azure-sentinel/DCR.json b/apps/siem/azure-sentinel/DCR.json
new file mode 100644
index 000000000..d8e2fe29e
--- /dev/null
+++ b/apps/siem/azure-sentinel/DCR.json
@@ -0,0 +1,115 @@
+{
+ "name": "FirecrawlScrapeActivityPushDCR",
+ "apiVersion": "2021-09-01-preview",
+ "type": "Microsoft.Insights/dataCollectionRules",
+ "location": "[parameters('workspace-location')]",
+ "properties": {
+ "streamDeclarations": {
+ "Custom-FirecrawlScrapeActivity": {
+ "columns": [
+ {
+ "name": "schema_version",
+ "type": "int"
+ },
+ {
+ "name": "event_type",
+ "type": "string"
+ },
+ {
+ "name": "scrape_id",
+ "type": "string"
+ },
+ {
+ "name": "request_id",
+ "type": "string"
+ },
+ {
+ "name": "endpoint",
+ "type": "string"
+ },
+ {
+ "name": "team_id",
+ "type": "string"
+ },
+ {
+ "name": "org_id",
+ "type": "string"
+ },
+ {
+ "name": "api_key",
+ "type": "dynamic"
+ },
+ {
+ "name": "audit_metadata",
+ "type": "dynamic"
+ },
+ {
+ "name": "started_at",
+ "type": "datetime"
+ },
+ {
+ "name": "completed_at",
+ "type": "datetime"
+ },
+ {
+ "name": "url",
+ "type": "string"
+ },
+ {
+ "name": "domain",
+ "type": "string"
+ },
+ {
+ "name": "http_method",
+ "type": "string"
+ },
+ {
+ "name": "http_status",
+ "type": "int"
+ },
+ {
+ "name": "result",
+ "type": "string"
+ },
+ {
+ "name": "error",
+ "type": "dynamic"
+ },
+ {
+ "name": "origin",
+ "type": "string"
+ },
+ {
+ "name": "integration",
+ "type": "string"
+ },
+ {
+ "name": "zero_data_retention",
+ "type": "bool"
+ },
+ {
+ "name": "threat",
+ "type": "dynamic"
+ }
+ ]
+ }
+ },
+ "destinations": {
+ "logAnalytics": [
+ {
+ "workspaceResourceId": "[variables('workspaceResourceId')]",
+ "name": "firecrawlWorkspace"
+ }
+ ]
+ },
+ "dataFlows": [
+ {
+ "streams": ["Custom-FirecrawlScrapeActivity"],
+ "destinations": ["firecrawlWorkspace"],
+ "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tobool(threat.security_alert.detected), \"High\", tostring(threat.decision) == \"deny\", \"Low\", result == \"success\", \"Informational\", \"Low\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"audit_metadata\", audit_metadata, \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)",
+ "outputStream": "Microsoft-ASimWebSessionLogs"
+ }
+ ],
+ "dataCollectionEndpointId": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/', parameters('resourceGroupName'), '/providers/Microsoft.Insights/dataCollectionEndpoints/', parameters('workspace'))]"
+ }
+}
diff --git a/apps/siem/azure-sentinel/README.md b/apps/siem/azure-sentinel/README.md
new file mode 100644
index 000000000..7bb0bf43d
--- /dev/null
+++ b/apps/siem/azure-sentinel/README.md
@@ -0,0 +1,28 @@
+# SIEM Logging for Microsoft Sentinel
+
+Deployment artifacts for sending Firecrawl scrape activity to Microsoft
+Sentinel through the Azure Monitor Logs Ingestion API.
+
+## Files
+
+- `azuredeploy.json` creates the data collection endpoint and rule.
+- `DCR.json` contains the CCF data collection rule and ASIM transform.
+- `connectorDefinition.json` and `dataConnector.json` define the CCF Push
+ connector.
+- `sample-event.json` is a valid input-stream event.
+
+The database schema is maintained as a migration in the Firecrawl database
+repository, not in this package.
+
+## Runtime
+
+`PUT /v2/team/siem` stores an organization destination and its encrypted,
+write-only client secret. Scrape activity is placed on a shared BullMQ queue;
+index-worker replicas consume that queue and retry transient delivery failures.
+Queue jobs contain the organization ID and event, never the destination secret.
+
+Set `SIEM_LOGGING_ENCRYPTION_KEY` in API and index-worker environments. If
+partner egress is required, also set `PARTNER_EGRESS_PROXY_URL`.
+
+Use `POST /v2/team/siem/test` to validate credentials and the destination
+schema after deploying the Azure artifacts.
diff --git a/apps/siem/azure-sentinel/azuredeploy.json b/apps/siem/azure-sentinel/azuredeploy.json
new file mode 100644
index 000000000..955d7a188
--- /dev/null
+++ b/apps/siem/azure-sentinel/azuredeploy.json
@@ -0,0 +1,178 @@
+{
+ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+ "contentVersion": "1.0.0.0",
+ "parameters": {
+ "location": {
+ "type": "string",
+ "defaultValue": "[resourceGroup().location]"
+ },
+ "workspaceName": {
+ "type": "string"
+ },
+ "dataCollectionEndpointName": {
+ "type": "string",
+ "defaultValue": "firecrawl-scrape-activity"
+ },
+ "dataCollectionRuleName": {
+ "type": "string",
+ "defaultValue": "FirecrawlScrapeActivityPushDCR"
+ }
+ },
+ "variables": {
+ "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspaceName'))]",
+ "streamName": "Custom-FirecrawlScrapeActivity"
+ },
+ "resources": [
+ {
+ "type": "Microsoft.Insights/dataCollectionEndpoints",
+ "apiVersion": "2022-06-01",
+ "name": "[parameters('dataCollectionEndpointName')]",
+ "location": "[parameters('location')]",
+ "properties": {
+ "networkAcls": {
+ "publicNetworkAccess": "Enabled"
+ }
+ }
+ },
+ {
+ "type": "Microsoft.Insights/dataCollectionRules",
+ "apiVersion": "2021-09-01-preview",
+ "name": "[parameters('dataCollectionRuleName')]",
+ "location": "[parameters('location')]",
+ "dependsOn": [
+ "[resourceId('Microsoft.Insights/dataCollectionEndpoints', parameters('dataCollectionEndpointName'))]"
+ ],
+ "properties": {
+ "streamDeclarations": {
+ "Custom-FirecrawlScrapeActivity": {
+ "columns": [
+ {
+ "name": "schema_version",
+ "type": "int"
+ },
+ {
+ "name": "event_type",
+ "type": "string"
+ },
+ {
+ "name": "scrape_id",
+ "type": "string"
+ },
+ {
+ "name": "request_id",
+ "type": "string"
+ },
+ {
+ "name": "endpoint",
+ "type": "string"
+ },
+ {
+ "name": "team_id",
+ "type": "string"
+ },
+ {
+ "name": "org_id",
+ "type": "string"
+ },
+ {
+ "name": "api_key",
+ "type": "dynamic"
+ },
+ {
+ "name": "audit_metadata",
+ "type": "dynamic"
+ },
+ {
+ "name": "started_at",
+ "type": "datetime"
+ },
+ {
+ "name": "completed_at",
+ "type": "datetime"
+ },
+ {
+ "name": "url",
+ "type": "string"
+ },
+ {
+ "name": "domain",
+ "type": "string"
+ },
+ {
+ "name": "http_method",
+ "type": "string"
+ },
+ {
+ "name": "http_status",
+ "type": "int"
+ },
+ {
+ "name": "result",
+ "type": "string"
+ },
+ {
+ "name": "error",
+ "type": "dynamic"
+ },
+ {
+ "name": "origin",
+ "type": "string"
+ },
+ {
+ "name": "integration",
+ "type": "string"
+ },
+ {
+ "name": "zero_data_retention",
+ "type": "bool"
+ },
+ {
+ "name": "threat",
+ "type": "dynamic"
+ }
+ ]
+ }
+ },
+ "destinations": {
+ "logAnalytics": [
+ {
+ "workspaceResourceId": "[variables('workspaceResourceId')]",
+ "name": "firecrawlWorkspace"
+ }
+ ]
+ },
+ "dataFlows": [
+ {
+ "streams": ["Custom-FirecrawlScrapeActivity"],
+ "destinations": ["firecrawlWorkspace"],
+ "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tobool(threat.security_alert.detected), \"High\", tostring(threat.decision) == \"deny\", \"Low\", result == \"success\", \"Informational\", \"Low\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"audit_metadata\", audit_metadata, \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)",
+ "outputStream": "Microsoft-ASimWebSessionLogs"
+ }
+ ],
+ "dataCollectionEndpointId": "[resourceId('Microsoft.Insights/dataCollectionEndpoints', parameters('dataCollectionEndpointName'))]"
+ }
+ }
+ ],
+ "outputs": {
+ "streamName": {
+ "type": "string",
+ "value": "[variables('streamName')]"
+ },
+ "dataCollectionEndpointResourceId": {
+ "type": "string",
+ "value": "[resourceId('Microsoft.Insights/dataCollectionEndpoints', parameters('dataCollectionEndpointName'))]"
+ },
+ "dataCollectionEndpoint": {
+ "type": "string",
+ "value": "[reference(resourceId('Microsoft.Insights/dataCollectionEndpoints', parameters('dataCollectionEndpointName')), '2022-06-01').properties.logsIngestion.endpoint]"
+ },
+ "dataCollectionRuleResourceId": {
+ "type": "string",
+ "value": "[resourceId('Microsoft.Insights/dataCollectionRules', parameters('dataCollectionRuleName'))]"
+ },
+ "dataCollectionRuleImmutableId": {
+ "type": "string",
+ "value": "[reference(resourceId('Microsoft.Insights/dataCollectionRules', parameters('dataCollectionRuleName')), '2021-09-01-preview').properties.immutableId]"
+ }
+ }
+}
diff --git a/apps/siem/azure-sentinel/connectorDefinition.json b/apps/siem/azure-sentinel/connectorDefinition.json
new file mode 100644
index 000000000..bb503e2ee
--- /dev/null
+++ b/apps/siem/azure-sentinel/connectorDefinition.json
@@ -0,0 +1,143 @@
+{
+ "name": "FirecrawlScrapeActivityPush",
+ "apiVersion": "2022-09-01-preview",
+ "type": "Microsoft.SecurityInsights/dataConnectorDefinitions",
+ "location": "[parameters('workspace-location')]",
+ "kind": "Customizable",
+ "properties": {
+ "connectorUiConfig": {
+ "id": "FirecrawlScrapeActivityPush",
+ "title": "Firecrawl Scrape Activity (Push)",
+ "publisher": "Firecrawl",
+ "descriptionMarkdown": "Pushes Firecrawl scrape activity into Microsoft Sentinel through the Azure Monitor Logs Ingestion API. The customer-owned DCR normalizes native events into ASimWebSessionLogs.",
+ "graphQueries": [
+ {
+ "metricName": "Scrape activity",
+ "legend": "Firecrawl",
+ "baseQuery": "ASimWebSessionLogs | where EventVendor == \"Firecrawl\""
+ }
+ ],
+ "sampleQueries": [
+ {
+ "description": "Recent scrape activity",
+ "query": "ASimWebSessionLogs | where EventVendor == \"Firecrawl\" | sort by TimeGenerated desc"
+ },
+ {
+ "description": "Blocked activity and security alerts",
+ "query": "ASimWebSessionLogs | where EventVendor == \"Firecrawl\" and DvcAction == \"Deny\" | project TimeGenerated, SrcUsername, Url, RuleName, ThreatCategory, EventSeverity"
+ }
+ ],
+ "dataTypes": [
+ {
+ "name": "ASimWebSessionLogs",
+ "lastDataReceivedQuery": "ASimWebSessionLogs | where EventVendor == \"Firecrawl\" | summarize Time = max(TimeGenerated) | where isnotempty(Time)"
+ }
+ ],
+ "connectivityCriteria": [
+ {
+ "type": "IsConnectedQuery",
+ "value": [
+ "ASimWebSessionLogs | where EventVendor == \"Firecrawl\" | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(7d)"
+ ]
+ }
+ ],
+ "availability": {
+ "status": 1
+ },
+ "permissions": {
+ "resourceProvider": [
+ {
+ "provider": "Microsoft.OperationalInsights/workspaces",
+ "permissionsDisplayText": "Read and write permissions are required.",
+ "providerDisplayName": "Workspace",
+ "scope": "Workspace",
+ "requiredPermissions": {
+ "write": true,
+ "read": true,
+ "delete": true
+ }
+ }
+ ],
+ "customs": [
+ {
+ "name": "Microsoft Entra",
+ "description": "Permission to create an app registration and client secret."
+ },
+ {
+ "name": "Microsoft Azure",
+ "description": "Permission to assign Monitoring Metrics Publisher on the data collection rule."
+ }
+ ]
+ },
+ "instructionSteps": [
+ {
+ "title": "1. Deploy the push connector resources",
+ "description": "Create the data collection endpoint, data collection rule, Entra application, and role assignment.",
+ "instructions": [
+ {
+ "type": "Markdown",
+ "parameters": {
+ "content": "The DCR transforms Firecrawl's native scrape activity schema into ASimWebSessionLogs. Review the transform before deployment because it is controlled by your workspace."
+ }
+ },
+ {
+ "type": "DeployPushConnectorButton",
+ "parameters": {
+ "label": "Deploy Firecrawl push connector resources",
+ "applicationDisplayName": "Firecrawl Scrape Activity Push Connector"
+ }
+ }
+ ]
+ },
+ {
+ "title": "2. Configure Firecrawl",
+ "description": "Copy the generated values into the Firecrawl Enterprise Controls SIEM form.",
+ "instructions": [
+ {
+ "type": "CopyableLabel",
+ "parameters": {
+ "label": "Tenant ID",
+ "fillWith": ["TenantId"]
+ }
+ },
+ {
+ "type": "CopyableLabel",
+ "parameters": {
+ "label": "Application ID",
+ "fillWith": ["ApplicationId"]
+ }
+ },
+ {
+ "type": "CopyableLabel",
+ "parameters": {
+ "label": "Application Secret",
+ "fillWith": ["ApplicationSecret"]
+ }
+ },
+ {
+ "type": "CopyableLabel",
+ "parameters": {
+ "label": "Data Collection Endpoint",
+ "fillWith": ["DataCollectionEndpoint"]
+ }
+ },
+ {
+ "type": "CopyableLabel",
+ "parameters": {
+ "label": "DCR Immutable ID",
+ "fillWith": ["DataCollectionRuleId"]
+ }
+ },
+ {
+ "type": "CopyableLabel",
+ "parameters": {
+ "label": "Stream Name",
+ "value": "Custom-FirecrawlScrapeActivity"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ }
+}
diff --git a/apps/siem/azure-sentinel/dataConnector.json b/apps/siem/azure-sentinel/dataConnector.json
new file mode 100644
index 000000000..95ccffa73
--- /dev/null
+++ b/apps/siem/azure-sentinel/dataConnector.json
@@ -0,0 +1,25 @@
+{
+ "name": "FirecrawlScrapeActivityPushConnector",
+ "apiVersion": "2024-09-01",
+ "type": "Microsoft.SecurityInsights/dataConnectors",
+ "kind": "Push",
+ "properties": {
+ "connectorDefinitionName": "FirecrawlScrapeActivityPush",
+ "dcrConfig": {
+ "streamName": "Custom-FirecrawlScrapeActivity",
+ "dataCollectionEndpoint": "[[parameters('dcrConfig').dataCollectionEndpoint]]",
+ "dataCollectionRuleImmutableId": "[[parameters('dcrConfig').dataCollectionRuleImmutableId]]"
+ },
+ "auth": {
+ "type": "Push",
+ "AppId": "[[parameters('auth').appId]]",
+ "ServicePrincipalId": "[[parameters('auth').servicePrincipalId]]"
+ },
+ "request": {
+ "RetryCount": 1
+ },
+ "response": {
+ "eventsJsonPaths": ["$"]
+ }
+ }
+}
diff --git a/apps/siem/azure-sentinel/sample-event.json b/apps/siem/azure-sentinel/sample-event.json
new file mode 100644
index 000000000..e790e77a4
--- /dev/null
+++ b/apps/siem/azure-sentinel/sample-event.json
@@ -0,0 +1,39 @@
+[
+ {
+ "schema_version": 1,
+ "event_type": "scrape_activity",
+ "scrape_id": "00000000-0000-0000-0000-000000000001",
+ "request_id": "00000000-0000-0000-0000-000000000002",
+ "endpoint": "scrape",
+ "team_id": "00000000-0000-0000-0000-000000000003",
+ "org_id": "00000000-0000-0000-0000-000000000004",
+ "api_key": {
+ "id": "123",
+ "name": "production"
+ },
+ "audit_metadata": {
+ "username": "[email protected]"
+ },
+ "started_at": "2026-07-24T10:00:00.000Z",
+ "completed_at": "2026-07-24T10:00:01.500Z",
+ "url": "https://example.com/path?q=1",
+ "domain": "example.com",
+ "http_method": "GET",
+ "http_status": 200,
+ "result": "success",
+ "error": null,
+ "origin": "api",
+ "integration": "node-sdk",
+ "zero_data_retention": true,
+ "threat": {
+ "decision": "allow",
+ "rule": "default-allow",
+ "provider": "google-web-risk",
+ "categories": [],
+ "security_alert": {
+ "detected": false,
+ "category": null
+ }
+ }
+ }
+]