Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add replica-safe SIEM Logging (#4122)
Gergő Móricz28c8d30207fc
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 90fdab093..9554111c9 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -95,6 +95,12 @@ MENU_EXTRACTION_SERVICE_URL=PROXY_SERVER=PROXY_USERNAME=PROXY_PASSWORD=++# Enterprise SIEM Logging delivery. The encryption key must be 32 random bytes+# encoded as 64 hex characters or base64. Route both Entra and Azure Monitor+# ingestion calls through the dedicated partner egress proxy.+SIEM_LOGGING_ENCRYPTION_KEY=+PARTNER_EGRESS_PROXY_URL=# set if you'd like to block media requests to save proxy bandwidthBLOCK_MEDIA=diff --git a/apps/api/src/__tests__/snips/v2/team-siem-logging.test.ts b/apps/api/src/__tests__/snips/v2/team-siem-logging.test.tsnew file mode 100644index 000000000..6376ae37f--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/team-siem-logging.test.ts@@ -0,0 +1,107 @@+import request from "supertest";+import {+ describeIf,+ idmux,+ type Identity,+ TEST_API_URL,+ TEST_PRODUCTION,+} from "../lib";++async function getConfig(identity: Identity) {+ return request(TEST_API_URL)+ .get("/v2/team/siem")+ .set("Authorization", `Bearer ${identity.apiKey}`);+}++async function putConfig(identity: Identity, body: unknown) {+ return request(TEST_API_URL)+ .put("/v2/team/siem")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send(body as object);+}++describeIf(TEST_PRODUCTION)("Team SIEM logging config API", () => {+ it("rejects a team without the enterprise flag", async () => {+ const identity = await idmux({ name: "team-siem-logging/no-flag" });+ const response = await getConfig(identity);++ expect(response.statusCode).toBe(403);+ expect(response.body.error).toContain("enterprise feature");+ });++ describeIf(!!process.env.SIEM_LOGGING_ENCRYPTION_KEY)(+ "with the enterprise flag",+ () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "team-siem-logging/flagged",+ flags: { siemLogging: true },+ });+ });++ it("stores an Azure destination while keeping the secret write-only", async () => {+ const response = await putConfig(identity, {+ enabled: true,+ destination: {+ type: "azure_sentinel",+ tenantId: "00000000-0000-0000-0000-000000000001",+ clientId: "00000000-0000-0000-0000-000000000002",+ clientSecret: "never-return-this-value",+ dceUrl: "https://firecrawl-test.eastus.ingest.monitor.azure.com",+ dcrImmutableId: "dcr-00000000000000000000000000000000",+ streamName: "Custom-FirecrawlScrapeActivity",+ },+ });++ // A rollout can deploy API code before the out-of-band DDL. Match the+ // existing enterprise-config snips by making that state explicit.+ if (response.statusCode === 500) {+ console.warn(+ "siem_logging_config table is unavailable; skipping config round-trip",+ );+ return;+ }++ expect(response.statusCode).toBe(200);+ expect(response.body.data).toMatchObject({+ configured: true,+ enabled: true,+ destination: {+ type: "azure_sentinel",+ clientSecretConfigured: true,+ },+ });+ expect(JSON.stringify(response.body)).not.toContain(+ "never-return-this-value",+ );++ const read = await getConfig(identity);+ expect(read.statusCode).toBe(200);+ expect(JSON.stringify(read.body)).not.toContain(+ "never-return-this-value",+ );+ });++ it("rejects non-Azure ingestion endpoints", async () => {+ const response = await putConfig(identity, {+ enabled: true,+ destination: {+ type: "azure_sentinel",+ tenantId: "tenant",+ clientId: "client",+ clientSecret: "secret",+ dceUrl: "https://example.com/collect",+ dcrImmutableId: "dcr-id",+ streamName: "Custom-FirecrawlScrapeActivity",+ },+ });++ expect(response.statusCode).toBe(400);+ expect(response.body.success).toBe(false);+ });+ },+ );+});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex e29e1dbac..827e8bf06 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -81,6 +81,12 @@ const configSchema = z.object({.positive().default(6 * 60 * 60),+ // Organization SIEM logging delivery. The encryption key must decode to+ // exactly 32 bytes; validation happens when a secret is encrypted/decrypted+ // so self-hosted deployments that do not use this feature need no key.+ SIEM_LOGGING_ENCRYPTION_KEY: z.string().optional(),+ PARTNER_EGRESS_PROXY_URL: z.string().url().optional(),+// API Keys & AuthenticationBULL_AUTH_KEY: z.string().optional(),OPENAI_API_KEY: z.string().optional(),diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex d20906b9f..1e69f7bce 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -44,6 +44,8 @@ import {} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";+import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";+import { CrawlDenialError } from "../../lib/error";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -108,6 +110,7 @@ export async function batchScrapeController(let urls: string[] = req.body.urls;let unnormalizedURLs = preNormalizedBody.urls;let invalidURLs: string[] | undefined = undefined;+ const locallyBlockedURLs: string[] = [];if (req.body.ignoreInvalidURLs) {invalidURLs = [];@@ -129,21 +132,39 @@ export async function batchScrapeController(unnormalizedURLs.push(u);} else {invalidURLs.push(u);+ locallyBlockedURLs.push(nu);}} catch (_) {invalidURLs.push(u);}}} else {- if (- req.body.urls?.some((url: string) =>+ const blockedURLs =+ req.body.urls?.filter((url: string) =>isUrlBlocked(url, req.acuc?.flags ?? null, {team_id: req.auth.team_id,org_id: req.acuc?.org_id ?? null,origin: req.body.origin ?? null,}),- )- ) {+ ) ?? [];+ if (blockedURLs.length > 0) {+ locallyBlockedURLs.push(...blockedURLs);+ emitRejectedScrapeActivityEvents(+ locallyBlockedURLs.map(url => ({+ scrapeId: uuidv7(),+ requestId: id,+ endpoint: "batch_scrape",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.auditMetadata,+ url,+ error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention: zeroDataRetention ?? false,+ })),+ );+ locallyBlockedURLs.length = 0;if (!res.headersSent) {return res.status(403).json({success: false,@@ -153,6 +174,22 @@ export async function batchScrapeController(}}+ emitRejectedScrapeActivityEvents(+ locallyBlockedURLs.map(url => ({+ scrapeId: uuidv7(),+ requestId: id,+ endpoint: "batch_scrape",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.auditMetadata,+ url,+ error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention: zeroDataRetention ?? false,+ })),+ );+// Threat protection: reject/report blocked URLs at enqueue time so they// never consume scrape slots (mirrors the isUrlBlocked handling above).if (threatProtection.policy) {@@ -184,6 +221,25 @@ export async function batchScrapeController();});}+ emitRejectedScrapeActivityEvents(+ blocked.map(blockedUrl => ({+ scrapeId: uuidv7(),+ requestId: id,+ endpoint: "batch_scrape",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.auditMetadata,+ url: blockedUrl.url,+ error: new UnsafeDomainBlockedError(+ blockedUrl.url,+ blockedUrl.decision,+ ),+ threatDecisions: [blockedUrl.decision],+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention: zeroDataRetention ?? false,+ })),+ );if (req.body.ignoreInvalidURLs) {const blockedSet = new Set(blocked.map(x => x.url));const keptUnnormalized: string[] = [];diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex acdf87c15..4557802d5 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -35,6 +35,7 @@ import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";+import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -42,6 +43,9 @@ export async function crawlController() {const preNormalizedBody = req.body;req.body = crawlRequestSchema.parse(req.body);+ const id = uuidv7();+ const zeroDataRetention =+ getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;const threatProtection = await resolveThreatProtection({teamId: req.auth.team_id,@@ -91,6 +95,20 @@ export async function crawlController(});}const error = new UnsafeDomainBlockedError(req.body.url, decision);+ emitRejectedScrapeActivityEvent({+ scrapeId: uuidv7(),+ requestId: id,+ endpoint: "crawl",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.scrapeOptions?.auditMetadata,+ url: req.body.url,+ error,+ threatDecisions: [decision],+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention: zeroDataRetention ?? false,+ });return res.status(403).json({success: false,code: error.code,@@ -112,10 +130,6 @@ export async function crawlController(});}- const zeroDataRetention =- getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;-- const id = uuidv7();const logger = _logger.child({crawlId: id,module: "api/v1",diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex b1b778a31..9b4fbf893 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -30,6 +30,8 @@ import {import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";+import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";+import { CrawlDenialError } from "../../lib/error";async function oldExtract(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,res: Response<ExtractResponse>,@@ -111,6 +113,7 @@ export async function extractController(const selfHosted = config.USE_DB_AUTHENTICATION !== true;const originalRequest = { ...req.body };req.body = extractRequestSchema.parse(req.body);+ const extractId = uuidv7();if (getScrapeZDR(req.acuc?.flags) === "forced") {return res.status(400).json({@@ -129,6 +132,22 @@ export async function extractController(}),) ?? [];+ emitRejectedScrapeActivityEvents(+ invalidURLs.map(url => ({+ scrapeId: uuidv7(),+ requestId: extractId,+ endpoint: "extract",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.scrapeOptions?.auditMetadata,+ url,+ error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention: false,+ })),+ );+const createdAt = Date.now();if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) {@@ -181,6 +200,27 @@ export async function extractController(});}if (blocked.length > 0) {+ emitRejectedScrapeActivityEvents(+ blocked+ .filter(blockedUrl => !invalidURLs.includes(blockedUrl.url))+ .map(blockedUrl => ({+ scrapeId: uuidv7(),+ requestId: extractId,+ endpoint: "extract",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.scrapeOptions?.auditMetadata,+ url: blockedUrl.url,+ error: new UnsafeDomainBlockedError(+ blockedUrl.url,+ blockedUrl.decision,+ ),+ threatDecisions: [blockedUrl.decision],+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention: false,+ })),+ );if (req.body.ignoreInvalidURLs) {invalidURLs.push(...blocked.map(x => x.url));} else {@@ -195,8 +235,6 @@ export async function extractController(}}- const extractId = uuidv7();-_logger.info("Extract starting...", {request: req.body,originalRequest,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex b4db0a1bc..3600d5f5b 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -20,6 +20,7 @@ import { BrandingProfile } from "../../types/branding";import { ProductProfile } from "../../types/product";import { MenuProfile } from "../../types/menu";import { threatProtectionOverrideSchema } from "../../lib/threat-protection/config";+import { auditMetadataSchema } from "../../lib/siem-logging/types";type Format =| "markdown"@@ -532,6 +533,7 @@ const baseScrapeOptions = z.strictObject({// Enterprise: per-request field-level override of the org's threat// protection policy. Gated on the team flag + org config (checkPermissions).threatProtection: threatProtectionOverrideSchema.optional(),+ auditMetadata: auditMetadataSchema.optional(),// @deprecated__experimental_cache: z.boolean().prefault(false).optional(),__searchPreviewToken: z.string().optional(),@@ -978,6 +980,7 @@ const mapRequestSchemaBase = crawlerOptionslocation: locationSchema,headers: z.record(z.string(), z.string()).optional(),threatProtection: threatProtectionOverrideSchema.optional(),+ auditMetadata: auditMetadataSchema.optional(),});export const mapRequestSchema = mapRequestSchemaBase.strict();@@ -1275,6 +1278,7 @@ export type TeamFlags = {ignoreRobots?: "disabled" | "allowed" | "forced";customRobotsAgent?: "disabled" | "allowed";threatProtection?: "disabled" | "allowed" | "forced";+ siemLogging?: boolean;unblockedDomains?: string[];forceZDR?: boolean;allowZDR?: boolean;diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex 9cd86ef7b..86ee7f346 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -18,6 +18,7 @@ import {import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";+import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";export async function agentController(req: RequestWithAuth<{}, AgentResponse, AgentRequest>,@@ -97,6 +98,25 @@ export async function agentController(}const first = blocked[0];const error = new UnsafeDomainBlockedError(first.url, first.decision);+ emitRejectedScrapeActivityEvents(+ blocked.map(blockedUrl => ({+ scrapeId: uuidv7(),+ requestId: agentId,+ endpoint: "agent",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.auditMetadata,+ url: blockedUrl.url,+ error: new UnsafeDomainBlockedError(+ blockedUrl.url,+ blockedUrl.decision,+ ),+ threatDecisions: [blockedUrl.decision],+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention: false,+ })),+ );return res.status(403).json({success: false,code: error.code,@@ -158,6 +178,7 @@ export async function agentController(strictConstrainToURLs: req.body.strictConstrainToURLs ?? undefined,webhook: req.body.webhook ?? undefined,model: req.body.model,+ auditMetadata: req.body.auditMetadata,}),},);diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex d393df0e5..e4ac42355 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -45,6 +45,8 @@ import {import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";+import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";+import { CrawlDenialError } from "../../lib/error";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -129,6 +131,7 @@ export async function batchScrapeController(let urls: string[] = req.body.urls;let unnormalizedURLs = preNormalizedBody.urls;let invalidURLs: string[] | undefined = undefined;+ const locallyBlockedURLs: string[] = [];if (req.body.ignoreInvalidURLs) {invalidURLs = [];@@ -150,21 +153,39 @@ export async function batchScrapeController(unnormalizedURLs.push(u);} else {invalidURLs.push(u);+ locallyBlockedURLs.push(nu);}} catch (_) {invalidURLs.push(u);}}} else {- if (- req.body.urls?.some((url: string) =>+ const blockedURLs =+ req.body.urls?.filter((url: string) =>isUrlBlocked(url, req.acuc?.flags ?? null, {team_id: req.auth.team_id,org_id: req.acuc?.org_id ?? null,origin: req.body.origin ?? null,}),- )- ) {+ ) ?? [];+ if (blockedURLs.length > 0) {+ locallyBlockedURLs.push(...blockedURLs);+ emitRejectedScrapeActivityEvents(+ locallyBlockedURLs.map(url => ({+ scrapeId: uuidv7(),+ requestId: req.body.__agentInterop?.requestId ?? id,+ endpoint: req.body.__agentInterop ? "agent" : "batch_scrape",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.auditMetadata,+ url,+ error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention,+ })),+ );+ locallyBlockedURLs.length = 0;if (!res.headersSent) {return res.status(403).json({success: false,@@ -174,6 +195,22 @@ export async function batchScrapeController(}}+ emitRejectedScrapeActivityEvents(+ locallyBlockedURLs.map(url => ({+ scrapeId: uuidv7(),+ requestId: req.body.__agentInterop?.requestId ?? id,+ endpoint: req.body.__agentInterop ? "agent" : "batch_scrape",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.auditMetadata,+ url,+ error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention,+ })),+ );+// Threat protection: reject/report blocked URLs at enqueue time so they// never consume scrape slots. Mirrors the isUrlBlocked handling above:// with ignoreInvalidURLs they are reported in invalidURLs; without it the@@ -212,6 +249,25 @@ export async function batchScrapeController();});}+ emitRejectedScrapeActivityEvents(+ blocked.map(blockedUrl => ({+ scrapeId: uuidv7(),+ requestId: req.body.__agentInterop?.requestId ?? id,+ endpoint: req.body.__agentInterop ? "agent" : "batch_scrape",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.auditMetadata,+ url: blockedUrl.url,+ error: new UnsafeDomainBlockedError(+ blockedUrl.url,+ blockedUrl.decision,+ ),+ threatDecisions: [blockedUrl.decision],+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention,+ })),+ );if (req.body.ignoreInvalidURLs) {const blockedSet = new Set(blocked.map(x => x.url));const keptUnnormalized: string[] = [];diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 0ed8a5029..090aa94b2 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -37,6 +37,7 @@ import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";+import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -44,6 +45,9 @@ export async function crawlController() {const preNormalizedBody = req.body;req.body = crawlRequestSchema.parse(req.body);+ const id = uuidv7();+ const zeroDataRetention =+ getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;const threatProtection = await resolveThreatProtection({teamId: req.auth.team_id,@@ -95,6 +99,20 @@ export async function crawlController(});}const error = new UnsafeDomainBlockedError(req.body.url, decision);+ emitRejectedScrapeActivityEvent({+ scrapeId: uuidv7(),+ requestId: id,+ endpoint: "crawl",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.scrapeOptions?.auditMetadata,+ url: req.body.url,+ error,+ threatDecisions: [decision],+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention: zeroDataRetention ?? false,+ });return res.status(403).json({success: false,code: error.code,@@ -116,10 +134,6 @@ export async function crawlController(});}- const zeroDataRetention =- getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;-- const id = uuidv7();const logger = _logger.child({crawlId: id,module: "api/v2",diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex fb21af95f..5691cf498 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -21,6 +21,8 @@ import {import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";+import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";+import { CrawlDenialError } from "../../lib/error";/*** Extracts data from the provided URLs based on the request parameters.@@ -72,6 +74,22 @@ export async function extractController(}),) ?? [];+ emitRejectedScrapeActivityEvents(+ invalidURLs.map(url => ({+ scrapeId: uuidv7(),+ requestId: extractId,+ endpoint: "extract",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.scrapeOptions?.auditMetadata,+ url,+ error: new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE),+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention: false,+ })),+ );+if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) {if (!res.headersSent) {return res.status(403).json({@@ -122,6 +140,27 @@ export async function extractController(});}if (blocked.length > 0) {+ emitRejectedScrapeActivityEvents(+ blocked+ .filter(blockedUrl => !invalidURLs.includes(blockedUrl.url))+ .map(blockedUrl => ({+ scrapeId: uuidv7(),+ requestId: extractId,+ endpoint: "extract",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ auditMetadata: req.body.scrapeOptions?.auditMetadata,+ url: blockedUrl.url,+ error: new UnsafeDomainBlockedError(+ blockedUrl.url,+ blockedUrl.decision,+ ),+ threatDecisions: [blockedUrl.decision],+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention: false,+ })),+ );if (req.body.ignoreInvalidURLs) {invalidURLs.push(...blocked.map(x => x.url));} else {diff --git a/apps/api/src/controllers/v2/siem-logging-schema.test.ts b/apps/api/src/controllers/v2/siem-logging-schema.test.tsnew file mode 100644index 000000000..cc5fe808e--- /dev/null+++ b/apps/api/src/controllers/v2/siem-logging-schema.test.ts@@ -0,0 +1,171 @@+import { describe, expect, it } from "vitest";+import {+ agentRequestSchema,+ batchScrapeRequestSchema,+ crawlRequestSchema,+ extractRequestSchema,+ mapRequestSchema,+ scrapeRequestSchema,+ searchRequestSchema,+} from "./types";+import {+ batchScrapeRequestSchema as batchScrapeRequestSchemaV1,+ crawlRequestSchema as crawlRequestSchemaV1,+ extractRequestSchema as extractRequestSchemaV1,+ mapRequestSchema as mapRequestSchemaV1,+ scrapeRequestSchema as scrapeRequestSchemaV1,+ searchRequestSchema as searchRequestSchemaV1,+} from "../v1/types";+import { siemLoggingConfigInputSchema } from "../../lib/siem-logging/types";++const auditMetadata = {+ username: "[email protected]",+ session: "session-123",+};++describe("auditMetadata request validation", () => {+ it("accepts audit metadata across every scrape-producing request schema", () => {+ expect(+ scrapeRequestSchema.parse({+ url: "https://example.com",+ auditMetadata,+ }).auditMetadata,+ ).toEqual(auditMetadata);+ expect(+ batchScrapeRequestSchema.parse({+ urls: ["https://example.com"],+ auditMetadata,+ }).auditMetadata,+ ).toEqual(auditMetadata);+ expect(+ crawlRequestSchema.parse({+ url: "https://example.com",+ scrapeOptions: { auditMetadata },+ }).scrapeOptions.auditMetadata,+ ).toEqual(auditMetadata);+ expect(+ searchRequestSchema.parse({+ query: "example",+ scrapeOptions: { auditMetadata },+ }).scrapeOptions?.auditMetadata,+ ).toEqual(auditMetadata);+ expect(+ extractRequestSchema.parse({+ urls: ["https://example.com"],+ scrapeOptions: { auditMetadata },+ }).scrapeOptions?.auditMetadata,+ ).toEqual(auditMetadata);+ expect(+ mapRequestSchema.parse({+ url: "https://example.com",+ auditMetadata,+ }).auditMetadata,+ ).toEqual(auditMetadata);+ expect(+ agentRequestSchema.parse({+ prompt: "Find the pricing page",+ auditMetadata,+ }).auditMetadata,+ ).toEqual(auditMetadata);+ });++ it("rejects oversized metadata", () => {+ const tooManyFields = Object.fromEntries(+ Array.from({ length: 33 }, (_, index) => [`field-${index}`, "value"]),+ );+ expect(() =>+ scrapeRequestSchema.parse({+ url: "https://example.com",+ auditMetadata: tooManyFields,+ }),+ ).toThrow("at most 32 fields");+ });++ it("accepts audit metadata on the corresponding v1 request schemas", () => {+ expect(+ scrapeRequestSchemaV1.parse({+ url: "https://example.com",+ auditMetadata,+ }).auditMetadata,+ ).toEqual(auditMetadata);+ expect(+ batchScrapeRequestSchemaV1.parse({+ urls: ["https://example.com"],+ auditMetadata,+ }).auditMetadata,+ ).toEqual(auditMetadata);+ expect(+ crawlRequestSchemaV1.parse({+ url: "https://example.com",+ scrapeOptions: { auditMetadata },+ }).scrapeOptions.auditMetadata,+ ).toEqual(auditMetadata);+ expect(+ searchRequestSchemaV1.parse({+ query: "example",+ scrapeOptions: { auditMetadata },+ }).scrapeOptions.auditMetadata,+ ).toEqual(auditMetadata);+ expect(+ extractRequestSchemaV1.parse({+ urls: ["https://example.com"],+ scrapeOptions: { auditMetadata },+ }).scrapeOptions?.auditMetadata,+ ).toEqual(auditMetadata);+ expect(+ mapRequestSchemaV1.parse({+ url: "https://example.com",+ auditMetadata,+ }).auditMetadata,+ ).toEqual(auditMetadata);+ });+});++describe("SIEM destination validation", () => {+ const destination = {+ type: "azure_sentinel" as const,+ tenantId: "tenant",+ clientId: "client",+ clientSecret: "secret",+ dceUrl: "https://example.eastus.ingest.monitor.azure.com",+ dcrImmutableId: "dcr-id",+ streamName: "Custom-FirecrawlScrapeActivity",+ };++ it("trims credential and destination fields", () => {+ const parsed = siemLoggingConfigInputSchema.parse({+ enabled: true,+ destination: {+ ...destination,+ tenantId: " tenant ",+ clientId: " client ",+ clientSecret: " secret ",+ },+ });++ expect(parsed.destination).toMatchObject({+ tenantId: "tenant",+ clientId: "client",+ clientSecret: "secret",+ });+ });++ it("rejects whitespace-only credentials and endpoint userinfo", () => {+ expect(() =>+ siemLoggingConfigInputSchema.parse({+ enabled: true,+ destination: { ...destination, clientSecret: " " },+ }),+ ).toThrow();+ expect(() =>+ siemLoggingConfigInputSchema.parse({+ enabled: true,+ destination: {+ ...destination,+ dceUrl:+ "https://user:[email protected]",+ },+ }),+ ).toThrow("credential-free");+ });+});diff --git a/apps/api/src/controllers/v2/team-siem-logging.ts b/apps/api/src/controllers/v2/team-siem-logging.tsnew file mode 100644index 000000000..b69a141e2--- /dev/null+++ b/apps/api/src/controllers/v2/team-siem-logging.ts@@ -0,0 +1,176 @@+import { randomUUID } from "crypto";+import type { Response } from "express";+import type { RequestWithAuth } from "./types";+import { logger as _logger } from "../../lib/logger";+import { sendAzureSentinelEvents } from "../../lib/siem-logging/azure-sentinel";+import {+ getOrgSiemLoggingConfig,+ upsertOrgSiemLoggingConfig,+} from "../../lib/siem-logging/store";+import {+ SiemDeliveryError,+ siemLoggingConfigInputSchema,+ type OrgSiemLoggingConfig,+ type ScrapeActivityEvent,+} from "../../lib/siem-logging/types";+import { getOrgIdForTeam } from "../../lib/threat-protection/store";++const logger = _logger.child({ module: "team-siem-logging" });+const SUPPORT_EMAIL = "[email protected]";++function rejectWithoutFlag(+ req: RequestWithAuth<any, any, any>,+ res: Response,+): boolean {+ if (req.acuc?.flags?.siemLogging !== true) {+ res.status(403).json({+ success: false,+ error: `SIEM Logging is an enterprise feature and is not enabled for your team. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,+ });+ return true;+ }+ return false;+}++async function resolveOrgId(+ req: RequestWithAuth<any, any, any>,+ res: Response,+): Promise<string | null> {+ const orgId = req.auth.org_id ?? (await getOrgIdForTeam(req.auth.team_id));+ if (orgId) return orgId;+ logger.error("Failed to resolve org for SIEM configuration", {+ teamId: req.auth.team_id,+ });+ res.status(500).json({+ success: false,+ error: "Failed to resolve the organization for this team.",+ });+ return null;+}++function serializeConfig(config: OrgSiemLoggingConfig | null) {+ if (!config) return { configured: false, enabled: false, destination: null };+ const { clientSecret: _clientSecret, ...destination } = config.destination;+ return {+ configured: true,+ enabled: config.enabled,+ destination: {+ ...destination,+ clientSecretConfigured: true,+ },+ updatedAt: config.updatedAt,+ };+}++export async function getTeamSiemLoggingController(+ req: RequestWithAuth,+ res: Response,+): Promise<void> {+ if (rejectWithoutFlag(req, res)) return;+ const orgId = await resolveOrgId(req, res);+ if (!orgId) return;+ const config = await getOrgSiemLoggingConfig(orgId);+ res.status(200).json({ success: true, data: serializeConfig(config) });+}++export async function putTeamSiemLoggingController(+ req: RequestWithAuth<{}, any, unknown>,+ res: Response,+): Promise<void> {+ if (rejectWithoutFlag(req, res)) return;+ const input = siemLoggingConfigInputSchema.parse(req.body);+ const orgId = await resolveOrgId(req, res);+ if (!orgId) return;++ try {+ const updated = await upsertOrgSiemLoggingConfig(orgId, input);+ logger.info("SIEM logging configuration updated", {+ teamId: req.auth.team_id,+ orgId,+ enabled: updated.enabled,+ destinationType: updated.destination.type,+ streamName: updated.destination.streamName,+ });+ res.status(200).json({+ success: true,+ data: serializeConfig(updated),+ });+ } catch (error) {+ if (+ error instanceof Error &&+ error.message === "clientSecret is required when SIEM is first configured"+ ) {+ res.status(400).json({ success: false, error: error.message });+ return;+ }+ throw error;+ }+}++export async function testTeamSiemLoggingController(+ req: RequestWithAuth,+ res: Response,+): Promise<void> {+ if (rejectWithoutFlag(req, res)) return;+ const orgId = await resolveOrgId(req, res);+ if (!orgId) return;+ const siemConfig = await getOrgSiemLoggingConfig(orgId);+ if (!siemConfig?.enabled) {+ res.status(409).json({+ success: false,+ error: "SIEM Logging is not configured and enabled.",+ });+ return;+ }++ const now = new Date().toISOString();+ const id = randomUUID();+ const event: ScrapeActivityEvent = {+ schema_version: 1,+ event_type: "scrape_activity",+ scrape_id: id,+ request_id: id,+ endpoint: "scrape",+ team_id: req.auth.team_id,+ org_id: orgId,+ api_key: {+ id: req.acuc?.api_key_id == null ? null : String(req.acuc.api_key_id),+ name: null,+ },+ audit_metadata: { synthetic: "true" },+ started_at: now,+ completed_at: now,+ url: "https://example.com/siem-test",+ domain: "example.com",+ http_method: "GET",+ http_status: 200,+ result: "success",+ error: null,+ origin: "siem-test",+ integration: null,+ zero_data_retention: true,+ };++ try {+ await sendAzureSentinelEvents(siemConfig.destination, [event]);+ res.status(200).json({ success: true, data: { delivered: true } });+ } catch (error) {+ if (error instanceof SiemDeliveryError) {+ const status =+ error.kind === "invalid_credentials" ||+ error.kind === "schema_rejection"+ ? 400+ : 502;+ res.status(status).json({+ success: false,+ error: {+ type: error.kind,+ message: error.message,+ statusCode: error.statusCode ?? null,+ },+ });+ return;+ }+ throw error;+ }+}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex e73875a29..fe104e09d 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -29,6 +29,7 @@ import { BrandingProfile } from "../../types/branding";import { ProductProfile } from "../../types/product";import { MenuProfile } from "../../types/menu";import { threatProtectionOverrideSchema } from "../../lib/threat-protection/config";+import { auditMetadataSchema } from "../../lib/siem-logging/types";// Base URL schema with common validation logicexport const URL = z.preprocess(@@ -696,6 +697,7 @@ const baseScrapeOptions = z.strictObject({// Enterprise: per-request field-level override of the org's threat// protection policy. Gated on the team flag + org config (checkPermissions).threatProtection: threatProtectionOverrideSchema.optional(),+ auditMetadata: auditMetadataSchema.optional(),profile: z.object({@@ -946,6 +948,7 @@ export const agentRequestSchema = z.strictObject({overrideWhitelist: z.string().optional(),model: z.enum(["spark-1-pro", "spark-1-mini"]).default("spark-1-pro"),threatProtection: threatProtectionOverrideSchema.optional(),+ auditMetadata: auditMetadataSchema.optional(),});export type AgentRequest = z.infer<typeof agentRequestSchema>;@@ -1198,6 +1201,7 @@ const mapRequestSchemaBase = crawlerOptionslocation: locationSchema,headers: z.record(z.string(), z.string()).optional(),threatProtection: threatProtectionOverrideSchema.optional(),+ auditMetadata: auditMetadataSchema.optional(),});export const mapRequestSchema = strictWithMessage(mapRequestSchemaBase);@@ -1543,6 +1547,7 @@ export type TeamFlags = {ignoreRobots?: "disabled" | "allowed" | "forced";customRobotsAgent?: "disabled" | "allowed";threatProtection?: "disabled" | "allowed" | "forced";+ siemLogging?: boolean;unblockedDomains?: string[];forceZDR?: boolean;allowZDR?: boolean;diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex 5acb8afb4..af0485a39 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -703,6 +703,16 @@ export const threat_protection_config = pgTable("threat_protection_config", {updated_at: ts("updated_at").notNull().defaultNow(),});+export const siem_logging_config = pgTable("siem_logging_config", {+ id: uuid("id").notNull().defaultRandom(),+ org_id: uuid("org_id").notNull().unique(),+ enabled: boolean("enabled").notNull().default(false),+ destination: jsonb("destination").notNull().default({}),+ secret_ciphertext: text("secret_ciphertext").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+});+export const user_notifications = pgTable("user_notifications", {id: uuid("id").notNull().defaultRandom(),team_id: uuid("team_id"),diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex ed9b5d50a..177b7f7af 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -11,6 +11,7 @@ import {getDeepResearchQueue,getBillingQueue,getPrecrawlQueue,+ getSiemLoggingQueue,} from "./services/queue-service";import { v0Router } from "./routes/v0";import os from "os";@@ -103,6 +104,7 @@ const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({new BullMQAdapter(getDeepResearchQueue()),new BullMQAdapter(getBillingQueue()),new BullMQAdapter(getPrecrawlQueue()),+ new BullMQAdapter(getSiemLoggingQueue()),],serverAdapter: serverAdapter,});diff --git a/apps/api/src/lib/siem-logging/azure-sentinel.test.ts b/apps/api/src/lib/siem-logging/azure-sentinel.test.tsnew file mode 100644index 000000000..63e6145d3--- /dev/null+++ b/apps/api/src/lib/siem-logging/azure-sentinel.test.ts@@ -0,0 +1,331 @@+import { gunzipSync } from "zlib";+import { Response } from "undici";+import { afterEach, describe, expect, it, vi } from "vitest";+import type { AzureSentinelDestination, ScrapeActivityEvent } from "./types";+import {+ buildCompressedBatches,+ clearAzureTokenCache,+ sendAzureSentinelEvents,+ type SenderDependencies,+} from "./azure-sentinel";++const destination: AzureSentinelDestination = {+ type: "azure_sentinel",+ tenantId: "tenant",+ clientId: "client",+ clientSecret: "secret",+ dceUrl: "https://example.eastus.ingest.monitor.azure.com",+ dcrImmutableId: "dcr-immutable",+ streamName: "Custom-FirecrawlScrapeActivity",+};++function event(id = "scrape-id"): ScrapeActivityEvent {+ return {+ schema_version: 1,+ event_type: "scrape_activity",+ scrape_id: id,+ request_id: "request-id",+ endpoint: "scrape",+ team_id: "team-id",+ org_id: "org-id",+ api_key: { id: "1", name: "default" },+ audit_metadata: { username: "[email protected]" },+ started_at: "2026-07-24T00:00:00.000Z",+ completed_at: "2026-07-24T00:00:01.000Z",+ url: "https://example.com",+ domain: "example.com",+ http_method: "GET",+ http_status: 200,+ result: "success",+ error: null,+ origin: "api",+ integration: null,+ zero_data_retention: true,+ };+}++afterEach(() => clearAzureTokenCache());++describe("Azure Sentinel sender", () => {+ it("uses client credentials and sends a gzip JSON array", async () => {+ const calls: Array<{ url: string; init: any }> = [];+ const fetchImpl: SenderDependencies["fetchImpl"] = async (url, init) => {+ calls.push({ url, init });+ if (url.includes("/token")) {+ return new Response(+ JSON.stringify({ access_token: "access-token", expires_in: 3600 }),+ { status: 200, headers: { "content-type": "application/json" } },+ );+ }+ return new Response(null, { status: 204 });+ };++ await sendAzureSentinelEvents(destination, [event()], {+ fetchImpl,+ tokenUrl: "https://login.example.test/token",+ });++ expect(calls).toHaveLength(2);+ expect(calls[0].init.body).toContain("client_secret=secret");+ expect(calls[0].init.body).toContain(+ "scope=https%3A%2F%2Fmonitor.azure.com%2F%2F.default",+ );+ expect(calls[1].url).toContain(+ "/dataCollectionRules/dcr-immutable/streams/Custom-FirecrawlScrapeActivity?api-version=2023-01-01",+ );+ expect(calls[1].init.headers["content-encoding"]).toBe("gzip");+ expect(calls[1].init.headers.authorization).toBe("Bearer access-token");+ expect(JSON.parse(gunzipSync(calls[1].init.body).toString("utf8"))).toEqual(+ [event()],+ );+ });++ it("distinguishes invalid credentials from a DCR schema rejection", async () => {+ const invalidCredentials: SenderDependencies["fetchImpl"] = async () =>+ new Response("invalid_client", { status: 400 });+ await expect(+ sendAzureSentinelEvents(destination, [event()], {+ fetchImpl: invalidCredentials,+ tokenUrl: "https://login.example.test/token",+ }),+ ).rejects.toMatchObject({+ kind: "invalid_credentials",+ statusCode: 400,+ });++ clearAzureTokenCache();+ const schemaRejection: SenderDependencies["fetchImpl"] = async url =>+ url.includes("/token")+ ? new Response(+ JSON.stringify({ access_token: "token", expires_in: 3600 }),+ { status: 200, headers: { "content-type": "application/json" } },+ )+ : new Response("InvalidOutputTable", { status: 400 });+ await expect(+ sendAzureSentinelEvents(destination, [event()], {+ fetchImpl: schemaRejection,+ tokenUrl: "https://login.example.test/token",+ }),+ ).rejects.toMatchObject({+ kind: "schema_rejection",+ statusCode: 400,+ });+ });++ it("respects Retry-After and makes three total attempts on 429", async () => {+ let ingestionAttempts = 0;+ const sleep = vi.fn(async () => {});+ const fetchImpl: SenderDependencies["fetchImpl"] = async url => {+ if (url.includes("/token")) {+ return new Response(+ JSON.stringify({ access_token: "token", expires_in: 3600 }),+ { status: 200, headers: { "content-type": "application/json" } },+ );+ }+ ingestionAttempts++;+ return ingestionAttempts < 3+ ? new Response("slow down", {+ status: 429,+ headers: { "retry-after": "2" },+ })+ : new Response(null, { status: 204 });+ };++ await sendAzureSentinelEvents(destination, [event()], {+ fetchImpl,+ sleep,+ tokenUrl: "https://login.example.test/token",+ });++ expect(ingestionAttempts).toBe(3);+ expect(sleep).toHaveBeenNthCalledWith(1, 2000);+ expect(sleep).toHaveBeenNthCalledWith(2, 2000);+ });++ it("caps Retry-After and releases retry response bodies", async () => {+ const sleep = vi.fn(async () => {});+ const retryResponses: Response[] = [];+ let ingestionAttempts = 0;+ const fetchImpl: SenderDependencies["fetchImpl"] = async url => {+ if (url.includes("/token")) {+ return new Response(+ JSON.stringify({ access_token: "token", expires_in: 3600 }),+ { status: 200, headers: { "content-type": "application/json" } },+ );+ }+ ingestionAttempts++;+ if (ingestionAttempts < 3) {+ const response = new Response("try later", {+ status: 503,+ headers: { "retry-after": "86400" },+ });+ retryResponses.push(response);+ return response;+ }+ return new Response(null, { status: 204 });+ };++ await sendAzureSentinelEvents(destination, [event()], {+ fetchImpl,+ sleep,+ tokenUrl: "https://login.example.test/token",+ });++ expect(sleep).toHaveBeenNthCalledWith(1, 30_000);+ expect(sleep).toHaveBeenNthCalledWith(2, 30_000);+ expect(retryResponses.every(response => response.bodyUsed)).toBe(true);+ });++ it("exposes the capped Retry-After delay after the final attempt", async () => {+ const sleep = vi.fn(async () => {});+ const fetchImpl: SenderDependencies["fetchImpl"] = async url => {+ if (url.includes("/token")) {+ return new Response(+ JSON.stringify({ access_token: "token", expires_in: 3600 }),+ { status: 200 },+ );+ }+ return new Response("try later", {+ status: 429,+ headers: { "retry-after": "86400" },+ });+ };++ await expect(+ sendAzureSentinelEvents(destination, [event()], {+ fetchImpl,+ sleep,+ tokenUrl: "https://login.example.test/token",+ }),+ ).rejects.toMatchObject({+ kind: "rate_limited",+ retryAfterMs: 30_000,+ });+ });++ it("retries transient network failures three times", async () => {+ let ingestionAttempts = 0;+ const sleep = vi.fn(async () => {});+ const fetchImpl: SenderDependencies["fetchImpl"] = async url => {+ if (url.includes("/token")) {+ return new Response(+ JSON.stringify({ access_token: "token", expires_in: 3600 }),+ { status: 200, headers: { "content-type": "application/json" } },+ );+ }+ ingestionAttempts++;+ throw new Error("connection reset");+ };++ await expect(+ sendAzureSentinelEvents(destination, [event()], {+ fetchImpl,+ sleep,+ tokenUrl: "https://login.example.test/token",+ }),+ ).rejects.toMatchObject({ kind: "delivery_error" });++ expect(ingestionAttempts).toBe(3);+ expect(sleep).toHaveBeenNthCalledWith(1, 250);+ expect(sleep).toHaveBeenNthCalledWith(2, 500);+ });++ it("classifies ingestion authorization failures as invalid credentials", async () => {+ const fetchImpl: SenderDependencies["fetchImpl"] = async url =>+ url.includes("/token")+ ? new Response(+ JSON.stringify({ access_token: "token", expires_in: 3600 }),+ { status: 200, headers: { "content-type": "application/json" } },+ )+ : new Response("Forbidden", { status: 403 });++ await expect(+ sendAzureSentinelEvents(destination, [event()], {+ fetchImpl,+ tokenUrl: "https://login.example.test/token",+ }),+ ).rejects.toMatchObject({+ kind: "invalid_credentials",+ statusCode: 403,+ });+ });++ it("refreshes a rejected cached token once", async () => {+ let tokenRequests = 0;+ const ingestionTokens: string[] = [];+ const fetchImpl: SenderDependencies["fetchImpl"] = async (url, init) => {+ if (url.includes("/token")) {+ tokenRequests++;+ return new Response(+ JSON.stringify({+ access_token: `token-${tokenRequests}`,+ expires_in: 3600,+ }),+ { status: 200, headers: { "content-type": "application/json" } },+ );+ }+ ingestionTokens.push(init.headers.authorization);+ return init.headers.authorization === "Bearer token-1"+ ? new Response("revoked", { status: 401 })+ : new Response(null, { status: 204 });+ };++ await sendAzureSentinelEvents(destination, [event()], {+ fetchImpl,+ tokenUrl: "https://login.example.test/token",+ });++ expect(tokenRequests).toBe(2);+ expect(ingestionTokens).toEqual(["Bearer token-1", "Bearer token-2"]);+ });++ it("does not reuse a token after client-secret rotation", async () => {+ let tokenRequests = 0;+ const fetchImpl: SenderDependencies["fetchImpl"] = async url => {+ if (url.includes("/token")) {+ tokenRequests++;+ return new Response(+ JSON.stringify({+ access_token: `token-${tokenRequests}`,+ expires_in: 3600,+ }),+ { status: 200, headers: { "content-type": "application/json" } },+ );+ }+ return new Response(null, { status: 204 });+ };++ await sendAzureSentinelEvents(destination, [event("first")], {+ fetchImpl,+ tokenUrl: "https://login.example.test/token",+ });+ await sendAzureSentinelEvents(+ { ...destination, clientSecret: "rotated-secret" },+ [event("second")],+ { fetchImpl, tokenUrl: "https://login.example.test/token" },+ );++ expect(tokenRequests).toBe(2);+ });++ it("splits compressed payloads at the configured byte ceiling", () => {+ const noisyEvents = Array.from({ length: 8 }, (_, index) =>+ event(+ `${index}-${Array.from({ length: 30 }, () => crypto.randomUUID()).join("")}`,+ ),+ );+ const batches = buildCompressedBatches(noisyEvents, 1200);+ expect(Array.isArray(batches)).toBe(false);+ const compressedBodies = [...batches];++ expect(compressedBodies.length).toBeGreaterThan(1);+ expect(compressedBodies.every(batch => batch.byteLength <= 1200)).toBe(+ true,+ );+ expect(+ compressedBodies.flatMap(batch =>+ JSON.parse(gunzipSync(batch).toString("utf8")),+ ),+ ).toHaveLength(noisyEvents.length);+ });+});diff --git a/apps/api/src/lib/siem-logging/azure-sentinel.ts b/apps/api/src/lib/siem-logging/azure-sentinel.tsnew file mode 100644index 000000000..8bef81489--- /dev/null+++ b/apps/api/src/lib/siem-logging/azure-sentinel.ts@@ -0,0 +1,364 @@+import { gzipSync } from "zlib";+import { createHash, randomUUID } from "crypto";+import {+ fetch as undiciFetch,+ ProxyAgent,+ type Dispatcher,+ type Response,+} from "undici";+import { config } from "../../config";+import { siemLoggingDeliveryBatchesTotal } from "./metrics";+import type { AzureSentinelDestination, ScrapeActivityEvent } from "./types";+import { SiemDeliveryError } from "./types";++const MAX_COMPRESSED_BODY_BYTES = 1_000_000;+const MAX_ATTEMPTS = 3;+const MAX_RETRY_DELAY_MS = 30_000;+const MAX_TOKEN_CACHE_ENTRIES = 1000;+const REQUEST_TIMEOUT_MS = 15_000;++type FetchImpl = (+ input: string,+ init: {+ method: string;+ headers: Record<string, string>;+ body?: string | Buffer;+ dispatcher?: Dispatcher;+ signal?: AbortSignal;+ },+) => Promise<Response>;++export interface SenderDependencies {+ fetchImpl?: FetchImpl;+ sleep?: (ms: number) => Promise<void>;+ dispatcher?: Dispatcher;+ tokenUrl?: string;+}++interface CachedToken {+ token: string;+ expiresAt: number;+}++const tokenCache = new Map<string, CachedToken>();+let proxyDispatcher: ProxyAgent | undefined;+let proxyDispatcherUrl: string | undefined;++function getDispatcher(deps: SenderDependencies): Dispatcher | undefined {+ if (deps.dispatcher) return deps.dispatcher;+ if (!config.PARTNER_EGRESS_PROXY_URL) return undefined;+ if (+ !proxyDispatcher ||+ proxyDispatcherUrl !== config.PARTNER_EGRESS_PROXY_URL+ ) {+ proxyDispatcher = new ProxyAgent(config.PARTNER_EGRESS_PROXY_URL);+ proxyDispatcherUrl = config.PARTNER_EGRESS_PROXY_URL;+ }+ return proxyDispatcher;+}++function tokenCacheKey(destination: AzureSentinelDestination): string {+ const credentialDigest = createHash("sha256")+ .update(destination.clientSecret)+ .digest("base64url");+ return `${destination.tenantId}:${destination.clientId}:${credentialDigest}`;+}++function cacheToken(cacheKey: string, token: CachedToken): void {+ const now = Date.now();+ for (const [key, cached] of tokenCache) {+ if (cached.expiresAt <= now) tokenCache.delete(key);+ }+ tokenCache.delete(cacheKey);+ while (tokenCache.size >= MAX_TOKEN_CACHE_ENTRIES) {+ const oldestKey = tokenCache.keys().next().value;+ if (oldestKey === undefined) break;+ tokenCache.delete(oldestKey);+ }+ tokenCache.set(cacheKey, token);+}++function invalidateToken(destination: AzureSentinelDestination): void {+ tokenCache.delete(tokenCacheKey(destination));+}++function retryAfterMs(response: Response): number | undefined {+ const value = response.headers.get("retry-after");+ if (!value) return undefined;+ const seconds = Number(value);+ if (Number.isFinite(seconds)) {+ return Math.min(Math.max(0, seconds * 1000), MAX_RETRY_DELAY_MS);+ }+ const date = Date.parse(value);+ if (Number.isNaN(date)) return undefined;+ return Math.min(Math.max(0, date - Date.now()), MAX_RETRY_DELAY_MS);+}++async function errorText(response: Response): Promise<string> {+ const text = await response.text();+ return text.slice(0, 2048) || response.statusText || "Request failed";+}++async function waitBeforeRetry(+ attempt: number,+ response: Response,+ sleep: (ms: number) => Promise<void>,+): Promise<void> {+ const delay =+ retryAfterMs(response) ?? Math.min(250 * 2 ** (attempt - 1), 2000);+ await response.body?.cancel();+ await sleep(delay);+}++function networkErrorMessage(error: unknown): string {+ return error instanceof Error ? error.message : "Network request failed";+}++async function getAccessToken(+ destination: AzureSentinelDestination,+ deps: SenderDependencies,+): Promise<string> {+ const cacheKey = tokenCacheKey(destination);+ const cached = tokenCache.get(cacheKey);+ if (cached && cached.expiresAt > Date.now() + 60_000) return cached.token;++ const fetchImpl = deps.fetchImpl ?? (undiciFetch as FetchImpl);+ const sleep =+ deps.sleep ??+ ((ms: number) => new Promise(resolve => setTimeout(resolve, ms)));+ const dispatcher = getDispatcher(deps);+ const tokenUrl =+ deps.tokenUrl ??+ `https://login.microsoftonline.com/${encodeURIComponent(destination.tenantId)}/oauth2/v2.0/token`;+ const body = new URLSearchParams({+ client_id: destination.clientId,+ client_secret: destination.clientSecret,+ scope: "https://monitor.azure.com//.default",+ grant_type: "client_credentials",+ }).toString();++ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {+ let response: Response;+ try {+ response = await fetchImpl(tokenUrl, {+ method: "POST",+ headers: { "content-type": "application/x-www-form-urlencoded" },+ body,+ dispatcher,+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),+ });+ } catch (error) {+ if (attempt < MAX_ATTEMPTS) {+ await sleep(Math.min(250 * 2 ** (attempt - 1), 2000));+ continue;+ }+ throw new SiemDeliveryError(+ "delivery_error",+ `Failed to obtain an Entra token: ${networkErrorMessage(error)}`,+ );+ }++ if (response.ok) {+ const payload = (await response.json()) as {+ access_token?: unknown;+ expires_in?: unknown;+ };+ if (typeof payload.access_token !== "string") {+ throw new SiemDeliveryError(+ "invalid_credentials",+ "Entra token response did not include an access token",+ response.status,+ );+ }+ const expiresIn =+ typeof payload.expires_in === "number" ? payload.expires_in : 3600;+ cacheToken(cacheKey, {+ token: payload.access_token,+ expiresAt: Date.now() + expiresIn * 1000,+ });+ return payload.access_token;+ }++ if (+ (response.status === 429 || response.status >= 500) &&+ attempt < MAX_ATTEMPTS+ ) {+ await waitBeforeRetry(attempt, response, sleep);+ continue;+ }++ const message = await errorText(response);+ if (+ response.status === 400 ||+ response.status === 401 ||+ response.status === 403+ ) {+ throw new SiemDeliveryError(+ "invalid_credentials",+ `Entra rejected the SIEM credentials: ${message}`,+ response.status,+ );+ }+ throw new SiemDeliveryError(+ response.status === 429 ? "rate_limited" : "delivery_error",+ `Failed to obtain an Entra token: ${message}`,+ response.status,+ retryAfterMs(response),+ );+ }++ throw new SiemDeliveryError(+ "delivery_error",+ "Failed to obtain an Entra token",+ );+}++export function* buildCompressedBatches(+ events: ScrapeActivityEvent[],+ maxBytes = MAX_COMPRESSED_BODY_BYTES,+): Generator<Buffer> {+ let pending: ScrapeActivityEvent[] = [];++ for (const event of events) {+ const candidate = [...pending, event];+ const compressed = gzipSync(Buffer.from(JSON.stringify(candidate), "utf8"));+ if (compressed.byteLength <= maxBytes) {+ pending = candidate;+ continue;+ }++ if (pending.length === 0) {+ throw new SiemDeliveryError(+ "payload_too_large",+ `A single SIEM event exceeds the ${maxBytes}-byte compressed payload limit`,+ );+ }+ yield gzipSync(Buffer.from(JSON.stringify(pending), "utf8"));+ pending = [event];+ const single = gzipSync(Buffer.from(JSON.stringify(pending), "utf8"));+ if (single.byteLength > maxBytes) {+ throw new SiemDeliveryError(+ "payload_too_large",+ `A single SIEM event exceeds the ${maxBytes}-byte compressed payload limit`,+ );+ }+ }++ if (pending.length > 0) {+ yield gzipSync(Buffer.from(JSON.stringify(pending), "utf8"));+ }+}++async function sendBatch(+ destination: AzureSentinelDestination,+ token: string,+ body: Buffer,+ deps: SenderDependencies,+): Promise<void> {+ const fetchImpl = deps.fetchImpl ?? (undiciFetch as FetchImpl);+ const sleep =+ deps.sleep ??+ ((ms: number) => new Promise(resolve => setTimeout(resolve, ms)));+ const dispatcher = getDispatcher(deps);+ const url =+ `${destination.dceUrl.replace(/\/+$/, "")}/dataCollectionRules/` ++ `${encodeURIComponent(destination.dcrImmutableId)}/streams/` ++ `${encodeURIComponent(destination.streamName)}?api-version=2023-01-01`;++ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {+ let response: Response;+ try {+ response = await fetchImpl(url, {+ method: "POST",+ headers: {+ authorization: `Bearer ${token}`,+ "content-type": "application/json",+ "content-encoding": "gzip",+ "x-ms-client-request-id": randomUUID(),+ },+ body,+ dispatcher,+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),+ });+ } catch (error) {+ if (attempt < MAX_ATTEMPTS) {+ await sleep(Math.min(250 * 2 ** (attempt - 1), 2000));+ continue;+ }+ throw new SiemDeliveryError(+ "delivery_error",+ `Azure SIEM delivery failed: ${networkErrorMessage(error)}`,+ );+ }+ if (response.ok) return;++ if (+ (response.status === 429 || response.status >= 500) &&+ attempt < MAX_ATTEMPTS+ ) {+ await waitBeforeRetry(attempt, response, sleep);+ continue;+ }++ const message = await errorText(response);+ if (response.status >= 400 && response.status < 500) {+ throw new SiemDeliveryError(+ response.status === 401 || response.status === 403+ ? "invalid_credentials"+ : response.status === 429+ ? "rate_limited"+ : "schema_rejection",+ `Azure rejected the SIEM event batch: ${message}`,+ response.status,+ retryAfterMs(response),+ );+ }+ throw new SiemDeliveryError(+ "delivery_error",+ `Azure SIEM delivery failed: ${message}`,+ response.status,+ );+ }+}++export async function sendAzureSentinelEvents(+ destination: AzureSentinelDestination,+ events: ScrapeActivityEvent[],+ deps: SenderDependencies = {},+): Promise<void> {+ if (events.length === 0) return;+ let token: string | undefined;+ for (const body of buildCompressedBatches(events)) {+ try {+ token ??= await getAccessToken(destination, deps);+ try {+ await sendBatch(destination, token, body, deps);+ } catch (error) {+ if (+ error instanceof SiemDeliveryError &&+ (error.statusCode === 401 || error.statusCode === 403)+ ) {+ invalidateToken(destination);+ token = await getAccessToken(destination, deps);+ await sendBatch(destination, token, body, deps);+ } else {+ throw error;+ }+ }+ siemLoggingDeliveryBatchesTotal.inc({ result: "success" });+ } catch (error) {+ if (+ error instanceof SiemDeliveryError &&+ (error.statusCode === 401 || error.statusCode === 403)+ ) {+ invalidateToken(destination);+ }+ siemLoggingDeliveryBatchesTotal.inc({ result: "failure" });+ throw error;+ }+ }+}++export function clearAzureTokenCache(): void {+ tokenCache.clear();+}diff --git a/apps/api/src/lib/siem-logging/crypto.test.ts b/apps/api/src/lib/siem-logging/crypto.test.tsnew file mode 100644index 000000000..ea0c98a14--- /dev/null+++ b/apps/api/src/lib/siem-logging/crypto.test.ts@@ -0,0 +1,34 @@+import crypto from "crypto";+import { afterEach, describe, expect, it } from "vitest";+import { config } from "../../config";+import { decryptSiemSecret, encryptSiemSecret } from "./crypto";++const originalKey = config.SIEM_LOGGING_ENCRYPTION_KEY;++afterEach(() => {+ config.SIEM_LOGGING_ENCRYPTION_KEY = originalKey;+});++describe("SIEM secret encryption", () => {+ it("round-trips with AES-256-GCM without storing plaintext", () => {+ config.SIEM_LOGGING_ENCRYPTION_KEY = crypto.randomBytes(32).toString("hex");+ const encrypted = encryptSiemSecret("client-secret-value", "org-one");++ expect(encrypted).toMatch(/^gcm:/);+ expect(encrypted).not.toContain("client-secret-value");+ expect(decryptSiemSecret(encrypted, "org-one")).toBe("client-secret-value");+ expect(() => decryptSiemSecret(encrypted, "org-two")).toThrow();+ });++ it("refuses to store a secret without a valid external key", () => {+ config.SIEM_LOGGING_ENCRYPTION_KEY = undefined;+ expect(() => encryptSiemSecret("secret", "org-one")).toThrow(+ "SIEM_LOGGING_ENCRYPTION_KEY is not configured",+ );++ config.SIEM_LOGGING_ENCRYPTION_KEY = "too-short";+ expect(() => encryptSiemSecret("secret", "org-one")).toThrow(+ "must be a 32-byte",+ );+ });+});diff --git a/apps/api/src/lib/siem-logging/crypto.ts b/apps/api/src/lib/siem-logging/crypto.tsnew file mode 100644index 000000000..04cd3fc4e--- /dev/null+++ b/apps/api/src/lib/siem-logging/crypto.ts@@ -0,0 +1,64 @@+import crypto from "crypto";+import { config } from "../../config";++const GCM_PREFIX = "gcm:";++function getEncryptionKey(): Buffer {+ const raw = config.SIEM_LOGGING_ENCRYPTION_KEY?.trim();+ if (!raw) {+ throw new Error("SIEM_LOGGING_ENCRYPTION_KEY is not configured");+ }++ let key: Buffer | null = null;+ if (/^[0-9a-fA-F]{64}$/.test(raw)) {+ key = Buffer.from(raw, "hex");+ } else {+ const decoded = Buffer.from(raw, "base64");+ if (decoded.length === 32) key = decoded;+ }++ if (!key || key.length !== 32) {+ throw new Error(+ "SIEM_LOGGING_ENCRYPTION_KEY must be a 32-byte hex or base64 value",+ );+ }+ return key;+}++export function encryptSiemSecret(secret: string, orgId: string): string {+ const key = getEncryptionKey();+ const iv = crypto.randomBytes(12);+ const cipher = crypto.createCipheriv("aes-256-gcm", key, iv);+ cipher.setAAD(Buffer.from(orgId, "utf8"));+ const ciphertext = Buffer.concat([+ cipher.update(secret, "utf8"),+ cipher.final(),+ ]);+ const tag = cipher.getAuthTag();+ return `${GCM_PREFIX}${iv.toString("base64")}:${tag.toString("base64")}:${ciphertext.toString("base64")}`;+}++export function decryptSiemSecret(stored: string, orgId: string): string {+ if (!stored.startsWith(GCM_PREFIX)) {+ throw new Error("SIEM client secret is not encrypted with AES-256-GCM");+ }++ const [ivB64, tagB64, ciphertextB64] = stored+ .slice(GCM_PREFIX.length)+ .split(":");+ if (!ivB64 || !tagB64 || !ciphertextB64) {+ throw new Error("Malformed encrypted SIEM client secret");+ }++ const decipher = crypto.createDecipheriv(+ "aes-256-gcm",+ getEncryptionKey(),+ Buffer.from(ivB64, "base64"),+ );+ decipher.setAAD(Buffer.from(orgId, "utf8"));+ decipher.setAuthTag(Buffer.from(tagB64, "base64"));+ return Buffer.concat([+ decipher.update(Buffer.from(ciphertextB64, "base64")),+ decipher.final(),+ ]).toString("utf8");+}diff --git a/apps/api/src/lib/siem-logging/event.test.ts b/apps/api/src/lib/siem-logging/event.test.tsnew file mode 100644index 000000000..55d622597--- /dev/null+++ b/apps/api/src/lib/siem-logging/event.test.ts@@ -0,0 +1,202 @@+import { describe, expect, it } from "vitest";+import type { ScrapeJobSingleUrls } from "../../types";+import { scrapeOptions } from "../../controllers/v2/types";+import { UnsafeDomainBlockedError } from "../threat-protection/error";+import type { ThreatDecision } from "../threat-protection/types";+import { CrawlDenialError } from "../error";+import {+ buildRejectedScrapeActivityEvent,+ buildScrapeActivityEvent,+} from "./event";++const allowedDecision: ThreatDecision = {+ allowed: true,+ rule: "default-allow",+ url: "https://example.com/",+ domain: "example.com",+ providerConsulted: true,+ verdict: {+ provider: "google-web-risk",+ riskScore: 0,+ categories: [],+ fromCache: false,+ raw: { mustNotLeak: true },+ },+ mode: "normal",+};++function job(+ overrides: Partial<ScrapeJobSingleUrls> = {},+): ScrapeJobSingleUrls {+ return {+ mode: "single_urls",+ url: "https://example.com/private?q=1",+ team_id: "00000000-0000-0000-0000-000000000001",+ scrapeOptions: scrapeOptions.parse({+ auditMetadata: { username: "[email protected]" },+ }),+ internalOptions: {+ teamId: "00000000-0000-0000-0000-000000000001",+ orgId: "00000000-0000-0000-0000-000000000002",+ },+ origin: "api",+ integration: "sdk",+ zeroDataRetention: true,+ apiKeyId: 42,+ requestId: "00000000-0000-0000-0000-000000000003",+ billing: { endpoint: "search" },+ ...overrides,+ };+}++describe("ScrapeActivityEvent", () => {+ it("builds an attributable success event without raw classifier data", () => {+ const event = buildScrapeActivityEvent(+ "00000000-0000-0000-0000-000000000004",+ job(),+ "00000000-0000-0000-0000-000000000002",+ "production",+ {+ success: true,+ document: {+ markdown: "not included",+ metadata: { statusCode: 200, proxyUsed: "basic" },+ },+ threatDecisions: [allowedDecision],+ startedAt: 1_000,+ completedAt: 2_500,+ },+ );++ expect(event).toMatchObject({+ endpoint: "search",+ api_key: { id: "42", name: "production" },+ audit_metadata: { username: "[email protected]" },+ domain: "example.com",+ result: "success",+ zero_data_retention: true,+ threat: {+ decision: "allow",+ provider: "google-web-risk",+ categories: [],+ },+ });+ expect(JSON.stringify(event)).not.toContain("mustNotLeak");+ expect(JSON.stringify(event)).not.toContain("not included");+ });++ it("normalizes a blocked threat decision and error", () => {+ const denied: ThreatDecision = {+ ...allowedDecision,+ allowed: false,+ rule: "risk-score",+ verdict: {+ ...allowedDecision.verdict!,+ riskScore: 100,+ categories: ["MALWARE"],+ },+ };+ const error = new UnsafeDomainBlockedError(+ "https://example.com/private",+ denied,+ );+ const event = buildScrapeActivityEvent("scrape-id", job(), "org-id", null, {+ success: false,+ error,+ threatDecisions: [denied],+ startedAt: 1_000,+ completedAt: 2_000,+ });++ expect(event.result).toBe("blocked");+ expect(event.error?.code).toBe("unsafe_domain_blocked");+ expect(event.threat).toMatchObject({+ decision: "deny",+ rule: "risk-score",+ categories: ["MALWARE"],+ security_alert: { detected: true, category: "MALWARE" },+ });+ });++ it("serializes circular and bigint non-Error failures", () => {+ const thrown: { count: bigint; self?: unknown } = { count: 42n };+ thrown.self = thrown;+ const event = buildScrapeActivityEvent("scrape-id", job(), "org-id", null, {+ success: false,+ error: thrown,+ startedAt: 1_000,+ completedAt: 2_000,+ });++ expect(event.result).toBe("failure");+ expect(event.error?.message).toContain('"count":"42"');+ expect(event.error?.message).toContain("[Circular]");+ });++ it("builds blocked events for requests rejected before job creation", () => {+ const denied: ThreatDecision = {+ ...allowedDecision,+ allowed: false,+ rule: "risk-score",+ verdict: {+ ...allowedDecision.verdict!,+ categories: ["SOCIAL_ENGINEERING"],+ },+ };+ const error = new UnsafeDomainBlockedError(denied.url, denied);+ const event = buildRejectedScrapeActivityEvent(+ {+ scrapeId: "blocked-scrape-id",+ requestId: "parent-request-id",+ endpoint: "batch_scrape",+ teamId: "team-id",+ apiKeyId: 42,+ auditMetadata: { username: "[email protected]" },+ startedAt: 1_000,+ completedAt: 1_500,+ url: denied.url,+ error,+ threatDecisions: [denied],+ origin: "api",+ integration: "sdk",+ zeroDataRetention: true,+ },+ "org-id",+ "production",+ );++ expect(event).toMatchObject({+ scrape_id: "blocked-scrape-id",+ request_id: "parent-request-id",+ endpoint: "batch_scrape",+ result: "blocked",+ http_status: null,+ api_key: { id: "42", name: "production" },+ threat: {+ decision: "deny",+ categories: ["SOCIAL_ENGINEERING"],+ },+ });+ });++ it("classifies non-threat policy denials as blocked", () => {+ const event = buildRejectedScrapeActivityEvent(+ {+ scrapeId: "blocked-scrape-id",+ requestId: "parent-request-id",+ endpoint: "extract",+ teamId: "team-id",+ url: "https://example.com",+ error: new CrawlDenialError("URL blocked by policy"),+ origin: "api",+ zeroDataRetention: false,+ },+ "org-id",+ null,+ );++ expect(event.result).toBe("blocked");+ expect(event.error?.code).toBe("CRAWL_DENIAL");+ expect(event.threat).toBeUndefined();+ });+});diff --git a/apps/api/src/lib/siem-logging/event.ts b/apps/api/src/lib/siem-logging/event.tsnew file mode 100644index 000000000..8fa713dcd--- /dev/null+++ b/apps/api/src/lib/siem-logging/event.ts@@ -0,0 +1,250 @@+import type { Document } from "../../controllers/v2/types";+import type { ScrapeJobSingleUrls } from "../../types";+import type { ThreatDecision } from "../threat-protection/types";+import {+ CrawlDenialError,+ JobCancelledError,+ TransportableError,+} from "../error";+import { UnsafeDomainBlockedError } from "../threat-protection/error";+import type {+ AuditMetadata,+ ScrapeActivityEvent,+ ScrapeActivityResult,+ ScrapeActivityThreat,+} from "./types";++export interface ScrapeActivityOutcome {+ success: boolean;+ document?: Document | null;+ error?: unknown;+ threatDecisions?: ThreatDecision[];+ startedAt: number;+ completedAt: number;+}++export interface RejectedScrapeActivity {+ scrapeId: string;+ requestId: string;+ endpoint: ScrapeActivityEvent["endpoint"];+ teamId: string;+ apiKeyId?: string | number | null;+ auditMetadata?: AuditMetadata;+ startedAt?: number;+ completedAt?: number;+ url: string;+ error: unknown;+ threatDecisions?: ThreatDecision[];+ origin: string;+ integration?: string | null;+ zeroDataRetention: boolean;+}++function endpointForJob(+ job: ScrapeJobSingleUrls,+): ScrapeActivityEvent["endpoint"] {+ const endpoint = job.billing?.endpoint;+ if (+ endpoint === "scrape" ||+ endpoint === "crawl" ||+ endpoint === "batch_scrape" ||+ endpoint === "search" ||+ endpoint === "extract" ||+ endpoint === "agent" ||+ endpoint === "parse"+ ) {+ return endpoint;+ }+ if (job.internalOptions?.isParse) return "parse";+ if (job.from_extract) return "extract";+ if (job.crawl_id) {+ return job.crawlerOptions === null ? "batch_scrape" : "crawl";+ }+ return "scrape";+}++function resultForOutcome(+ outcome: ScrapeActivityOutcome,+): ScrapeActivityResult {+ if (outcome.success) return "success";+ if (+ outcome.error instanceof CrawlDenialError ||+ outcome.error instanceof UnsafeDomainBlockedError ||+ outcome.threatDecisions?.some(decision => !decision.allowed)+ ) {+ return "blocked";+ }+ if (outcome.error instanceof JobCancelledError) return "cancelled";+ return "failure";+}++function threatForDecisions(+ decisions: ThreatDecision[] | undefined,+): ScrapeActivityThreat | undefined {+ if (!decisions || decisions.length === 0) return undefined;+ const decision =+ decisions.find(candidate => !candidate.allowed) ??+ decisions[decisions.length - 1];+ const categories = [+ ...new Set(+ decisions.flatMap(candidate => candidate.verdict?.categories ?? []),+ ),+ ];+ return {+ decision: decision.allowed ? "allow" : "deny",+ rule: decision.rule,+ provider: decision.verdict?.provider ?? null,+ categories,+ security_alert: {+ detected: categories.length > 0,+ category: categories[0] ?? null,+ },+ };+}++function errorForOutcome(+ outcome: ScrapeActivityOutcome,+): ScrapeActivityEvent["error"] {+ if (outcome.success || outcome.error === undefined) return null;+ const error =+ outcome.error instanceof Error+ ? outcome.error+ : new Error(serializeThrownValue(outcome.error));+ return {+ code: error instanceof TransportableError ? error.code : null,+ message: error.message.slice(0, 2048),+ };+}++function serializeThrownValue(value: unknown): string {+ if (typeof value === "string") return value;+ try {+ const seen = new WeakSet<object>();+ const serialized = JSON.stringify(value, (_key, candidate: unknown) => {+ if (typeof candidate === "bigint") return candidate.toString();+ if (candidate && typeof candidate === "object") {+ if (seen.has(candidate)) return "[Circular]";+ seen.add(candidate);+ }+ return candidate;+ });+ return serialized ?? String(value);+ } catch {+ return "Non-Error value thrown";+ }+}++interface ScrapeActivityEventSource {+ scrapeId: string;+ requestId: string;+ endpoint: ScrapeActivityEvent["endpoint"];+ teamId: string;+ apiKeyId: string | number | null | undefined;+ auditMetadata: AuditMetadata | undefined;+ url: string;+ origin: string;+ integration: string | null | undefined;+ zeroDataRetention: boolean;+ httpStatus: number | null;+}++function buildEvent(+ source: ScrapeActivityEventSource,+ orgId: string,+ apiKeyName: string | null,+ outcome: ScrapeActivityOutcome,+): ScrapeActivityEvent {+ let domain = "";+ try {+ domain = new URL(source.url).hostname.toLowerCase();+ } catch {}++ const event: ScrapeActivityEvent = {+ schema_version: 1,+ event_type: "scrape_activity",+ scrape_id: source.scrapeId,+ request_id: source.requestId,+ endpoint: source.endpoint,+ team_id: source.teamId,+ org_id: orgId,+ api_key: {+ id: source.apiKeyId == null ? null : String(source.apiKeyId),+ name: apiKeyName,+ },+ audit_metadata: source.auditMetadata ?? {},+ started_at: new Date(outcome.startedAt).toISOString(),+ completed_at: new Date(outcome.completedAt).toISOString(),+ url: source.url,+ domain,+ http_method: "GET",+ http_status: source.httpStatus,+ result: resultForOutcome(outcome),+ error: errorForOutcome(outcome),+ origin: source.origin,+ integration: source.integration ?? null,+ zero_data_retention: source.zeroDataRetention,+ };+ const threat = threatForDecisions(outcome.threatDecisions);+ if (threat) event.threat = threat;+ return event;+}++export function buildScrapeActivityEvent(+ jobId: string,+ job: ScrapeJobSingleUrls,+ orgId: string,+ apiKeyName: string | null,+ outcome: ScrapeActivityOutcome,+): ScrapeActivityEvent {+ return buildEvent(+ {+ scrapeId: jobId,+ requestId: job.requestId ?? job.crawl_id ?? jobId,+ endpoint: endpointForJob(job),+ teamId: job.team_id,+ apiKeyId: job.apiKeyId,+ auditMetadata: job.scrapeOptions.auditMetadata,+ url: job.url,+ origin: job.origin,+ integration: job.integration,+ zeroDataRetention: job.zeroDataRetention,+ httpStatus: outcome.document?.metadata.statusCode ?? null,+ },+ orgId,+ apiKeyName,+ outcome,+ );+}++export function buildRejectedScrapeActivityEvent(+ input: RejectedScrapeActivity,+ orgId: string,+ apiKeyName: string | null,+): ScrapeActivityEvent {+ const startedAt = input.startedAt ?? Date.now();+ const outcome: ScrapeActivityOutcome = {+ success: false,+ error: input.error,+ threatDecisions: input.threatDecisions,+ startedAt,+ completedAt: input.completedAt ?? startedAt,+ };+ return buildEvent(+ {+ scrapeId: input.scrapeId,+ requestId: input.requestId,+ endpoint: input.endpoint,+ teamId: input.teamId,+ apiKeyId: input.apiKeyId,+ auditMetadata: input.auditMetadata,+ url: input.url,+ origin: input.origin,+ integration: input.integration,+ zeroDataRetention: input.zeroDataRetention,+ httpStatus: null,+ },+ orgId,+ apiKeyName,+ outcome,+ );+}diff --git a/apps/api/src/lib/siem-logging/index.test.ts b/apps/api/src/lib/siem-logging/index.test.tsnew file mode 100644index 000000000..78631474a--- /dev/null+++ b/apps/api/src/lib/siem-logging/index.test.ts@@ -0,0 +1,62 @@+import { describe, expect, it, vi } from "vitest";+import type { ScrapeJobSingleUrls } from "../../types";++const mocks = vi.hoisted(() => ({+ getTeam: vi.fn(),+ isEnabled: vi.fn().mockResolvedValue(true),+ getApiKeyName: vi.fn().mockResolvedValue("production"),+ enqueue: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("../../controllers/auth", () => ({+ getACUCTeam: mocks.getTeam,+}));++vi.mock("./store", () => ({+ isOrgSiemLoggingEnabled: mocks.isEnabled,+ getApiKeyName: mocks.getApiKeyName,+}));++vi.mock("./queue", () => ({+ enqueueSiemLoggingEvent: mocks.enqueue,+ enqueueSiemLoggingEvents: vi.fn().mockResolvedValue(undefined),+}));++import { emitScrapeActivityEvent } from "./index";++const job = {+ mode: "single_urls",+ url: "https://example.com",+ team_id: "team-id",+ scrapeOptions: {},+ internalOptions: { teamId: "team-id", orgId: "org-id" },+ origin: "api",+ zeroDataRetention: false,+ apiKeyId: 42,+} as ScrapeJobSingleUrls;++describe("SIEM logging emission", () => {+ it("does not wait for lookups or queue insertion", async () => {+ let resolveTeam!: (value: unknown) => void;+ mocks.getTeam.mockReturnValueOnce(+ new Promise(resolve => {+ resolveTeam = resolve;+ }),+ );++ expect(+ emitScrapeActivityEvent("scrape-id", job, {+ success: true,+ startedAt: 1_000,+ completedAt: 2_000,+ }),+ ).toBeUndefined();+ expect(mocks.enqueue).not.toHaveBeenCalled();++ resolveTeam({+ flags: { siemLogging: true },+ org_id: "org-id",+ });+ await vi.waitFor(() => expect(mocks.enqueue).toHaveBeenCalledOnce());+ });+});diff --git a/apps/api/src/lib/siem-logging/index.ts b/apps/api/src/lib/siem-logging/index.tsnew file mode 100644index 000000000..e400fdae9--- /dev/null+++ b/apps/api/src/lib/siem-logging/index.ts@@ -0,0 +1,112 @@+import type { ScrapeJobSingleUrls } from "../../types";+import { getACUCTeam } from "../../controllers/auth";+import { logger as _logger } from "../logger";+import {+ buildRejectedScrapeActivityEvent,+ buildScrapeActivityEvent,+ type RejectedScrapeActivity,+ type ScrapeActivityOutcome,+} from "./event";+import { enqueueSiemLoggingEvent, enqueueSiemLoggingEvents } from "./queue";+import { getApiKeyName, isOrgSiemLoggingEnabled } from "./store";++const logger = _logger.child({ module: "siem-logging" });++export function emitScrapeActivityEvent(+ jobId: string,+ job: ScrapeJobSingleUrls,+ outcome: ScrapeActivityOutcome,+): void {+ void emitScrapeActivityEventAsync(jobId, job, outcome);+}++async function emitScrapeActivityEventAsync(+ jobId: string,+ job: ScrapeJobSingleUrls,+ outcome: ScrapeActivityOutcome,+): Promise<void> {+ try {+ const team = await getACUCTeam(job.team_id);+ if (team?.flags?.siemLogging !== true) return;+ const orgId = job.internalOptions?.orgId ?? team.org_id;+ if (!orgId) return;++ if (!(await isOrgSiemLoggingEnabled(orgId))) return;++ const apiKeyName = await getApiKeyName(job.team_id, job.apiKeyId);+ const event = buildScrapeActivityEvent(+ jobId,+ job,+ orgId,+ apiKeyName,+ outcome,+ );+ await enqueueSiemLoggingEvent(orgId, event);+ } catch (error) {+ logger.error("Failed to enqueue scrape activity event", {+ error,+ jobId,+ teamId: job.team_id,+ });+ }+}++export function emitRejectedScrapeActivityEvent(+ input: RejectedScrapeActivity,+): void {+ emitRejectedScrapeActivityEvents([input]);+}++export function emitRejectedScrapeActivityEvents(+ inputs: RejectedScrapeActivity[],+): void {+ if (inputs.length === 0) return;+ void emitRejectedScrapeActivityEventsAsync(inputs);+}++async function emitRejectedScrapeActivityEventsAsync(+ inputs: RejectedScrapeActivity[],+): Promise<void> {+ try {+ const byTeam = new Map<string, RejectedScrapeActivity[]>();+ for (const input of inputs) {+ const teamInputs = byTeam.get(input.teamId) ?? [];+ teamInputs.push(input);+ byTeam.set(input.teamId, teamInputs);+ }++ for (const [teamId, teamInputs] of byTeam) {+ const team = await getACUCTeam(teamId);+ if (team?.flags?.siemLogging !== true) continue;+ const orgId = team.org_id;+ if (!orgId) continue;++ if (!(await isOrgSiemLoggingEnabled(orgId))) continue;++ const apiKeyNames = new Map<string | null, string | null>();+ const apiKeyIds = teamInputs.map(input =>+ input.apiKeyId == null ? null : String(input.apiKeyId),+ );+ await Promise.all(+ [...new Set(apiKeyIds)].map(async apiKeyId => {+ apiKeyNames.set(apiKeyId, await getApiKeyName(teamId, apiKeyId));+ }),+ );+ const events = teamInputs.map(input => {+ const apiKeyId = input.apiKeyId == null ? null : String(input.apiKeyId);+ return buildRejectedScrapeActivityEvent(+ input,+ orgId,+ apiKeyNames.get(apiKeyId) ?? null,+ );+ });+ await enqueueSiemLoggingEvents(orgId, events);+ }+ } catch (error) {+ logger.error("Failed to enqueue rejected scrape activity events", {+ error,+ eventCount: inputs.length,+ teamIds: [...new Set(inputs.map(input => input.teamId))],+ });+ }+}diff --git a/apps/api/src/lib/siem-logging/metrics.ts b/apps/api/src/lib/siem-logging/metrics.tsnew file mode 100644index 000000000..f8837e310--- /dev/null+++ b/apps/api/src/lib/siem-logging/metrics.ts@@ -0,0 +1,19 @@+import { Counter } from "prom-client";++export const siemLoggingEventsTotal = new Counter({+ name: "firecrawl_siem_logging_events_total",+ help: "SIEM logging event lifecycle outcomes",+ labelNames: ["result"] as const,+});++export const siemLoggingDeliveryBatchesTotal = new Counter({+ name: "firecrawl_siem_logging_delivery_batches_total",+ help: "SIEM delivery batches by outcome",+ labelNames: ["result"] as const,+});++export const siemLoggingDeliveryFailuresTotal = new Counter({+ name: "firecrawl_siem_logging_delivery_failures_total",+ help: "SIEM delivery failures by normalized reason",+ labelNames: ["reason"] as const,+});diff --git a/apps/api/src/lib/siem-logging/queue.test.ts b/apps/api/src/lib/siem-logging/queue.test.tsnew file mode 100644index 000000000..6fe00ee56--- /dev/null+++ b/apps/api/src/lib/siem-logging/queue.test.ts@@ -0,0 +1,58 @@+import { beforeEach, describe, expect, it, vi } from "vitest";+import type { ScrapeActivityEvent } from "./types";++const mocks = vi.hoisted(() => ({+ addBulk: vi.fn().mockResolvedValue([]),+ inc: vi.fn(),+}));++vi.mock("../../services/queue-service", () => ({+ getSiemLoggingQueue: () => ({ addBulk: mocks.addBulk }),+}));++vi.mock("./metrics", () => ({+ siemLoggingEventsTotal: { inc: mocks.inc },+}));++import { enqueueSiemLoggingEvent } from "./queue";++const event: ScrapeActivityEvent = {+ schema_version: 1,+ event_type: "scrape_activity",+ scrape_id: "scrape-id",+ request_id: "request-id",+ endpoint: "scrape",+ team_id: "team-id",+ org_id: "org-id",+ api_key: { id: null, name: null },+ audit_metadata: {},+ started_at: "2026-07-27T00:00:00.000Z",+ completed_at: "2026-07-27T00:00:01.000Z",+ url: "https://example.com",+ domain: "example.com",+ http_method: "GET",+ http_status: 200,+ result: "success",+ error: null,+ origin: "api",+ integration: null,+ zero_data_retention: false,+};++describe("SIEM logging queue", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("deduplicates repeat terminal events regardless of completion time", async () => {+ await enqueueSiemLoggingEvent("org-id", event);+ await enqueueSiemLoggingEvent("org-id", {+ ...event,+ completed_at: "2026-07-27T00:01:00.000Z",+ });++ const firstJobId = mocks.addBulk.mock.calls[0][0][0].opts.jobId;+ const secondJobId = mocks.addBulk.mock.calls[1][0][0].opts.jobId;+ expect(firstJobId).toBe(secondJobId);+ });+});diff --git a/apps/api/src/lib/siem-logging/queue.ts b/apps/api/src/lib/siem-logging/queue.tsnew file mode 100644index 000000000..45e343400--- /dev/null+++ b/apps/api/src/lib/siem-logging/queue.ts@@ -0,0 +1,46 @@+import { createHash } from "node:crypto";+import { getSiemLoggingQueue } from "../../services/queue-service";+import { siemLoggingEventsTotal } from "./metrics";+import type { ScrapeActivityEvent, SiemLoggingJobData } from "./types";++function jobIdFor(orgId: string, event: ScrapeActivityEvent): string {+ return createHash("sha256")+ .update(+ JSON.stringify([+ orgId,+ event.schema_version,+ event.event_type,+ event.scrape_id,+ event.request_id,+ event.result,+ ]),+ )+ .digest("hex");+}++export async function enqueueSiemLoggingEvent(+ orgId: string,+ event: ScrapeActivityEvent,+): Promise<void> {+ await enqueueSiemLoggingEvents(orgId, [event]);+}++export async function enqueueSiemLoggingEvents(+ orgId: string,+ events: ScrapeActivityEvent[],+): Promise<void> {+ if (events.length === 0) return;+ try {+ await getSiemLoggingQueue().addBulk(+ events.map(event => ({+ name: "deliver",+ data: { orgId, event } satisfies SiemLoggingJobData,+ opts: { jobId: jobIdFor(orgId, event) },+ })),+ );+ siemLoggingEventsTotal.inc({ result: "queued" }, events.length);+ } catch (error) {+ siemLoggingEventsTotal.inc({ result: "enqueue_failed" }, events.length);+ throw error;+ }+}diff --git a/apps/api/src/lib/siem-logging/store.test.ts b/apps/api/src/lib/siem-logging/store.test.tsnew file mode 100644index 000000000..003cab309--- /dev/null+++ b/apps/api/src/lib/siem-logging/store.test.ts@@ -0,0 +1,64 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ redisGet: vi.fn(),+ redisSet: vi.fn().mockResolvedValue("OK"),+ limit: vi.fn(),+}));++const query = {+ from: vi.fn(() => query),+ where: vi.fn(() => query),+ limit: mocks.limit,+};++vi.mock("../../db/connection", () => ({+ db: { select: vi.fn(() => query) },+ dbRr: { select: vi.fn(() => query) },+}));++vi.mock("../../services/queue-service", () => ({+ getRedisConnection: () => ({+ get: mocks.redisGet,+ set: mocks.redisSet,+ }),+}));++import { isOrgSiemLoggingEnabled } from "./store";++describe("SIEM logging enabled cache", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mocks.redisGet.mockResolvedValue(null);+ });++ it("uses an organization-specific Redis Cluster hash tag", async () => {+ mocks.limit.mockResolvedValue([{ enabled: true }]);++ await expect(isOrgSiemLoggingEnabled("org-one")).resolves.toBe(true);++ expect(mocks.redisGet).toHaveBeenCalledWith(+ "{siemLoggingConfig:org-one}:enabled",+ );+ expect(mocks.redisSet).toHaveBeenCalledWith(+ "{siemLoggingConfig:org-one}:enabled",+ "1",+ "EX",+ 60,+ );+ });++ it("treats a missing configuration table as disabled", async () => {+ mocks.limit.mockRejectedValue({+ cause: { code: "42P01" },+ });++ await expect(isOrgSiemLoggingEnabled("org-two")).resolves.toBe(false);+ expect(mocks.redisSet).toHaveBeenCalledWith(+ "{siemLoggingConfig:org-two}:enabled",+ "0",+ "EX",+ 60,+ );+ });+});diff --git a/apps/api/src/lib/siem-logging/store.ts b/apps/api/src/lib/siem-logging/store.tsnew file mode 100644index 000000000..4c69885f3--- /dev/null+++ b/apps/api/src/lib/siem-logging/store.ts@@ -0,0 +1,227 @@+import { and, eq, sql } from "drizzle-orm";+import { z } from "zod";+import { db, dbRr } from "../../db/connection";+import * as schema from "../../db/schema";+import { getRedisConnection } from "../../services/queue-service";+import { logger as _logger } from "../logger";+import { decryptSiemSecret, encryptSiemSecret } from "./crypto";+import type { OrgSiemLoggingConfig, SiemLoggingConfigInput } from "./types";++const CACHE_TTL_MS = 60_000;+const CACHE_MAX_ENTRIES = 1000;+const ENABLED_CACHE_TTL_SECONDS = 60;+const logger = _logger.child({ module: "siem-logging-store" });+type SiemConfigRow = typeof schema.siem_logging_config.$inferSelect;++const configCache = new Map<+ string,+ { expiresAt: number; value: SiemConfigRow | null }+>();++function enabledCacheKey(orgId: string): string {+ return `{siemLoggingConfig:${orgId}}:enabled`;+}++const storedDestinationSchema = z.strictObject({+ type: z.literal("azure_sentinel"),+ tenantId: z.string(),+ clientId: z.string(),+ dceUrl: z.string(),+ dcrImmutableId: z.string(),+ streamName: z.string(),+});++function isMissingTableError(error: unknown): boolean {+ let current: unknown = error;+ for (let depth = 0; depth < 5 && current; depth++) {+ if ((current as { code?: unknown }).code === "42P01") return true;+ current = (current as { cause?: unknown }).cause;+ }+ return false;+}++function rowToConfig(row: SiemConfigRow): OrgSiemLoggingConfig {+ const destination = storedDestinationSchema.parse(row.destination);+ return {+ orgId: row.org_id,+ enabled: row.enabled,+ destination: {+ ...destination,+ clientSecret: decryptSiemSecret(row.secret_ciphertext, row.org_id),+ },+ createdAt: row.created_at,+ updatedAt: row.updated_at,+ };+}++function getCachedRow(orgId: string): SiemConfigRow | null | undefined {+ const cached = configCache.get(orgId);+ if (!cached) return undefined;+ if (cached.expiresAt <= Date.now()) {+ configCache.delete(orgId);+ return undefined;+ }+ configCache.delete(orgId);+ configCache.set(orgId, cached);+ return cached.value;+}++function cacheRow(orgId: string, value: SiemConfigRow | null): void {+ const now = Date.now();+ for (const [cachedOrgId, cached] of configCache) {+ if (cached.expiresAt <= now) configCache.delete(cachedOrgId);+ }+ configCache.delete(orgId);+ while (configCache.size >= CACHE_MAX_ENTRIES) {+ const oldestOrgId = configCache.keys().next().value;+ if (oldestOrgId === undefined) break;+ configCache.delete(oldestOrgId);+ }+ configCache.set(orgId, {+ expiresAt: now + CACHE_TTL_MS,+ value,+ });+}++export async function getOrgSiemLoggingConfig(+ orgId: string,+ options: { fresh?: boolean; primary?: boolean } = {},+): Promise<OrgSiemLoggingConfig | null> {+ const row = await getOrgSiemLoggingConfigRow(orgId, options);+ return row ? rowToConfig(row) : null;+}++async function getOrgSiemLoggingConfigRow(+ orgId: string,+ options: { fresh?: boolean; primary?: boolean } = {},+): Promise<SiemConfigRow | null> {+ if (!options.fresh) {+ const cached = getCachedRow(orgId);+ if (cached !== undefined) return cached;+ }++ let rows: SiemConfigRow[];+ try {+ rows = await (options.primary ? db : dbRr)+ .select()+ .from(schema.siem_logging_config)+ .where(eq(schema.siem_logging_config.org_id, orgId))+ .limit(1);+ } catch (error) {+ if (!isMissingTableError(error)) throw error;+ rows = [];+ }++ const row = rows[0] ?? null;+ cacheRow(orgId, row);+ return row;+}++export async function isOrgSiemLoggingEnabled(orgId: string): Promise<boolean> {+ const key = enabledCacheKey(orgId);+ const redis = getRedisConnection();+ const cached = await redis.get(key);+ if (cached !== null) return cached === "1";++ let rows: { enabled: boolean }[] = [];+ try {+ rows = await db+ .select({ enabled: schema.siem_logging_config.enabled })+ .from(schema.siem_logging_config)+ .where(eq(schema.siem_logging_config.org_id, orgId))+ .limit(1);+ } catch (error) {+ if (!isMissingTableError(error)) throw error;+ }+ const enabled = rows[0]?.enabled === true;+ await redis.set(key, enabled ? "1" : "0", "EX", ENABLED_CACHE_TTL_SECONDS);+ return enabled;+}++export async function upsertOrgSiemLoggingConfig(+ orgId: string,+ input: SiemLoggingConfigInput,+): Promise<OrgSiemLoggingConfig> {+ const destination = {+ type: input.destination.type,+ tenantId: input.destination.tenantId,+ clientId: input.destination.clientId,+ dceUrl: input.destination.dceUrl.replace(/\/+$/, ""),+ dcrImmutableId: input.destination.dcrImmutableId,+ streamName: input.destination.streamName,+ };+ const existingRows = await db+ .select({+ secret_ciphertext: schema.siem_logging_config.secret_ciphertext,+ })+ .from(schema.siem_logging_config)+ .where(eq(schema.siem_logging_config.org_id, orgId))+ .limit(1);++ const existingCiphertext = existingRows[0]?.secret_ciphertext;+ if (!input.destination.clientSecret && !existingCiphertext) {+ throw new Error("clientSecret is required when SIEM is first configured");+ }++ const secretCiphertext = input.destination.clientSecret+ ? encryptSiemSecret(input.destination.clientSecret, orgId)+ : existingCiphertext!;+ const values = {+ org_id: orgId,+ enabled: input.enabled,+ destination,+ secret_ciphertext: secretCiphertext,+ };+ const [row] = await db+ .insert(schema.siem_logging_config)+ .values(values)+ .onConflictDoUpdate({+ target: schema.siem_logging_config.org_id,+ set: {+ enabled: values.enabled,+ destination: values.destination,+ secret_ciphertext: input.destination.clientSecret+ ? values.secret_ciphertext+ : sql`${schema.siem_logging_config.secret_ciphertext}`,+ updated_at: new Date().toISOString(),+ },+ })+ .returning();++ const updated = rowToConfig(row);+ cacheRow(orgId, row);+ await getRedisConnection()+ .set(+ enabledCacheKey(orgId),+ row.enabled ? "1" : "0",+ "EX",+ ENABLED_CACHE_TTL_SECONDS,+ )+ .catch(error => {+ logger.warn("Failed to update shared SIEM logging config cache", {+ error,+ orgId,+ });+ });+ return updated;+}++export async function getApiKeyName(+ teamId: string,+ apiKeyId: string | number | null | undefined,+): Promise<string | null> {+ if (apiKeyId == null) return null;+ const normalizedApiKeyId = String(apiKeyId);+ if (!/^\d+$/.test(normalizedApiKeyId)) return null;+ const rows = await dbRr+ .select({ name: schema.api_keys.name })+ .from(schema.api_keys)+ .where(+ and(+ sql`${schema.api_keys.id} = cast(${normalizedApiKeyId} as bigint)`,+ eq(schema.api_keys.team_id, teamId),+ ),+ )+ .limit(1);+ return rows[0]?.name ?? null;+}diff --git a/apps/api/src/lib/siem-logging/types.ts b/apps/api/src/lib/siem-logging/types.tsnew file mode 100644index 000000000..c6f4b3af1--- /dev/null+++ b/apps/api/src/lib/siem-logging/types.ts@@ -0,0 +1,150 @@+import { z } from "zod";+import type {+ ThreatDecisionRule,+ ThreatProvider,+} from "../threat-protection/types";++export const auditMetadataSchema = z+ .record(z.string().min(1).max(64), z.string().max(1024))+ .refine(value => Object.keys(value).length <= 32, {+ message: "auditMetadata may contain at most 32 fields",+ });++export type AuditMetadata = z.infer<typeof auditMetadataSchema>;++const azureSentinelDestinationInputSchema = z.strictObject({+ type: z.literal("azure_sentinel"),+ tenantId: z.string().trim().min(1).max(128),+ clientId: z.string().trim().min(1).max(128),+ clientSecret: z.string().trim().min(1).max(4096).optional(),+ dceUrl: z+ .string()+ .trim()+ .pipe(+ z.url().refine(value => {+ const url = new URL(value);+ return (+ url.protocol === "https:" &&+ url.username === "" &&+ url.password === "" &&+ (url.hostname === "ingest.monitor.azure.com" ||+ url.hostname.endsWith(".ingest.monitor.azure.com"))+ );+ }, "dceUrl must be a credential-free Azure Monitor ingestion endpoint"),+ ),+ dcrImmutableId: z.string().trim().min(1).max(256),+ streamName: z+ .string()+ .trim()+ .min(1)+ .max(256)+ .refine(value => value.startsWith("Custom-"), {+ message: "streamName must start with Custom-",+ }),+});++export const siemLoggingConfigInputSchema = z.strictObject({+ enabled: z.boolean(),+ destination: azureSentinelDestinationInputSchema,+});++export type SiemLoggingConfigInput = z.infer<+ typeof siemLoggingConfigInputSchema+>;++export interface AzureSentinelDestination {+ type: "azure_sentinel";+ tenantId: string;+ clientId: string;+ clientSecret: string;+ dceUrl: string;+ dcrImmutableId: string;+ streamName: string;+}++export interface OrgSiemLoggingConfig {+ orgId: string;+ enabled: boolean;+ destination: AzureSentinelDestination;+ createdAt: string | null;+ updatedAt: string | null;+}++export type ScrapeActivityResult =+ | "success"+ | "failure"+ | "blocked"+ | "cancelled";++export interface ScrapeActivityThreat {+ decision: "allow" | "deny";+ rule: ThreatDecisionRule;+ provider: ThreatProvider | null;+ categories: string[];+ security_alert: {+ detected: boolean;+ category: string | null;+ };+}++export interface ScrapeActivityEvent {+ schema_version: 1;+ event_type: "scrape_activity";+ scrape_id: string;+ request_id: string;+ endpoint:+ | "scrape"+ | "crawl"+ | "batch_scrape"+ | "search"+ | "extract"+ | "agent"+ | "parse"+ | "unknown";+ team_id: string;+ org_id: string;+ api_key: {+ id: string | null;+ name: string | null;+ };+ audit_metadata: AuditMetadata;+ started_at: string;+ completed_at: string;+ url: string;+ domain: string;+ http_method: "GET";+ http_status: number | null;+ result: ScrapeActivityResult;+ error: {+ code: string | null;+ message: string;+ } | null;+ origin: string;+ integration: string | null;+ zero_data_retention: boolean;+ threat?: ScrapeActivityThreat;+}++export interface SiemLoggingJobData {+ orgId: string;+ event: ScrapeActivityEvent;+}++type SiemDeliveryErrorKind =+ | "invalid_credentials"+ | "schema_rejection"+ | "rate_limited"+ | "delivery_error"+ | "payload_too_large";++export class SiemDeliveryError extends Error {+ constructor(+ public readonly kind: SiemDeliveryErrorKind,+ message: string,+ public readonly statusCode?: number,+ public readonly retryAfterMs?: number,+ ) {+ super(message);+ this.name = "SiemDeliveryError";+ }+}diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 3cc6ef36e..a2767c5f2 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -70,6 +70,11 @@ import {getTeamThreatProtectionController,putTeamThreatProtectionController,} from "../controllers/v2/team-threat-protection";+import {+ getTeamSiemLoggingController,+ putTeamSiemLoggingController,+ testTeamSiemLoggingController,+} from "../controllers/v2/team-siem-logging";import { supportProxyController } from "../controllers/v2/support-proxy";import { createResearchRouter } from "../controllers/v2/research-proxy";import {@@ -439,6 +444,24 @@ v2Router.put(wrap(putTeamThreatProtectionController),);+v2Router.get(+ "/team/siem",+ authMiddleware(RateLimiterMode.Account),+ wrap(getTeamSiemLoggingController),+);++v2Router.put(+ "/team/siem",+ authMiddleware(RateLimiterMode.Account),+ wrap(putTeamSiemLoggingController),+);++v2Router.post(+ "/team/siem/test",+ authMiddleware(RateLimiterMode.Account),+ wrap(testTeamSiemLoggingController),+);+v2Router.post("/monitor",authMiddleware(RateLimiterMode.Crawl),diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 4858328ab..5bb2efa9a 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -9,6 +9,7 @@ import {getRedisConnection,getBillingQueue,getPrecrawlQueue,+ getSiemLoggingQueue,precrawlQueueName,} from "../queue-service";import {@@ -42,6 +43,7 @@ import { crawlGroup, resolveNewGroupBackend } from "../worker/nuq-router";import { getACUCTeam } from "../../controllers/auth";import { processEngpickerJob } from "../../lib/engpicker";import { logRequest } from "../logging/log_job";+import { processSiemLoggingJob } from "../siem-logging/worker";const workerLockDuration = config.WORKER_LOCK_DURATION;const workerStalledCheckInterval = config.WORKER_STALLED_CHECK_INTERVAL;@@ -698,6 +700,10 @@ const BROWSER_ACTIVITY_INSERT_INTERVAL = 10000;: (async () => {logger.warn("PRECRAWL_TEAM_ID not set, skipping precrawl worker");})();+ const siemLoggingWorkerPromise = workerFun(+ getSiemLoggingQueue(),+ processSiemLoggingJob,+ );const indexInserterInterval = setInterval(async () => {if (isShuttingDown) {@@ -779,6 +785,7 @@ const BROWSER_ACTIVITY_INSERT_INTERVAL = 10000;await Promise.all([billingWorkerPromise,precrawlWorkerPromise,+ siemLoggingWorkerPromise,engpickerPromise,]);diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 04dc0b263..1c5bf0ccd 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -3,6 +3,7 @@ import { config } from "../config";import { logger } from "../lib/logger";import IORedis from "ioredis";import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";+import type { SiemLoggingJobData } from "../lib/siem-logging/types";import { addExtractJob, ExtractJobData } from "./extract-queue";let loggingQueue: Queue;@@ -11,6 +12,7 @@ let deepResearchQueue: Queue;let generateLlmsTxtQueue: Queue;let billingQueue: Queue;let precrawlQueue: Queue;+let siemLoggingQueue: Queue<SiemLoggingJobData>;let redisConnection: IORedis;export function getRedisConnection(): IORedis {@@ -29,6 +31,7 @@ const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";const deepResearchQueueName = "{deepResearchQueue}";const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";+const siemLoggingQueueName = "{siemLoggingQueue}";export async function addExtractJobToQueue(extractId: string,@@ -107,3 +110,25 @@ export function getPrecrawlQueue() {}return precrawlQueue;}++export function getSiemLoggingQueue(): Queue<SiemLoggingJobData> {+ if (!siemLoggingQueue) {+ siemLoggingQueue = new Queue<SiemLoggingJobData>(siemLoggingQueueName, {+ connection: getRedisConnection(),+ defaultJobOptions: {+ attempts: 8,+ backoff: {+ type: "exponential",+ delay: 1000,+ },+ removeOnComplete: {+ age: 60,+ },+ removeOnFail: {+ age: 24 * 60 * 60,+ },+ },+ });+ }+ return siemLoggingQueue;+}diff --git a/apps/api/src/services/siem-logging/worker.test.ts b/apps/api/src/services/siem-logging/worker.test.tsnew file mode 100644index 000000000..e83e2a57b--- /dev/null+++ b/apps/api/src/services/siem-logging/worker.test.ts@@ -0,0 +1,147 @@+import type { Job } from "bullmq";+import { describe, expect, it, vi } from "vitest";+import {+ SiemDeliveryError,+ type OrgSiemLoggingConfig,+ type ScrapeActivityEvent,+ type SiemLoggingJobData,+} from "../../lib/siem-logging/types";+import { processSiemLoggingJob } from "./worker";++const event: ScrapeActivityEvent = {+ schema_version: 1,+ event_type: "scrape_activity",+ scrape_id: "scrape-id",+ request_id: "request-id",+ endpoint: "scrape",+ team_id: "team-id",+ org_id: "org-id",+ api_key: { id: null, name: null },+ audit_metadata: {},+ started_at: "2026-07-27T00:00:00.000Z",+ completed_at: "2026-07-27T00:00:01.000Z",+ url: "https://example.com",+ domain: "example.com",+ http_method: "GET",+ http_status: 200,+ result: "success",+ error: null,+ origin: "api",+ integration: null,+ zero_data_retention: false,+};++const config: OrgSiemLoggingConfig = {+ orgId: "org-id",+ enabled: true,+ destination: {+ type: "azure_sentinel",+ tenantId: "tenant",+ clientId: "client",+ clientSecret: "secret",+ dceUrl: "https://example.eastus.ingest.monitor.azure.com",+ dcrImmutableId: "dcr-id",+ streamName: "Custom-FirecrawlScrapeActivity",+ },+ createdAt: null,+ updatedAt: null,+};++function createJob() {+ return {+ id: "job-id",+ data: { orgId: "org-id", event },+ extendLock: vi.fn().mockResolvedValue(1),+ moveToCompleted: vi.fn().mockResolvedValue(undefined),+ moveToFailed: vi.fn().mockResolvedValue(undefined),+ discard: vi.fn(),+ attemptsMade: 0,+ opts: {+ attempts: 8,+ backoff: { type: "exponential", delay: 1000 },+ },+ } as unknown as Job<SiemLoggingJobData>;+}++describe("SIEM logging worker", () => {+ it("loads the current secret at delivery time and completes the job", async () => {+ const job = createJob();+ const deliver = vi.fn().mockResolvedValue(undefined);++ await processSiemLoggingJob("token", job, {+ getConfig: vi.fn().mockResolvedValue(config),+ deliver,+ });++ expect(deliver).toHaveBeenCalledWith(config.destination, [event]);+ expect(job.moveToCompleted).toHaveBeenCalledWith(+ { delivered: true },+ "token",+ false,+ );+ expect(job.moveToFailed).not.toHaveBeenCalled();+ });++ it("skips queued events after logging is disabled", async () => {+ const job = createJob();+ const deliver = vi.fn();++ await processSiemLoggingJob("token", job, {+ getConfig: vi.fn().mockResolvedValue({ ...config, enabled: false }),+ deliver,+ });++ expect(deliver).not.toHaveBeenCalled();+ expect(job.moveToCompleted).toHaveBeenCalledWith(+ { skipped: true },+ "token",+ false,+ );+ });++ it("leaves transient delivery failures retryable", async () => {+ const job = createJob();++ await processSiemLoggingJob("token", job, {+ getConfig: vi.fn().mockResolvedValue(config),+ deliver: vi+ .fn()+ .mockRejectedValue(new SiemDeliveryError("rate_limited", "retry")),+ });++ expect(job.discard).not.toHaveBeenCalled();+ expect(job.moveToFailed).toHaveBeenCalledOnce();+ });++ it("does not retry before the provider's Retry-After delay", async () => {+ const job = createJob();++ await processSiemLoggingJob("token", job, {+ getConfig: vi.fn().mockResolvedValue(config),+ deliver: vi+ .fn()+ .mockRejectedValue(+ new SiemDeliveryError("rate_limited", "retry", 429, 15_000),+ ),+ });++ expect(job.opts.backoff).toEqual({ type: "fixed", delay: 15_000 });+ expect(job.moveToFailed).toHaveBeenCalledOnce();+ });++ it("discards permanent destination failures", async () => {+ const job = createJob();++ await processSiemLoggingJob("token", job, {+ getConfig: vi.fn().mockResolvedValue(config),+ deliver: vi+ .fn()+ .mockRejectedValue(+ new SiemDeliveryError("invalid_credentials", "invalid"),+ ),+ });++ expect(job.discard).toHaveBeenCalledOnce();+ expect(job.moveToFailed).toHaveBeenCalledOnce();+ });+});diff --git a/apps/api/src/services/siem-logging/worker.ts b/apps/api/src/services/siem-logging/worker.tsnew file mode 100644index 000000000..c3075779e--- /dev/null+++ b/apps/api/src/services/siem-logging/worker.ts@@ -0,0 +1,121 @@+import type { Job } from "bullmq";+import { config } from "../../config";+import { sendAzureSentinelEvents } from "../../lib/siem-logging/azure-sentinel";+import {+ siemLoggingDeliveryFailuresTotal,+ siemLoggingEventsTotal,+} from "../../lib/siem-logging/metrics";+import { getOrgSiemLoggingConfig } from "../../lib/siem-logging/store";+import {+ SiemDeliveryError,+ type AzureSentinelDestination,+ type OrgSiemLoggingConfig,+ type ScrapeActivityEvent,+ type SiemLoggingJobData,+} from "../../lib/siem-logging/types";+import { logger as _logger } from "../../lib/logger";++interface WorkerDependencies {+ getConfig: (orgId: string) => Promise<OrgSiemLoggingConfig | null>;+ deliver: (+ destination: AzureSentinelDestination,+ events: ScrapeActivityEvent[],+ ) => Promise<void>;+}++const defaultDependencies: WorkerDependencies = {+ getConfig: orgId =>+ getOrgSiemLoggingConfig(orgId, { fresh: true, primary: true }),+ deliver: sendAzureSentinelEvents,+};++const permanentFailureKinds = new Set([+ "invalid_credentials",+ "schema_rejection",+ "payload_too_large",+]);++function asError(error: unknown): Error {+ return error instanceof Error ? error : new Error(String(error));+}++function configuredRetryDelay(job: Job<SiemLoggingJobData>): number {+ const backoff = job.opts.backoff;+ if (typeof backoff === "number") return backoff;+ if (!backoff) return 0;+ const delay = backoff.delay ?? 0;+ if (backoff.type === "exponential") {+ return delay * 2 ** job.attemptsMade;+ }+ return delay;+}++function applyRetryAfter(+ job: Job<SiemLoggingJobData>,+ error: SiemDeliveryError,+): void {+ if (+ error.kind !== "rate_limited" ||+ error.retryAfterMs === undefined ||+ error.retryAfterMs <= 0+ ) {+ return;+ }+ job.opts.backoff = {+ type: "fixed",+ delay: Math.max(configuredRetryDelay(job), error.retryAfterMs),+ };+}++export async function processSiemLoggingJob(+ token: string,+ job: Job<SiemLoggingJobData>,+ deps: WorkerDependencies = defaultDependencies,+): Promise<void> {+ const logger = _logger.child({+ module: "siem-logging-worker",+ jobId: job.id,+ orgId: job.data.orgId,+ });+ const extendLockInterval = setInterval(() => {+ void job.extendLock(token, config.JOB_LOCK_EXTENSION_TIME).catch(error => {+ logger.warn("Failed to extend SIEM logging job lock", { error });+ });+ }, config.JOB_LOCK_EXTEND_INTERVAL);+ extendLockInterval.unref?.();++ try {+ const currentConfig = await deps.getConfig(job.data.orgId);+ if (!currentConfig?.enabled) {+ siemLoggingEventsTotal.inc({ result: "skipped_disabled" });+ await job.moveToCompleted({ skipped: true }, token, false);+ return;+ }++ await deps.deliver(currentConfig.destination, [job.data.event]);+ siemLoggingEventsTotal.inc({ result: "delivered" });+ await job.moveToCompleted({ delivered: true }, token, false);+ } catch (error) {+ const reason =+ error instanceof SiemDeliveryError ? error.kind : "delivery_error";+ siemLoggingEventsTotal.inc({ result: "delivery_failed" });+ siemLoggingDeliveryFailuresTotal.inc({ reason });+ logger.error("SIEM logging event delivery failed", {+ error,+ reason,+ scrapeId: job.data.event.scrape_id,+ });+ if (+ error instanceof SiemDeliveryError &&+ permanentFailureKinds.has(error.kind)+ ) {+ job.discard();+ }+ if (error instanceof SiemDeliveryError) {+ applyRetryAfter(job, error);+ }+ await job.moveToFailed(asError(error), token, false);+ } finally {+ clearInterval(extendLockInterval);+ }+}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex bfabe4ead..fb62e9d90 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -104,6 +104,7 @@ import {warmExchangeCatalog,type ExchangeScrapeMetadata,} from "../../lib/exchange";+import { emitScrapeActivityEvent } from "../../lib/siem-logging";configDotenv();@@ -836,9 +837,25 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {}}+ emitScrapeActivityEvent(job.id, job.data, {+ success: true,+ document: doc,+ threatDecisions: pipeline.threatDecisions,+ startedAt: start,+ completedAt: Date.now(),+ });+logger.info(`🐂 Job done ${job.id}`);return data;} catch (error) {+ emitScrapeActivityEvent(job.id, job.data, {+ success: false,+ error,+ threatDecisions: pipeline?.threatDecisions,+ startedAt: start,+ completedAt: Date.now(),+ });+// Record top-level robots.txt rejections so crawl status can warntry {if (diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.csindex 3ab23a069..3d80e6503 100644--- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs+++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs@@ -17,7 +17,8 @@ public class ModelsTestsOnlyMainContent = true,Timeout = 30000,Mobile = false,- RedactPII = true+ RedactPII = true,+ AuditMetadata = new Dictionary<string, string> { ["requestId"] = "req-123" }};var json = JsonSerializer.Serialize(options, JsonOptions);@@ -28,6 +29,7 @@ public class ModelsTestsAssert.Contains("\"timeout\":30000", json);Assert.Contains("\"mobile\":false", json);Assert.Contains("\"redactPII\":true", json);+ Assert.Contains("\"auditMetadata\":{\"requestId\":\"req-123\"}", json);}[Fact]@@ -70,13 +72,15 @@ public class ModelsTests{Search = "pricing",Limit = 10,- IncludeSubdomains = true+ IncludeSubdomains = true,+ AuditMetadata = new Dictionary<string, string> { ["requestId"] = "req-123" }};var json = JsonSerializer.Serialize(options, JsonOptions);Assert.Contains("\"search\":\"pricing\"", json);Assert.Contains("\"limit\":10", json);Assert.Contains("\"includeSubdomains\":true", json);+ Assert.Contains("\"auditMetadata\":{\"requestId\":\"req-123\"}", json);}[Fact]diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.csindex 00f145294..8708c2b86 100644--- a/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs+++ b/apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs@@ -57,6 +57,7 @@ public class ParseTestsTimeout = 30000,Proxy = "auto",RedactPII = true,+ AuditMetadata = new Dictionary<string, string> { ["requestId"] = "req-123" },};var json = JsonSerializer.Serialize(options, FirecrawlHttpClient.JsonOptions);@@ -65,6 +66,7 @@ public class ParseTestsAssert.Contains("\"timeout\":30000", json);Assert.Contains("\"proxy\":\"auto\"", json);Assert.Contains("\"redactPII\":true", json);+ Assert.Contains("\"auditMetadata\":{\"requestId\":\"req-123\"}", json);}[Fact]diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csprojindex b84ed222a..56c8588de 100644--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj@@ -8,7 +8,7 @@<!-- NuGet package metadata --><PackageId>firecrawl-sdk</PackageId>- <Version>1.9.1</Version>+ <Version>1.10.0</Version><Authors>Firecrawl</Authors><Company>Firecrawl</Company><Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.csindex c7210428e..0fa4f9729 100644--- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs@@ -714,7 +714,7 @@ public class FirecrawlClient// INTERNAL UTILITIES// ================================================================- private const string SdkOrigin = "[email protected]";+ private const string SdkOrigin = "[email protected]";private static Dictionary<string, object> BuildBody(object? options){diff --git a/apps/dot-net-sdk/Firecrawl/Models/MapOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/MapOptions.csindex 912c628b7..8b33ec39c 100644--- a/apps/dot-net-sdk/Firecrawl/Models/MapOptions.cs+++ b/apps/dot-net-sdk/Firecrawl/Models/MapOptions.cs@@ -42,4 +42,8 @@ public class MapOptions[JsonPropertyName("ignoreCache")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]public bool? IgnoreCache { get; set; }++ [JsonPropertyName("auditMetadata")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public Dictionary<string, string>? AuditMetadata { get; set; }}diff --git a/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.csindex a77fb9dc7..b1c92bc6f 100644--- a/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs+++ b/apps/dot-net-sdk/Firecrawl/Models/ParseOptions.cs@@ -83,6 +83,10 @@ public class ParseOptions[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]public bool? RedactPII { get; set; }+ [JsonPropertyName("auditMetadata")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public Dictionary<string, string>? AuditMetadata { get; set; }+/// <summary>/// Validate the options against /v2/parse's supported surface./// </summary>diff --git a/apps/dot-net-sdk/Firecrawl/Models/ScrapeOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/ScrapeOptions.csindex 6dbc2612f..4496de609 100644--- a/apps/dot-net-sdk/Firecrawl/Models/ScrapeOptions.cs+++ b/apps/dot-net-sdk/Firecrawl/Models/ScrapeOptions.cs@@ -79,6 +79,10 @@ public class ScrapeOptions[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]public bool? RedactPII { get; set; }+ [JsonPropertyName("auditMetadata")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public Dictionary<string, string>? AuditMetadata { get; set; }+[JsonPropertyName("integration")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]public string? Integration { get; set; }diff --git a/apps/elixir-sdk/generate.exs b/apps/elixir-sdk/generate.exsindex ab781bdc5..54ba3f095 100644--- a/apps/elixir-sdk/generate.exs+++ b/apps/elixir-sdk/generate.exs@@ -145,7 +145,7 @@ defmodule Firecrawl.Generator do@type response :: {:ok, Req.Response.t()} | {:error, Exception.t() | Firecrawl.Error.t()}@base_url "#{base_url}"- @sdk_origin "[email protected]"+ @sdk_origin "[email protected]"defp client(opts) doapi_key =diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.exindex 760957ce5..e390b6cc9 100644--- a/apps/elixir-sdk/lib/firecrawl.ex+++ b/apps/elixir-sdk/lib/firecrawl.ex@@ -41,7 +41,7 @@ defmodule Firecrawl do@type response :: {:ok, Req.Response.t()} | {:error, Exception.t() | Firecrawl.Error.t()}@base_url "https://api.firecrawl.dev/v2"- @sdk_origin "[email protected]"+ @sdk_origin "[email protected]"defp client(opts) doapi_key =@@ -867,6 +867,7 @@ defmodule Firecrawl do@map_urls_schema NimbleOptions.new!([+ audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],ignore_cache: [type: :boolean, doc: "Bypass the sitemap cache to retrieve fresh URLs. Sitemap data is cached for up to 7 days; use this parameter when your sitemap has been recently updated."],ignore_query_parameters: [type: :boolean, doc: "Do not return URLs with query parameters"],include_subdomains: [type: :boolean, doc: "Include subdomains of the website"],@@ -878,7 +879,7 @@ defmodule Firecrawl dourl: [type: :string, required: true, doc: "The base URL to start crawling from"]])- @map_urls_key_mapping %{ignore_cache: "ignoreCache", ignore_query_parameters: "ignoreQueryParameters", include_subdomains: "includeSubdomains", limit: "limit", location: "location", search: "search", sitemap: "sitemap", timeout: "timeout", url: "url"}+ @map_urls_key_mapping %{audit_metadata: "auditMetadata", ignore_cache: "ignoreCache", ignore_query_parameters: "ignoreQueryParameters", include_subdomains: "includeSubdomains", limit: "limit", location: "location", search: "search", sitemap: "sitemap", timeout: "timeout", url: "url"}@doc """Map multiple URLs based on options@@ -916,6 +917,7 @@ defmodule Firecrawl do@parse_file_schema NimbleOptions.new!([+ audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],block_ads: [type: :boolean, doc: "Enable ad and cookie popup blocking."],exclude_tags: [type: {:list, :string}, doc: "Tags to exclude from the output."],formats: [type: {:list, :any}, doc: "Output formats supported for `/parse` uploads. Browser-rendering formats and change tracking are not supported."],@@ -933,7 +935,7 @@ defmodule Firecrawl dozero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this parse. To enable this feature, please contact [email protected]"]])- @parse_file_key_mapping %{block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", integration: "integration", only_main_content: "onlyMainContent", origin: "origin", parsers: "parsers", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", timeout: "timeout", zero_data_retention: "zeroDataRetention"}+ @parse_file_key_mapping %{audit_metadata: "auditMetadata", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", integration: "integration", only_main_content: "onlyMainContent", origin: "origin", parsers: "parsers", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", timeout: "timeout", zero_data_retention: "zeroDataRetention"}@doc """Upload and parse a file@@ -1017,6 +1019,7 @@ defmodule Firecrawl do@scrape_and_extract_from_url_schema NimbleOptions.new!([url: [type: :string, required: true, doc: "The URL to scrape"],+ audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],actions: [type: {:list, :any}, doc: "Actions to perform on the page before grabbing the content"],block_ads: [type: :boolean, doc: "Enables ad-blocking and cookie popup blocking."],exclude_tags: [type: {:list, :string}, doc: "Tags to exclude from the output."],@@ -1041,7 +1044,7 @@ defmodule Firecrawl dozero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this scrape. To enable this feature, please contact [email protected]"]])- @scrape_and_extract_from_url_key_mapping %{url: "url", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", lockdown: "lockdown", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}+ @scrape_and_extract_from_url_key_mapping %{url: "url", audit_metadata: "auditMetadata", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", lockdown: "lockdown", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}@doc """Scrape a single URL and optionally extract information using an LLM@@ -1079,6 +1082,7 @@ defmodule Firecrawl do@scrape_and_extract_from_urls_schema NimbleOptions.new!([+ audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],ignore_invalid_urls: [type: :boolean, doc: "If invalid URLs are specified in the urls array, they will be ignored. Instead of them failing the entire request, a batch scrape using the remaining valid URLs will be created, and the invalid URLs will be returned in the invalidURLs field of the response."],max_concurrency: [type: :integer, doc: "Maximum number of concurrent scrapes. This parameter allows you to set a concurrency limit for this batch scrape. If not specified, the batch scrape adheres to your team's concurrency limit."],urls: [type: {:list, :string}, required: true],@@ -1107,7 +1111,7 @@ defmodule Firecrawl dozero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this batch scrape. To enable this feature, please contact [email protected]"]])- @scrape_and_extract_from_urls_key_mapping %{ignore_invalid_urls: "ignoreInvalidURLs", max_concurrency: "maxConcurrency", urls: "urls", webhook: "webhook", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", lockdown: "lockdown", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}+ @scrape_and_extract_from_urls_key_mapping %{audit_metadata: "auditMetadata", ignore_invalid_urls: "ignoreInvalidURLs", max_concurrency: "maxConcurrency", urls: "urls", webhook: "webhook", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", redact_pii: "redactPII", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", lockdown: "lockdown", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}@doc """Scrape multiple URLs and optionally extract information using an LLM@@ -1342,6 +1346,7 @@ defmodule Firecrawl do@start_agent_schema NimbleOptions.new!([+ audit_metadata: [type: :map, doc: "Metadata to include with SIEM logging events."],max_credits: [type: {:or, [:integer, :float]}, doc: "Maximum credits to spend on this agent task. Defaults to 2500 if not set. Values above 2,500 are always billed as paid requests."],model: [type: {:or, [{:in, [:"spark-1-mini", :"spark-1-pro"]}, :string]}, doc: "The model to use for the agent task. spark-1-mini (default) is 60% cheaper, spark-1-pro offers higher accuracy for complex tasks"],prompt: [type: :string, required: true, doc: "The prompt describing what data to extract"],@@ -1350,7 +1355,7 @@ defmodule Firecrawl dourls: [type: {:list, :string}, doc: "Optional list of URLs to constrain the agent to"]])- @start_agent_key_mapping %{max_credits: "maxCredits", model: "model", prompt: "prompt", schema: "schema", strict_constrain_to_urls: "strictConstrainToURLs", urls: "urls"}+ @start_agent_key_mapping %{audit_metadata: "auditMetadata", max_credits: "maxCredits", model: "model", prompt: "prompt", schema: "schema", strict_constrain_to_urls: "strictConstrainToURLs", urls: "urls"}@doc """Start an agent task for agentic data extractiondiff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex 163203b4b..5013beee7 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.8.1"+ @version "1.9.0"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/elixir-sdk/test/firecrawl_test.exs b/apps/elixir-sdk/test/firecrawl_test.exsindex be8c4400e..4dcc3f4d3 100644--- a/apps/elixir-sdk/test/firecrawl_test.exs+++ b/apps/elixir-sdk/test/firecrawl_test.exs@@ -317,6 +317,62 @@ defmodule FirecrawlTest doassert body["urls"] == ["https://example.com"]end+ test "request endpoints map audit_metadata to auditMetadata" do+ parent = self()++ adapter = fn request ->+ send(parent, {:request, request})++ resp = Req.Response.new(+ status: 200,+ headers: %{"content-type" => ["application/json"]},+ body: Jason.encode!(%{"success" => true, "id" => "job", "links" => []})+ )++ {request, resp}+ end++ metadata = %{"requestId" => "req-123"}++ requests = [+ fn ->+ Firecrawl.scrape_and_extract_from_url(+ [url: "https://example.com", audit_metadata: metadata],+ api_key: "test-key",+ adapter: adapter+ )+ end,+ fn ->+ Firecrawl.map_urls(+ [url: "https://example.com", audit_metadata: metadata],+ api_key: "test-key",+ adapter: adapter+ )+ end,+ fn ->+ Firecrawl.start_agent(+ [prompt: "find pricing", audit_metadata: metadata],+ api_key: "test-key",+ adapter: adapter+ )+ end+ ]++ Enum.each(requests, fn make_request ->+ assert {:ok, %Req.Response{status: 200}} = make_request.()+ assert_receive {:request, request}++ body =+ cond do+ is_binary(request.body) -> Jason.decode!(request.body)+ is_map(request.body) -> request.body+ true -> request.options[:json]+ end++ assert body["auditMetadata"] == metadata+ end)+ end+test "search maps highlights to highlights" doparent = self()diff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.goindex c7022f57e..9cb22232f 100644--- a/apps/go-sdk/options.go+++ b/apps/go-sdk/options.go@@ -91,6 +91,7 @@ type ScrapeOptions struct {StoreInCache *bool `json:"storeInCache,omitempty"`Lockdown *bool `json:"lockdown,omitempty"`RedactPII *bool `json:"redactPII,omitempty"`+ AuditMetadata map[string]string `json:"auditMetadata,omitempty"`Integration *string `json:"integration,omitempty"`JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`}@@ -150,14 +151,15 @@ type BatchScrapeOptions struct {// MapOptions configures a map (URL discovery) request.type MapOptions struct {- Search *string `json:"search,omitempty"`- Sitemap *string `json:"sitemap,omitempty"`- IncludeSubdomains *bool `json:"includeSubdomains,omitempty"`- IgnoreQueryParameters *bool `json:"ignoreQueryParameters,omitempty"`- Limit *int `json:"limit,omitempty"`- Timeout *int `json:"timeout,omitempty"`- Integration *string `json:"integration,omitempty"`- Location *LocationConfig `json:"location,omitempty"`+ Search *string `json:"search,omitempty"`+ Sitemap *string `json:"sitemap,omitempty"`+ IncludeSubdomains *bool `json:"includeSubdomains,omitempty"`+ IgnoreQueryParameters *bool `json:"ignoreQueryParameters,omitempty"`+ Limit *int `json:"limit,omitempty"`+ Timeout *int `json:"timeout,omitempty"`+ Integration *string `json:"integration,omitempty"`+ Location *LocationConfig `json:"location,omitempty"`+ AuditMetadata map[string]string `json:"auditMetadata,omitempty"`}// SearchOptions configures a search request.@@ -186,6 +188,7 @@ type AgentOptions struct {StrictConstrainToURLs *bool `json:"strictConstrainToURLs,omitempty"`Model *string `json:"model,omitempty"`Webhook *WebhookConfig `json:"webhook,omitempty"`+ AuditMetadata map[string]string `json:"auditMetadata,omitempty"`}// LocationConfig specifies geolocation for requests.diff --git a/apps/go-sdk/options_test.go b/apps/go-sdk/options_test.goindex d41057c2d..4afbb89a7 100644--- a/apps/go-sdk/options_test.go+++ b/apps/go-sdk/options_test.go@@ -87,3 +87,23 @@ func TestSearchOptionsSerializesHighlights(t *testing.T) {t.Fatalf("serialized search options = %s", payload)}}++func TestAuditMetadataSerializesAcrossRequestOptions(t *testing.T) {+ metadata := map[string]string{"requestId": "req-123"}+ for name, options := range map[string]interface{}{+ "scrape": ScrapeOptions{AuditMetadata: metadata},+ "map": MapOptions{AuditMetadata: metadata},+ "agent": AgentOptions{Prompt: "find pricing", AuditMetadata: metadata},+ "parse": ParseOptions{AuditMetadata: metadata},+ } {+ t.Run(name, func(t *testing.T) {+ payload, err := json.Marshal(options)+ if err != nil {+ t.Fatalf("Marshal options: %v", err)+ }+ if !strings.Contains(string(payload), `"auditMetadata":{"requestId":"req-123"}`) {+ t.Fatalf("serialized options = %s", payload)+ }+ })+ }+}diff --git a/apps/go-sdk/parse.go b/apps/go-sdk/parse.goindex 57ca32806..6aacae6d3 100644--- a/apps/go-sdk/parse.go+++ b/apps/go-sdk/parse.go@@ -71,6 +71,7 @@ type ParseOptions struct {Proxy *string `json:"proxy,omitempty"`Integration *string `json:"integration,omitempty"`RedactPII *bool `json:"redactPII,omitempty"`+ AuditMetadata map[string]string `json:"auditMetadata,omitempty"`JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`}diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.goindex 79775e6d8..31f90dbfe 100644--- a/apps/go-sdk/version.go+++ b/apps/go-sdk/version.go@@ -9,4 +9,4 @@ package firecrawl// Bump this when preparing a new release. The publish-go-sdk GitHub workflow// reads this value and creates the corresponding monorepo-prefixed tag on// merge to main.-const Version = "1.8.1"+const Version = "1.9.0"diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex 0310669b8..b30daeb42 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.11.1"+version = "1.12.0"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.javaindex a2b43e53b..c605b3da4 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java@@ -37,7 +37,7 @@ import java.util.concurrent.ForkJoinPool;public class FirecrawlClient {private static final String DEFAULT_API_URL = "https://api.firecrawl.dev";- private static final String SDK_ORIGIN = "[email protected]";+ private static final String SDK_ORIGIN = "[email protected]";private static final long DEFAULT_TIMEOUT_MS = 300_000; // 5 minutesprivate static final int DEFAULT_MAX_RETRIES = 3;private static final double DEFAULT_BACKOFF_FACTOR = 0.5;diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/AgentOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/AgentOptions.javaindex e0e25fd62..c2873512b 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/AgentOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/AgentOptions.java@@ -1,6 +1,9 @@package com.firecrawl.models;import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import java.util.Collections;+import java.util.HashMap;import java.util.List;import java.util.Map;@@ -18,6 +21,7 @@ public class AgentOptions {private Boolean strictConstrainToURLs;private String model;private WebhookConfig webhook;+ private Map<String, String> auditMetadata;private AgentOptions() {}@@ -29,6 +33,8 @@ public class AgentOptions {public Boolean getStrictConstrainToURLs() { return strictConstrainToURLs; }public String getModel() { return model; }public WebhookConfig getWebhook() { return webhook; }+ @JsonProperty("auditMetadata")+ public Map<String, String> getAuditMetadata() { return auditMetadata; }public static Builder builder() { return new Builder(); }@@ -41,6 +47,7 @@ public class AgentOptions {private Boolean strictConstrainToURLs;private String model;private WebhookConfig webhook;+ private Map<String, String> auditMetadata;private Builder() {}@@ -60,6 +67,8 @@ public class AgentOptions {public Builder model(String model) { this.model = model; return this; }/** Webhook configuration. */public Builder webhook(WebhookConfig webhook) { this.webhook = webhook; return this; }+ /** Metadata to include with SIEM logging events. */+ public Builder auditMetadata(Map<String, String> auditMetadata) { this.auditMetadata = auditMetadata; return this; }public AgentOptions build() {if (prompt == null || prompt.isEmpty()) {@@ -74,6 +83,9 @@ public class AgentOptions {o.strictConstrainToURLs = this.strictConstrainToURLs;o.model = this.model;o.webhook = this.webhook;+ o.auditMetadata = this.auditMetadata != null+ ? Collections.unmodifiableMap(new HashMap<>(this.auditMetadata))+ : null;return o;}}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MapOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MapOptions.javaindex 06940939d..b4c14ce47 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/MapOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MapOptions.java@@ -1,6 +1,10 @@package com.firecrawl.models;import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import java.util.Collections;+import java.util.HashMap;+import java.util.Map;/*** Options for mapping (discovering URLs on) a website.@@ -16,6 +20,7 @@ public class MapOptions {private Integer timeout;private String integration;private LocationConfig location;+ private Map<String, String> auditMetadata;private MapOptions() {}@@ -28,6 +33,8 @@ public class MapOptions {public Integer getTimeout() { return timeout; }public String getIntegration() { return integration; }public LocationConfig getLocation() { return location; }+ @JsonProperty("auditMetadata")+ public Map<String, String> getAuditMetadata() { return auditMetadata; }public static Builder builder() { return new Builder(); }@@ -40,6 +47,7 @@ public class MapOptions {private Integer timeout;private String integration;private LocationConfig location;+ private Map<String, String> auditMetadata;private Builder() {}@@ -59,6 +67,8 @@ public class MapOptions {public Builder integration(String integration) { this.integration = integration; return this; }/** Geolocation configuration. */public Builder location(LocationConfig location) { this.location = location; return this; }+ /** Metadata to include with SIEM logging events. */+ public Builder auditMetadata(Map<String, String> auditMetadata) { this.auditMetadata = auditMetadata; return this; }public MapOptions build() {MapOptions o = new MapOptions();@@ -70,6 +80,9 @@ public class MapOptions {o.timeout = this.timeout;o.integration = this.integration;o.location = this.location;+ o.auditMetadata = this.auditMetadata != null+ ? Collections.unmodifiableMap(new HashMap<>(this.auditMetadata))+ : null;return o;}}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.javaindex 079bc29d4..c8094b264 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java@@ -31,6 +31,7 @@ public class ParseOptions {private String integration;@JsonProperty("redactPII")private Boolean redactPII;+ private Map<String, String> auditMetadata;private ParseOptions() {}@@ -48,6 +49,8 @@ public class ParseOptions {public String getIntegration() { return integration; }@JsonProperty("redactPII")public Boolean getRedactPII() { return redactPII; }+ @JsonProperty("auditMetadata")+ public Map<String, String> getAuditMetadata() { return auditMetadata; }public static Builder builder() { return new Builder(); }@@ -66,6 +69,7 @@ public class ParseOptions {b.proxy = this.proxy;b.integration = this.integration;b.redactPII = this.redactPII;+ b.auditMetadata = this.auditMetadata != null ? new HashMap<>(this.auditMetadata) : null;return b;}@@ -113,6 +117,7 @@ public class ParseOptions {private String proxy;private String integration;private Boolean redactPII;+ private Map<String, String> auditMetadata;private Builder() {}@@ -129,6 +134,7 @@ public class ParseOptions {public Builder proxy(String proxy) { this.proxy = proxy; return this; }public Builder integration(String integration) { this.integration = integration; return this; }public Builder redactPII(Boolean redactPII) { this.redactPII = redactPII; return this; }+ public Builder auditMetadata(Map<String, String> auditMetadata) { this.auditMetadata = auditMetadata; return this; }public ParseOptions build() {if (timeout != null && timeout <= 0) {@@ -162,6 +168,7 @@ public class ParseOptions {o.proxy = this.proxy;o.integration = this.integration;o.redactPII = this.redactPII;+ o.auditMetadata = this.auditMetadata != null ? Collections.unmodifiableMap(new HashMap<>(this.auditMetadata)) : null;return o;}}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.javaindex a1889f50e..e36050148 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java@@ -35,6 +35,7 @@ public class ScrapeOptions {private Boolean lockdown;@JsonProperty("redactPII")private Boolean redactPII;+ private Map<String, String> auditMetadata;private String integration;private ScrapeOptions() {}@@ -59,6 +60,8 @@ public class ScrapeOptions {public Boolean getLockdown() { return lockdown; }@JsonProperty("redactPII")public Boolean getRedactPII() { return redactPII; }+ @JsonProperty("auditMetadata")+ public Map<String, String> getAuditMetadata() { return auditMetadata; }public String getIntegration() { return integration; }public static Builder builder() { return new Builder(); }@@ -84,6 +87,7 @@ public class ScrapeOptions {b.storeInCache = this.storeInCache;b.lockdown = this.lockdown;b.redactPII = this.redactPII;+ b.auditMetadata = this.auditMetadata != null ? new HashMap<>(this.auditMetadata) : null;b.integration = this.integration;return b;}@@ -108,6 +112,7 @@ public class ScrapeOptions {private Boolean storeInCache;private Boolean lockdown;private Boolean redactPII;+ private Map<String, String> auditMetadata;private String integration;private Builder() {}@@ -173,6 +178,9 @@ public class ScrapeOptions {/** Redact personally identifiable information from returned content. */public Builder redactPII(Boolean redactPII) { this.redactPII = redactPII; return this; }+ /** Metadata to include with SIEM logging events. */+ public Builder auditMetadata(Map<String, String> auditMetadata) { this.auditMetadata = auditMetadata; return this; }+/** Integration identifier. */public Builder integration(String integration) { this.integration = integration; return this; }@@ -197,6 +205,7 @@ public class ScrapeOptions {o.storeInCache = this.storeInCache;o.lockdown = this.lockdown;o.redactPII = this.redactPII;+ o.auditMetadata = this.auditMetadata != null ? Collections.unmodifiableMap(new HashMap<>(this.auditMetadata)) : null;o.integration = this.integration;return o;}diff --git a/apps/java-sdk/src/test/java/com/firecrawl/AuditMetadataTest.java b/apps/java-sdk/src/test/java/com/firecrawl/AuditMetadataTest.javanew file mode 100644index 000000000..354ed6b61--- /dev/null+++ b/apps/java-sdk/src/test/java/com/firecrawl/AuditMetadataTest.java@@ -0,0 +1,54 @@+package com.firecrawl;++import com.fasterxml.jackson.databind.ObjectMapper;+import com.firecrawl.models.AgentOptions;+import com.firecrawl.models.MapOptions;+import com.firecrawl.models.ParseOptions;+import com.firecrawl.models.ScrapeOptions;+import org.junit.jupiter.api.Test;++import java.util.HashMap;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;++class AuditMetadataTest {++ private final ObjectMapper mapper = new ObjectMapper();++ @Test+ void serializesAuditMetadataAcrossRequestOptions() {+ Map<String, String> metadata = Map.of("requestId", "req-123");++ assertAuditMetadata(ScrapeOptions.builder().auditMetadata(metadata).build());+ assertAuditMetadata(MapOptions.builder().auditMetadata(metadata).build());+ assertAuditMetadata(AgentOptions.builder().prompt("find pricing").auditMetadata(metadata).build());+ assertAuditMetadata(ParseOptions.builder().auditMetadata(metadata).build());+ }++ @Test+ void mapAndAgentOptionsDefensivelyCopyAuditMetadata() {+ Map<String, String> metadata = new HashMap<>();+ metadata.put("requestId", "req-123");++ MapOptions map = MapOptions.builder().auditMetadata(metadata).build();+ AgentOptions agent = AgentOptions.builder()+ .prompt("find pricing")+ .auditMetadata(metadata)+ .build();+ metadata.put("requestId", "mutated");++ assertEquals("req-123", map.getAuditMetadata().get("requestId"));+ assertEquals("req-123", agent.getAuditMetadata().get("requestId"));+ assertThrows(UnsupportedOperationException.class,+ () -> map.getAuditMetadata().put("requestId", "mutated"));+ assertThrows(UnsupportedOperationException.class,+ () -> agent.getAuditMetadata().put("requestId", "mutated"));+ }++ private void assertAuditMetadata(Object options) {+ Map<?, ?> body = mapper.convertValue(options, Map.class);+ assertEquals(Map.of("requestId", "req-123"), body.get("auditMetadata"));+ }+}diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 82128f2ec..c3fbe5b6c 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.30.1",+ "version": "4.31.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/audit-metadata.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/audit-metadata.unit.test.tsnew file mode 100644index 000000000..620fde00a--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/audit-metadata.unit.test.ts@@ -0,0 +1,79 @@+import { describe, test, expect, jest } from "@jest/globals";+import { scrape } from "../../../v2/methods/scrape";+import { startBatchScrape } from "../../../v2/methods/batch";+import { startCrawl } from "../../../v2/methods/crawl";+import { search } from "../../../v2/methods/search";+import { map } from "../../../v2/methods/map";+import { startExtract } from "../../../v2/methods/extract";+import { startAgent } from "../../../v2/methods/agent";++const auditMetadata = { requestId: "req-123" };++function makeHttp(data: Record<string, unknown>) {+ const post = jest.fn(async () => ({ status: 200, data }));+ return {+ post,+ prepareHeaders: jest.fn(() => undefined),+ } as any;+}++describe("v2 auditMetadata request serialization", () => {+ test("scrape and batch scrape send metadata at the top level", async () => {+ const scrapeHttp = makeHttp({ success: true, data: {} });+ await scrape(scrapeHttp, "https://example.com", { auditMetadata });+ expect(scrapeHttp.post.mock.calls[0][1]).toEqual(+ expect.objectContaining({ auditMetadata }),+ );++ const batchHttp = makeHttp({ success: true, id: "job", url: "u" });+ await startBatchScrape(batchHttp, ["https://example.com"], {+ options: { auditMetadata },+ });+ expect(batchHttp.post.mock.calls[0][1]).toEqual(+ expect.objectContaining({ auditMetadata }),+ );+ });++ test("crawl, search, and extract send metadata under scrapeOptions", async () => {+ const crawlHttp = makeHttp({ success: true, id: "job", url: "u" });+ await startCrawl(crawlHttp, {+ url: "https://example.com",+ scrapeOptions: { auditMetadata },+ });+ expect(crawlHttp.post.mock.calls[0][1]).toEqual(+ expect.objectContaining({ scrapeOptions: expect.objectContaining({ auditMetadata }) }),+ );++ const searchHttp = makeHttp({ success: true, data: {} });+ await search(searchHttp, {+ query: "example",+ scrapeOptions: { auditMetadata },+ });+ expect(searchHttp.post.mock.calls[0][1]).toEqual(+ expect.objectContaining({ scrapeOptions: expect.objectContaining({ auditMetadata }) }),+ );++ const extractHttp = makeHttp({ success: true, id: "job" });+ await startExtract(extractHttp, {+ urls: ["https://example.com"],+ scrapeOptions: { auditMetadata },+ });+ expect(extractHttp.post.mock.calls[0][1]).toEqual(+ expect.objectContaining({ scrapeOptions: expect.objectContaining({ auditMetadata }) }),+ );+ });++ test("map and agent send metadata at the top level", async () => {+ const mapHttp = makeHttp({ success: true, links: [] });+ await map(mapHttp, "https://example.com", { auditMetadata });+ expect(mapHttp.post.mock.calls[0][1]).toEqual(+ expect.objectContaining({ auditMetadata }),+ );++ const agentHttp = makeHttp({ success: true, id: "job" });+ await startAgent(agentHttp, { prompt: "find pricing", auditMetadata });+ expect(agentHttp.post.mock.calls[0][1]).toEqual(+ expect.objectContaining({ auditMetadata }),+ );+ });+});diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex b293d10e4..665130d9b 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -135,6 +135,7 @@ export interface CrawlScrapeOptions {storeInCache?: boolean;maxAge?: number;parsePDF?: boolean;+ auditMetadata?: Record<string, string>;}export type Action = {@@ -310,6 +311,7 @@ export interface MapParams {timeout?: number;useIndex?: boolean;location?: LocationConfig;+ auditMetadata?: Record<string, string>;}/**diff --git a/apps/js-sdk/firecrawl/src/v2/methods/agent.ts b/apps/js-sdk/firecrawl/src/v2/methods/agent.tsindex 08ae78563..16cdcb388 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/agent.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/agent.ts@@ -15,6 +15,7 @@ function prepareAgentPayload(args: {model?: "spark-1-pro" | "spark-1-mini";webhook?: string | AgentWebhookConfig;threatProtection?: ThreatProtectionOptions;+ auditMetadata?: Record<string, string>;}): Record<string, unknown> {const body: Record<string, unknown> = {};if (args.urls) body.urls = args.urls;@@ -30,6 +31,8 @@ function prepareAgentPayload(args: {if (args.webhook != null) body.webhook = args.webhook;if (args.threatProtection != null)body.threatProtection = args.threatProtection;+ if (args.auditMetadata != null)+ body.auditMetadata = args.auditMetadata;return body;}diff --git a/apps/js-sdk/firecrawl/src/v2/methods/map.ts b/apps/js-sdk/firecrawl/src/v2/methods/map.tsindex eab0b6d06..2bc493e2d 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/map.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/map.ts@@ -26,6 +26,8 @@ function prepareMapPayload(if (options.location != null) payload.location = options.location;if (options.threatProtection != null)payload.threatProtection = options.threatProtection;+ if (options.auditMetadata != null)+ payload.auditMetadata = options.auditMetadata;}return payload;}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex da2ea3208..24583af32 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -209,6 +209,7 @@ export interface ScrapeOptions {lockdown?: boolean;redactPII?: boolean | RedactPIIOptions;threatProtection?: ThreatProtectionOptions;+ auditMetadata?: Record<string, string>;profile?: {name: string;saveChanges?: boolean;@@ -796,6 +797,7 @@ export interface MapOptions {origin?: string;location?: LocationConfig;threatProtection?: ThreatProtectionOptions;+ auditMetadata?: Record<string, string>;}export type FeedbackRating = "good" | "partial" | "bad";diff --git a/apps/php-sdk/src/Models/AgentOptions.php b/apps/php-sdk/src/Models/AgentOptions.phpindex 49061561a..381268549 100644--- a/apps/php-sdk/src/Models/AgentOptions.php+++ b/apps/php-sdk/src/Models/AgentOptions.php@@ -9,6 +9,7 @@ final class AgentOptions/*** @param list<string>|null $urls* @param array<string, mixed>|null $schema+ * @param array<string, string>|null $auditMetadata*/private function __construct(private readonly ?array $urls = null,@@ -19,11 +20,14 @@ final class AgentOptionsprivate readonly ?bool $strictConstrainToURLs = null,private readonly ?string $model = null,private readonly ?WebhookConfig $webhook = null,+ /** @var array<string, string>|null */+ private readonly ?array $auditMetadata = null,) {}/*** @param list<string>|null $urls* @param array<string, mixed>|null $schema+ * @param array<string, string>|null $auditMetadata*/public static function with(?array $urls = null,@@ -34,10 +38,11 @@ final class AgentOptions?bool $strictConstrainToURLs = null,?string $model = null,?WebhookConfig $webhook = null,+ ?array $auditMetadata = null,): self {return new self($urls, $prompt, $schema, $integration,- $maxCredits, $strictConstrainToURLs, $model, $webhook,+ $maxCredits, $strictConstrainToURLs, $model, $webhook, $auditMetadata,);}@@ -53,6 +58,7 @@ final class AgentOptions'strictConstrainToURLs' => $this->strictConstrainToURLs,'model' => $this->model,'webhook' => $this->webhook?->toArray(),+ 'auditMetadata' => $this->auditMetadata,];return array_filter($fields, fn (mixed $v): bool => $v !== null);diff --git a/apps/php-sdk/src/Models/MapOptions.php b/apps/php-sdk/src/Models/MapOptions.phpindex 6eebd94ef..397847898 100644--- a/apps/php-sdk/src/Models/MapOptions.php+++ b/apps/php-sdk/src/Models/MapOptions.php@@ -6,6 +6,7 @@ namespace Firecrawl\Models;final class MapOptions{+ /** @param array<string, string>|null $auditMetadata */private function __construct(private readonly ?string $search = null,private readonly ?string $sitemap = null,@@ -15,8 +16,11 @@ final class MapOptionsprivate readonly ?int $timeout = null,private readonly ?string $integration = null,private readonly ?LocationConfig $location = null,+ /** @var array<string, string>|null */+ private readonly ?array $auditMetadata = null,) {}+ /** @param array<string, string>|null $auditMetadata */public static function with(?string $search = null,?string $sitemap = null,@@ -26,10 +30,11 @@ final class MapOptions?int $timeout = null,?string $integration = null,?LocationConfig $location = null,+ ?array $auditMetadata = null,): self {return new self($search, $sitemap, $includeSubdomains, $ignoreQueryParameters,- $limit, $timeout, $integration, $location,+ $limit, $timeout, $integration, $location, $auditMetadata,);}@@ -45,6 +50,7 @@ final class MapOptions'timeout' => $this->timeout,'integration' => $this->integration,'location' => $this->location?->toArray(),+ 'auditMetadata' => $this->auditMetadata,];return array_filter($fields, fn (mixed $v): bool => $v !== null);diff --git a/apps/php-sdk/src/Models/ParseOptions.php b/apps/php-sdk/src/Models/ParseOptions.phpindex 08db06878..c8a3f0fc8 100644--- a/apps/php-sdk/src/Models/ParseOptions.php+++ b/apps/php-sdk/src/Models/ParseOptions.php@@ -32,6 +32,7 @@ final class ParseOptions* @param list<string>|null $includeTags* @param list<string>|null $excludeTags* @param list<mixed>|null $parsers+ * @param array<string, string>|null $auditMetadata*/private function __construct(private readonly ?array $formats = null,@@ -47,6 +48,8 @@ final class ParseOptionsprivate readonly ?string $proxy = null,private readonly ?string $integration = null,private readonly ?bool $redactPII = null,+ /** @var array<string, string>|null */+ private readonly ?array $auditMetadata = null,) {}/**@@ -55,6 +58,7 @@ final class ParseOptions* @param list<string>|null $includeTags* @param list<string>|null $excludeTags* @param list<mixed>|null $parsers+ * @param array<string, string>|null $auditMetadata*/public static function with(?array $formats = null,@@ -70,6 +74,7 @@ final class ParseOptions?string $proxy = null,?string $integration = null,?bool $redactPII = null,+ ?array $auditMetadata = null,): self {if ($timeout !== null && $timeout <= 0) {throw new FirecrawlException('timeout must be positive');@@ -102,6 +107,7 @@ final class ParseOptions$proxy,$integration,$redactPII,+ $auditMetadata,);}@@ -136,6 +142,7 @@ final class ParseOptions'proxy' => $this->proxy,'integration' => $this->integration,'redactPII' => $this->redactPII,+ 'auditMetadata' => $this->auditMetadata,];foreach ($fields as $key => $value) {@@ -234,4 +241,10 @@ final class ParseOptions{return $this->integration;}++ /** @return array<string, string>|null */+ public function getAuditMetadata(): ?array+ {+ return $this->auditMetadata;+ }}diff --git a/apps/php-sdk/src/Models/ScrapeOptions.php b/apps/php-sdk/src/Models/ScrapeOptions.phpindex 94c7f0d09..d264cd162 100644--- a/apps/php-sdk/src/Models/ScrapeOptions.php+++ b/apps/php-sdk/src/Models/ScrapeOptions.php@@ -13,6 +13,7 @@ final class ScrapeOptions* @param list<string>|null $excludeTags* @param list<mixed>|null $parsers* @param list<array<string, mixed>>|null $actions+ * @param array<string, string>|null $auditMetadata*/private function __construct(private readonly ?array $formats = null,@@ -39,6 +40,8 @@ final class ScrapeOptionsprivate readonly ?array $profile = null,private readonly ?bool $changeTracking = null,private readonly ?bool $redactPII = null,+ /** @var array<string, string>|null */+ private readonly ?array $auditMetadata = null,) {}/**@@ -49,6 +52,7 @@ final class ScrapeOptions* @param list<mixed>|null $parsers* @param list<array<string, mixed>>|null $actions* @param array<string, string>|null $profile+ * @param array<string, string>|null $auditMetadata*/public static function with(?array $formats = null,@@ -74,13 +78,14 @@ final class ScrapeOptions?array $profile = null,?bool $changeTracking = null,?bool $redactPII = null,+ ?array $auditMetadata = null,): self {return new self($formats, $headers, $includeTags, $excludeTags, $onlyMainContent,$timeout, $waitFor, $mobile, $parsers, $actions, $location,$skipTlsVerification, $removeBase64Images, $blockAds, $proxy,$maxAge, $minAge, $storeInCache, $lockdown, $integration, $profile,- $changeTracking, $redactPII,+ $changeTracking, $redactPII, $auditMetadata,);}@@ -126,6 +131,7 @@ final class ScrapeOptions'profile' => $this->profile,'changeTracking' => $this->changeTracking,'redactPII' => $this->redactPII,+ 'auditMetadata' => $this->auditMetadata,];foreach ($fields as $key => $value) {@@ -258,4 +264,10 @@ final class ScrapeOptions{return $this->changeTracking;}++ /** @return array<string, string>|null */+ public function getAuditMetadata(): ?array+ {+ return $this->auditMetadata;+ }}diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.phpindex 4e52a3fcf..d8f95f0bc 100644--- a/apps/php-sdk/src/Version.php+++ b/apps/php-sdk/src/Version.php@@ -6,5 +6,5 @@ namespace Firecrawl;final class Version{- public const SDK_VERSION = '1.9.1';+ public const SDK_VERSION = '1.10.0';}diff --git a/apps/php-sdk/tests/Unit/ModelsTest.php b/apps/php-sdk/tests/Unit/ModelsTest.phpindex e55ac7a9b..c78dbc863 100644--- a/apps/php-sdk/tests/Unit/ModelsTest.php+++ b/apps/php-sdk/tests/Unit/ModelsTest.php@@ -10,6 +10,9 @@ use Firecrawl\Models\MapData;use Firecrawl\Models\BatchScrapeJob;use Firecrawl\Models\CrawlJob;use Firecrawl\Models\HighlightsFormat;+use Firecrawl\Models\AgentOptions;+use Firecrawl\Models\MapOptions;+use Firecrawl\Models\ParseOptions;use Firecrawl\Models\QueryFormat;use Firecrawl\Models\QuestionFormat;use Firecrawl\Models\ScrapeOptions;@@ -340,6 +343,21 @@ it('serializes redactPII in ScrapeOptions', function (): void {expect(array_key_exists('formats', $options->toArray()))->toBeFalse();});+it('serializes audit metadata across request options', function (): void {+ $metadata = ['requestId' => 'req-123'];++ $scrape = ScrapeOptions::with(auditMetadata: $metadata);+ expect($scrape->toArray())->toMatchArray(['auditMetadata' => $metadata]);+ expect($scrape->getAuditMetadata())->toBe($metadata);+ expect(MapOptions::with(auditMetadata: $metadata)->toArray())+ ->toMatchArray(['auditMetadata' => $metadata]);+ expect(AgentOptions::with(prompt: 'find pricing', auditMetadata: $metadata)->toArray())+ ->toMatchArray(['auditMetadata' => $metadata]);+ $parse = ParseOptions::with(auditMetadata: $metadata);+ expect($parse->toArray())->toMatchArray(['auditMetadata' => $metadata]);+ expect($parse->getAuditMetadata())->toBe($metadata);+});+it('serializes query format mode in ScrapeOptions', function (): void {$options = ScrapeOptions::with(formats: [QueryFormat::with('What is Firecrawl?', QueryFormat::MODE_DIRECT_QUOTE)],diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex fc8b578bb..5ac05814e 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.32.1"+__version__ = "4.33.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_audit_metadata_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_audit_metadata_request_preparation.pynew file mode 100644index 000000000..685591abe--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_audit_metadata_request_preparation.py@@ -0,0 +1,61 @@+import json++from firecrawl.v2.methods.agent import _prepare_agent_request+from firecrawl.v2.methods.aio.batch import _prepare as prepare_async_batch_request+from firecrawl.v2.methods.batch import prepare_batch_scrape_request+from firecrawl.v2.methods.map import _prepare_map_request+from firecrawl.v2.methods.parse import _prepare_parse_request+from firecrawl.v2.types import MapOptions, ParseOptions, ScrapeOptions+from firecrawl.v2.utils.validation import prepare_scrape_options+++AUDIT_METADATA = {"requestId": "req-123", "actor": "sdk-test"}+++def test_scrape_and_batch_audit_metadata_request_preparation():+ options = ScrapeOptions(audit_metadata=AUDIT_METADATA)++ assert prepare_scrape_options(options)["auditMetadata"] == AUDIT_METADATA+ assert (+ prepare_batch_scrape_request(+ ["https://example.com"],+ audit_metadata=AUDIT_METADATA,+ )["auditMetadata"]+ == AUDIT_METADATA+ )+ assert (+ prepare_async_batch_request(+ ["https://example.com"],+ audit_metadata=AUDIT_METADATA,+ )["auditMetadata"]+ == AUDIT_METADATA+ )+++def test_map_and_agent_audit_metadata_request_preparation():+ assert (+ _prepare_map_request(+ "https://example.com",+ MapOptions(audit_metadata=AUDIT_METADATA),+ )["auditMetadata"]+ == AUDIT_METADATA+ )+ assert (+ _prepare_agent_request(+ None,+ prompt="Extract the page",+ audit_metadata=AUDIT_METADATA,+ )["auditMetadata"]+ == AUDIT_METADATA+ )+++def test_parse_audit_metadata_request_preparation():+ fields, _ = _prepare_parse_request(+ b"<html></html>",+ ParseOptions(audit_metadata=AUDIT_METADATA),+ filename="upload.html",+ content_type="text/html",+ )++ assert json.loads(fields["options"])["auditMetadata"] == AUDIT_METADATAdiff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex a8236e573..b0a21caba 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -78,7 +78,7 @@ _SCRAPE_OPTION_KEYS = frozenset({"parsers", "actions", "location", "skip_tls_verification","remove_base64_images", "fast_mode", "use_mock", "block_ads","proxy", "max_age", "store_in_cache", "lockdown", "threat_protection",- "profile",+ "profile", "audit_metadata",})@@ -161,6 +161,7 @@ class FirecrawlClient:lockdown: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,profile: Optional[Dict[str, Any]] = None,+ audit_metadata: Optional[Dict[str, str]] = None,integration: Optional[str] = None,) -> Document:"""@@ -189,6 +190,7 @@ class FirecrawlClient:lockdown: Serve only previously cached results; never make outbound requests. Returns 404 SCRAPE_LOCKDOWN_CACHE_MISS on cache miss.threat_protection: Enterprise per-request override of the team's threat protection policyprofile: Browser profile for persistent state (e.g. {"name": "my-profile", "saveChanges": True})+ audit_metadata: Metadata to include in SIEM logging eventsReturns:Document"""@@ -216,9 +218,10 @@ class FirecrawlClient:lockdown=lockdown,threat_protection=threat_protection,profile=profile,+ audit_metadata=audit_metadata,integration=integration,).items() if v is not None}- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection, profile, integration]) else None+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection, profile, audit_metadata, integration]) else Nonereturn scrape_module.scrape(self.http_client, url, options)def search_papers(self, query: str, **kwargs):@@ -483,6 +486,7 @@ class FirecrawlClient:lockdown: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,profile: Optional[Dict[str, Any]] = None,+ audit_metadata: Optional[Dict[str, str]] = None,regex_on_full_url: bool = False,deduplicate_similar_urls: bool = True,zero_data_retention: bool = False,@@ -533,6 +537,7 @@ class FirecrawlClient:lockdown: Serve only cached results (convenience kwarg)threat_protection: Enterprise threat protection override (convenience kwarg)profile: Browser profile (convenience kwarg)+ audit_metadata: Metadata to include in SIEM logging eventsregex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathnamededuplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)zero_data_retention: Whether to delete data after 24 hours@@ -558,6 +563,7 @@ class FirecrawlClient:use_mock=use_mock, block_ads=block_ads, proxy=proxy,max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,threat_protection=threat_protection, profile=profile,+ audit_metadata=audit_metadata,).items() if v is not None}if scrape_kwargs:scrape_options = ScrapeOptions(**scrape_kwargs)@@ -642,6 +648,7 @@ class FirecrawlClient:lockdown: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,profile: Optional[Dict[str, Any]] = None,+ audit_metadata: Optional[Dict[str, str]] = None,regex_on_full_url: bool = False,deduplicate_similar_urls: bool = True,zero_data_retention: bool = False,@@ -690,6 +697,7 @@ class FirecrawlClient:lockdown: Serve only cached results (convenience kwarg)threat_protection: Enterprise threat protection override (convenience kwarg)profile: Browser profile (convenience kwarg)+ audit_metadata: Metadata to include in SIEM logging eventsregex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathnamededuplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)zero_data_retention: Whether to delete data after 24 hours@@ -712,6 +720,7 @@ class FirecrawlClient:use_mock=use_mock, block_ads=block_ads, proxy=proxy,max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,threat_protection=threat_protection, profile=profile,+ audit_metadata=audit_metadata,).items() if v is not None}if scrape_kwargs:scrape_options = ScrapeOptions(**scrape_kwargs)@@ -843,6 +852,7 @@ class FirecrawlClient:integration: Optional[str] = None,location: Optional[Location] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,) -> MapData:"""Map a URL and return discovered links.@@ -856,6 +866,7 @@ class FirecrawlClient:timeout: Request timeout in millisecondsthreat_protection: Enterprise per-request override of the team'sthreat protection policy+ audit_metadata: Metadata to include in SIEM logging eventsReturns:MapData containing the discovered links@@ -869,8 +880,9 @@ class FirecrawlClient:timeout=timeout,integration=integration,location=location,- threat_protection=threat_protection- ) if any(v is not None for v in [search, include_subdomains, ignore_query_parameters, limit, sitemap, timeout, integration, location, threat_protection]) else None+ threat_protection=threat_protection,+ audit_metadata=audit_metadata,+ ) if any(v is not None for v in [search, include_subdomains, ignore_query_parameters, limit, sitemap, timeout, integration, location, threat_protection, audit_metadata]) else Nonereturn map_module.map(self.http_client, url, options)@@ -1156,6 +1168,7 @@ class FirecrawlClient:store_in_cache: Optional[bool] = None,lockdown: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,webhook: Optional[Union[str, WebhookConfig]] = None,append_to_id: Optional[str] = None,ignore_invalid_urls: Optional[bool] = None,@@ -1189,6 +1202,7 @@ class FirecrawlClient:store_in_cache: Whether to store results in cachelockdown: Serve only previously cached results; never make outbound requests.threat_protection: Enterprise per-request override of the team's threat protection policy+ audit_metadata: Metadata to include in SIEM logging eventswebhook: Webhook configurationappend_to_id: Append to an existing batch jobignore_invalid_urls: Skip invalid URLs without failing@@ -1236,6 +1250,7 @@ class FirecrawlClient:max_concurrency=max_concurrency,zero_data_retention=zero_data_retention,integration=integration,+ audit_metadata=audit_metadata,idempotency_key=idempotency_key,)@@ -1330,6 +1345,7 @@ class FirecrawlClient:model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,):"""Start an agent job (non-blocking).@@ -1343,6 +1359,7 @@ class FirecrawlClient:webhook: Webhook URL or configuration for notificationsthreat_protection: Enterprise per-request override of the team'sthreat protection policy+ audit_metadata: Metadata to include in SIEM logging eventsReturns:Response payload with job id/status (poll with get_agent_status)"""@@ -1357,6 +1374,7 @@ class FirecrawlClient:model=model,webhook=webhook,threat_protection=threat_protection,+ audit_metadata=audit_metadata,)def agent(@@ -1373,6 +1391,7 @@ class FirecrawlClient:model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,):"""Run an agent and wait until completion.@@ -1388,6 +1407,7 @@ class FirecrawlClient:webhook: Webhook URL or configuration for notificationsthreat_protection: Enterprise per-request override of the team'sthreat protection policy+ audit_metadata: Metadata to include in SIEM logging eventsReturns:Final agent response when completed"""@@ -1404,6 +1424,7 @@ class FirecrawlClient:model=model,webhook=webhook,threat_protection=threat_protection,+ audit_metadata=audit_metadata,)def get_agent_status(self, job_id: str):@@ -1583,6 +1604,7 @@ class FirecrawlClient:store_in_cache: Optional[bool] = None,lockdown: Optional[bool] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,webhook: Optional[Union[str, WebhookConfig]] = None,append_to_id: Optional[str] = None,ignore_invalid_urls: Optional[bool] = None,@@ -1632,6 +1654,7 @@ class FirecrawlClient:max_concurrency=max_concurrency,zero_data_retention=zero_data_retention,integration=integration,+ audit_metadata=audit_metadata,idempotency_key=idempotency_key,poll_interval=poll_interval,timeout=wait_timeout,diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 00fa01757..9ea7eaa4a 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -399,6 +399,7 @@ class AsyncFirecrawlClient:timeout: Optional[int] = None,integration: Optional[str] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,) -> MapData:options = MapOptions(search=search,@@ -408,7 +409,8 @@ class AsyncFirecrawlClient:timeout=timeout,integration=integration,threat_protection=threat_protection,- ) if any(v is not None for v in [search, include_subdomains, limit, sitemap, integration, timeout, threat_protection]) else None+ audit_metadata=audit_metadata,+ ) if any(v is not None for v in [search, include_subdomains, limit, sitemap, integration, timeout, threat_protection, audit_metadata]) else Nonereturn await async_map.map(self.async_http_client, url, options)async def create_monitor(@@ -680,6 +682,7 @@ class AsyncFirecrawlClient:model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,):return await async_agent.agent(self.async_http_client,@@ -694,6 +697,7 @@ class AsyncFirecrawlClient:model=model,webhook=webhook,threat_protection=threat_protection,+ audit_metadata=audit_metadata,)async def get_agent_status(self, job_id: str):@@ -711,6 +715,7 @@ class AsyncFirecrawlClient:model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,):return await async_agent.start_agent(self.async_http_client,@@ -723,6 +728,7 @@ class AsyncFirecrawlClient:model=model,webhook=webhook,threat_protection=threat_protection,+ audit_metadata=audit_metadata,)async def cancel_agent(self, job_id: str) -> bool:diff --git a/apps/python-sdk/firecrawl/v2/methods/agent.py b/apps/python-sdk/firecrawl/v2/methods/agent.pyindex e6bc6c412..6823ce3f6 100644--- a/apps/python-sdk/firecrawl/v2/methods/agent.py+++ b/apps/python-sdk/firecrawl/v2/methods/agent.py@@ -18,6 +18,7 @@ def _prepare_agent_request(model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,) -> Dict[str, Any]:body: Dict[str, Any] = {}if urls is not None:@@ -49,6 +50,8 @@ def _prepare_agent_request(body["threatProtection"] = threat_protection.model_dump(by_alias=True, exclude_none=True)+ if audit_metadata is not None:+ body["auditMetadata"] = audit_metadatareturn body@@ -73,6 +76,7 @@ def start_agent(model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,) -> AgentResponse:body = _prepare_agent_request(urls,@@ -84,6 +88,7 @@ def start_agent(model=model,webhook=webhook,threat_protection=threat_protection,+ audit_metadata=audit_metadata,)resp = client.post("/v2/agent", body)if not resp.ok:@@ -131,6 +136,7 @@ def agent(model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,) -> AgentResponse:started = start_agent(client,@@ -143,6 +149,7 @@ def agent(model=model,webhook=webhook,threat_protection=threat_protection,+ audit_metadata=audit_metadata,)job_id = getattr(started, "id", None)if not job_id:diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py b/apps/python-sdk/firecrawl/v2/methods/aio/agent.pyindex 601399871..86eecc076 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/agent.py@@ -18,6 +18,7 @@ def _prepare_agent_request(model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,) -> Dict[str, Any]:body: Dict[str, Any] = {}if urls is not None:@@ -49,6 +50,8 @@ def _prepare_agent_request(body["threatProtection"] = threat_protection.model_dump(by_alias=True, exclude_none=True)+ if audit_metadata is not None:+ body["auditMetadata"] = audit_metadatareturn body@@ -73,6 +76,7 @@ async def start_agent(model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,) -> AgentResponse:body = _prepare_agent_request(urls,@@ -84,6 +88,7 @@ async def start_agent(model=model,webhook=webhook,threat_protection=threat_protection,+ audit_metadata=audit_metadata,)resp = await client.post("/v2/agent", body)if not resp.ok:@@ -131,6 +136,7 @@ async def agent(model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,webhook: Optional[Union[str, AgentWebhookConfig]] = None,threat_protection: Optional[ThreatProtectionOptions] = None,+ audit_metadata: Optional[Dict[str, str]] = None,) -> AgentResponse:started = await start_agent(client,@@ -143,6 +149,7 @@ async def agent(model=model,webhook=webhook,threat_protection=threat_protection,+ audit_metadata=audit_metadata,)job_id = getattr(started, "id", None)if not job_id:diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py b/apps/python-sdk/firecrawl/v2/methods/aio/batch.pyindex 4585f4443..1cdd1a13a 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py@@ -40,6 +40,8 @@ def _prepare(urls: List[str], *, options: Optional[ScrapeOptions] = None, **kwaropts = prepare_scrape_options(options)if opts:payload.update(opts)+ if (v := kwargs.get("audit_metadata")) is not None:+ payload["auditMetadata"] = vif (w := kwargs.get("webhook")) is not None:payload["webhook"] = w if isinstance(w, str) else w.model_dump(exclude_none=True)if (v := kwargs.get("append_to_id")) is not None:diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/map.py b/apps/python-sdk/firecrawl/v2/methods/aio/map.pyindex 883b01cb5..4ef0e35ae 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/map.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/map.py@@ -32,6 +32,8 @@ def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dictdata["threatProtection"] = options.threat_protection.model_dump(by_alias=True, exclude_none=True)+ if options.audit_metadata is not None:+ data["auditMetadata"] = options.audit_metadatapayload.update(data)return payload@@ -68,4 +70,3 @@ async def map(client: AsyncHttpClient, url: str, options: Optional[MapOptions] =result_links.append(LinkResult(url=item))return MapData(links=result_links)-diff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.pyindex 2b3477d66..f81768018 100644--- a/apps/python-sdk/firecrawl/v2/methods/batch.py+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py@@ -53,6 +53,7 @@ def start_batch_scrape(max_concurrency: Optional[int] = None,zero_data_retention: Optional[bool] = None,integration: Optional[str] = None,+ audit_metadata: Optional[Dict[str, str]] = None,idempotency_key: Optional[str] = None,) -> BatchScrapeResponse:"""@@ -79,6 +80,7 @@ def start_batch_scrape(max_concurrency=max_concurrency,zero_data_retention=zero_data_retention,integration=integration,+ audit_metadata=audit_metadata,)# Make the API request@@ -341,6 +343,7 @@ def batch_scrape(max_concurrency: Optional[int] = None,zero_data_retention: Optional[bool] = None,integration: Optional[str] = None,+ audit_metadata: Optional[Dict[str, str]] = None,idempotency_key: Optional[str] = None,poll_interval: int = 2,timeout: Optional[int] = None@@ -373,6 +376,7 @@ def batch_scrape(max_concurrency=max_concurrency,zero_data_retention=zero_data_retention,integration=integration,+ audit_metadata=audit_metadata,idempotency_key=idempotency_key,)@@ -424,6 +428,7 @@ def prepare_batch_scrape_request(max_concurrency: Optional[int] = None,zero_data_retention: Optional[bool] = None,integration: Optional[str] = None,+ audit_metadata: Optional[Dict[str, str]] = None,) -> dict:"""Prepare a batch scrape request payload.@@ -443,6 +448,8 @@ def prepare_batch_scrape_request(scrape_data = prepare_scrape_options(options)if scrape_data:request_data.update(scrape_data)+ if audit_metadata is not None:+ request_data["auditMetadata"] = audit_metadata# Batch-specific fieldsif webhook is not None:diff --git a/apps/python-sdk/firecrawl/v2/methods/map.py b/apps/python-sdk/firecrawl/v2/methods/map.pyindex 5181931cf..7da8073ba 100644--- a/apps/python-sdk/firecrawl/v2/methods/map.py+++ b/apps/python-sdk/firecrawl/v2/methods/map.py@@ -39,6 +39,8 @@ def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dictdata["threatProtection"] = options.threat_protection.model_dump(by_alias=True, exclude_none=True)+ if options.audit_metadata is not None:+ data["auditMetadata"] = options.audit_metadatapayload.update(data)return payload@@ -86,4 +88,3 @@ def map(client: HttpClient, url: str, options: Optional[MapOptions] = None) -> Mresult_links.append(LinkResult(url=item))return MapData(links=result_links)-diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex a769e843a..f0012742f 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -822,6 +822,9 @@ class ScrapeOptions(BaseModel):threat_protection: Optional[ThreatProtectionOptions] = Field(default=None, alias="threatProtection")+ audit_metadata: Optional[Dict[str, str]] = Field(+ default=None, alias="auditMetadata"+ )profile: Optional[Dict[str, Any]] = Noneintegration: Optional[str] = None@@ -1054,6 +1057,7 @@ class MapOptions(BaseModel):integration: Optional[str] = Nonelocation: Optional["Location"] = Nonethreat_protection: Optional[ThreatProtectionOptions] = None+ audit_metadata: Optional[Dict[str, str]] = Noneclass MapRequest(BaseModel):diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.pyindex 1f16e06a7..ac87d8fb5 100644--- a/apps/python-sdk/firecrawl/v2/utils/validation.py+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py@@ -569,6 +569,7 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st"min_age": "minAge","redact_pii": "redactPII","threat_protection": "threatProtection",+ "audit_metadata": "auditMetadata",}# Apply field mappings@@ -818,4 +819,4 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st# For fields that don't need conversion, use as-isscrape_data[key] = value- return scrape_data\ No newline at end of file+ return scrape_datadiff --git a/apps/ruby-sdk/lib/firecrawl/models/agent_options.rb b/apps/ruby-sdk/lib/firecrawl/models/agent_options.rbindex 17daba219..c6f3313be 100644--- a/apps/ruby-sdk/lib/firecrawl/models/agent_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/agent_options.rb@@ -6,7 +6,7 @@ module Firecrawlclass AgentOptionsFIELDS = %i[urls prompt schema integration max_credits- strict_constrain_to_urls model webhook+ strict_constrain_to_urls model webhook audit_metadata].freezeattr_reader(*FIELDS)@@ -26,6 +26,7 @@ module Firecrawl"strictConstrainToURLs" => strict_constrain_to_urls,"model" => model,"webhook" => webhook.is_a?(Hash) ? webhook : webhook&.to_h,+ "auditMetadata" => audit_metadata,}.compactendenddiff --git a/apps/ruby-sdk/lib/firecrawl/models/map_options.rb b/apps/ruby-sdk/lib/firecrawl/models/map_options.rbindex 47c176ad7..1f3824f68 100644--- a/apps/ruby-sdk/lib/firecrawl/models/map_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/map_options.rb@@ -6,7 +6,7 @@ module Firecrawlclass MapOptionsFIELDS = %i[search sitemap include_subdomains ignore_query_parameters- limit timeout integration location+ limit timeout integration location audit_metadata].freezeattr_reader(*FIELDS)@@ -25,6 +25,7 @@ module Firecrawl"timeout" => timeout,"integration" => integration,"location" => location.is_a?(Hash) ? location : location&.to_h,+ "auditMetadata" => audit_metadata,}.compactendenddiff --git a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rbindex b29d4fa04..7ba9235af 100644--- a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb@@ -13,7 +13,7 @@ module FirecrawlFIELDS = %i[formats headers include_tags exclude_tags only_main_contenttimeout parsers skip_tls_verification remove_base64_images- block_ads proxy integration redact_pii json_options+ block_ads proxy integration redact_pii json_options audit_metadata].freezeattr_reader(*FIELDS)@@ -40,6 +40,7 @@ module Firecrawl"integration" => integration,"redactPII" => redact_pii,"jsonOptions" => json_options.is_a?(Hash) ? json_options : json_options&.to_h,+ "auditMetadata" => audit_metadata,}.compactenddiff --git a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rbindex 50d4d43d5..d860268b9 100644--- a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb@@ -8,7 +8,7 @@ module Firecrawlformats headers include_tags exclude_tags only_main_contenttimeout wait_for mobile parsers actions locationskip_tls_verification remove_base64_images block_ads proxy- max_age store_in_cache lockdown redact_pii integration+ max_age store_in_cache lockdown redact_pii integration audit_metadata].freezeattr_reader(*FIELDS)@@ -40,6 +40,7 @@ module Firecrawl"lockdown" => lockdown,"redactPII" => redact_pii,"integration" => integration,+ "auditMetadata" => audit_metadata,}.compactenddiff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rbindex fd14285d4..7706c5496 100644--- a/apps/ruby-sdk/lib/firecrawl/version.rb+++ b/apps/ruby-sdk/lib/firecrawl/version.rb@@ -1,5 +1,5 @@# frozen_string_literal: truemodule Firecrawl- VERSION = "1.10.1"+ VERSION = "1.11.0"enddiff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rbindex 8019993f9..0165206cc 100644--- a/apps/ruby-sdk/test/firecrawl/client_test.rb+++ b/apps/ruby-sdk/test/firecrawl/client_test.rb@@ -769,6 +769,23 @@ class ClientTest < Minitest::Testassert_equal "spark-1-pro", h["model"]end+ def test_audit_metadata_to_h+ metadata = { "requestId" => "req-123" }++ assert_equal metadata, Firecrawl::Models::ScrapeOptions.new(audit_metadata: metadata).to_h["auditMetadata"]+ assert_equal metadata, Firecrawl::Models::MapOptions.new(audit_metadata: metadata).to_h["auditMetadata"]+ assert_equal metadata,+ Firecrawl::Models::AgentOptions.new(+ prompt: "find pricing",+ audit_metadata: metadata+ ).to_h["auditMetadata"]+ assert_equal metadata, Firecrawl::Models::ParseOptions.new(audit_metadata: metadata).to_h["auditMetadata"]+ crawl = Firecrawl::Models::CrawlOptions.new(+ scrape_options: Firecrawl::Models::ScrapeOptions.new(audit_metadata: metadata)+ )+ assert_equal metadata, crawl.to_h.dig("scrapeOptions", "auditMetadata")+ end+def test_batch_scrape_options_to_hscrape_opts = Firecrawl::Models::ScrapeOptions.new(formats: ["markdown"])opts = Firecrawl::Models::BatchScrapeOptions.new(diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lockindex c5fda5043..1918d7a5e 100644--- a/apps/rust-sdk/Cargo.lock+++ b/apps/rust-sdk/Cargo.lock@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"[[package]]name = "firecrawl"-version = "2.11.1"+version = "2.12.0"dependencies = ["mockito","reqwest",diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex 40611dcad..3bdfa51a4 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,6 +1,6 @@[package]name = "firecrawl"-version = "2.11.1"+version = "2.12.0"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"diff --git a/apps/rust-sdk/src/agent.rs b/apps/rust-sdk/src/agent.rsindex bd4e77796..a4c18b5a2 100644--- a/apps/rust-sdk/src/agent.rs+++ b/apps/rust-sdk/src/agent.rs@@ -5,6 +5,7 @@use serde::{Deserialize, Serialize};use serde_json::Value;+use std::collections::HashMap;use crate::client::Client;use crate::types::{AgentModel, AgentWebhookConfig};@@ -39,6 +40,9 @@ pub struct AgentOptions {/// Webhook configuration for agent notifications.pub webhook: Option<AgentWebhookConfig>,+ /// Metadata to include with SIEM logging events.+ pub audit_metadata: Option<HashMap<String, String>>,+/// Poll interval for synchronous agent execution (milliseconds).#[serde(skip)]pub poll_interval: Option<u64>,diff --git a/apps/rust-sdk/src/map.rs b/apps/rust-sdk/src/map.rsindex 724e8bad2..1d686ae78 100644--- a/apps/rust-sdk/src/map.rs+++ b/apps/rust-sdk/src/map.rs@@ -1,6 +1,7 @@//! Map endpoint for Firecrawl API v2.use serde::{Deserialize, Serialize};+use std::collections::HashMap;use crate::client::Client;use crate::types::{LocationConfig, SearchResultWeb, SitemapMode};@@ -34,6 +35,9 @@ pub struct MapOptions {/// Location configuration for proxy routing.pub location: Option<LocationConfig>,++ /// Metadata to include with SIEM logging events.+ pub audit_metadata: Option<HashMap<String, String>>,}/// Request body for map endpoint.diff --git a/apps/rust-sdk/src/parse.rs b/apps/rust-sdk/src/parse.rsindex 843ef1dc7..a6a4643c2 100644--- a/apps/rust-sdk/src/parse.rs+++ b/apps/rust-sdk/src/parse.rs@@ -132,6 +132,8 @@ pub struct ParseOptions {/// Redact personally identifiable information from returned content.#[serde(rename = "redactPII")]pub redact_pii: Option<bool>,+ /// Metadata to include with SIEM logging events.+ pub audit_metadata: Option<HashMap<String, String>>,/// Request origin identifier.pub origin: Option<String>,/// Zero data retention mode.diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rsindex 94820b20b..f697a0a8b 100644--- a/apps/rust-sdk/src/scrape.rs+++ b/apps/rust-sdk/src/scrape.rs@@ -83,6 +83,9 @@ pub struct ScrapeOptions {#[serde(rename = "redactPII")]pub redact_pii: Option<bool>,+ /// Metadata to include with SIEM logging events.+ pub audit_metadata: Option<HashMap<String, String>>,+/// Persistent browser profile for maintaining state across scrapes.pub profile: Option<ProfileConfig>,diff --git a/apps/rust-sdk/tests/audit_metadata.rs b/apps/rust-sdk/tests/audit_metadata.rsnew file mode 100644index 000000000..55c46c242--- /dev/null+++ b/apps/rust-sdk/tests/audit_metadata.rs@@ -0,0 +1,32 @@+use std::collections::HashMap;++use firecrawl::{AgentOptions, MapOptions, ParseOptions, ScrapeOptions};+use serde::Serialize;++fn assert_audit_metadata<T: Serialize>(options: T) {+ let value = serde_json::to_value(options).expect("options should serialize");+ assert_eq!(value["auditMetadata"]["requestId"], "req-123");+}++#[test]+fn serializes_audit_metadata_across_request_options() {+ let metadata = HashMap::from([("requestId".to_string(), "req-123".to_string())]);++ assert_audit_metadata(ScrapeOptions {+ audit_metadata: Some(metadata.clone()),+ ..Default::default()+ });+ assert_audit_metadata(MapOptions {+ audit_metadata: Some(metadata.clone()),+ ..Default::default()+ });+ assert_audit_metadata(AgentOptions {+ prompt: "find pricing".to_string(),+ audit_metadata: Some(metadata.clone()),+ ..Default::default()+ });+ assert_audit_metadata(ParseOptions {+ audit_metadata: Some(metadata),+ ..Default::default()+ });+}diff --git a/apps/siem/azure-sentinel/DCR.json b/apps/siem/azure-sentinel/DCR.jsonnew file mode 100644index 000000000..d8e2fe29e--- /dev/null+++ b/apps/siem/azure-sentinel/DCR.json@@ -0,0 +1,115 @@+{+ "name": "FirecrawlScrapeActivityPushDCR",+ "apiVersion": "2021-09-01-preview",+ "type": "Microsoft.Insights/dataCollectionRules",+ "location": "[parameters('workspace-location')]",+ "properties": {+ "streamDeclarations": {+ "Custom-FirecrawlScrapeActivity": {+ "columns": [+ {+ "name": "schema_version",+ "type": "int"+ },+ {+ "name": "event_type",+ "type": "string"+ },+ {+ "name": "scrape_id",+ "type": "string"+ },+ {+ "name": "request_id",+ "type": "string"+ },+ {+ "name": "endpoint",+ "type": "string"+ },+ {+ "name": "team_id",+ "type": "string"+ },+ {+ "name": "org_id",+ "type": "string"+ },+ {+ "name": "api_key",+ "type": "dynamic"+ },+ {+ "name": "audit_metadata",+ "type": "dynamic"+ },+ {+ "name": "started_at",+ "type": "datetime"+ },+ {+ "name": "completed_at",+ "type": "datetime"+ },+ {+ "name": "url",+ "type": "string"+ },+ {+ "name": "domain",+ "type": "string"+ },+ {+ "name": "http_method",+ "type": "string"+ },+ {+ "name": "http_status",+ "type": "int"+ },+ {+ "name": "result",+ "type": "string"+ },+ {+ "name": "error",+ "type": "dynamic"+ },+ {+ "name": "origin",+ "type": "string"+ },+ {+ "name": "integration",+ "type": "string"+ },+ {+ "name": "zero_data_retention",+ "type": "bool"+ },+ {+ "name": "threat",+ "type": "dynamic"+ }+ ]+ }+ },+ "destinations": {+ "logAnalytics": [+ {+ "workspaceResourceId": "[variables('workspaceResourceId')]",+ "name": "firecrawlWorkspace"+ }+ ]+ },+ "dataFlows": [+ {+ "streams": ["Custom-FirecrawlScrapeActivity"],+ "destinations": ["firecrawlWorkspace"],+ "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tobool(threat.security_alert.detected), \"High\", tostring(threat.decision) == \"deny\", \"Low\", result == \"success\", \"Informational\", \"Low\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"audit_metadata\", audit_metadata, \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)",+ "outputStream": "Microsoft-ASimWebSessionLogs"+ }+ ],+ "dataCollectionEndpointId": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/', parameters('resourceGroupName'), '/providers/Microsoft.Insights/dataCollectionEndpoints/', parameters('workspace'))]"+ }+}diff --git a/apps/siem/azure-sentinel/README.md b/apps/siem/azure-sentinel/README.mdnew file mode 100644index 000000000..7bb0bf43d--- /dev/null+++ b/apps/siem/azure-sentinel/README.md@@ -0,0 +1,28 @@+# SIEM Logging for Microsoft Sentinel++Deployment artifacts for sending Firecrawl scrape activity to Microsoft+Sentinel through the Azure Monitor Logs Ingestion API.++## Files++- `azuredeploy.json` creates the data collection endpoint and rule.+- `DCR.json` contains the CCF data collection rule and ASIM transform.+- `connectorDefinition.json` and `dataConnector.json` define the CCF Push+ connector.+- `sample-event.json` is a valid input-stream event.++The database schema is maintained as a migration in the Firecrawl database+repository, not in this package.++## Runtime++`PUT /v2/team/siem` stores an organization destination and its encrypted,+write-only client secret. Scrape activity is placed on a shared BullMQ queue;+index-worker replicas consume that queue and retry transient delivery failures.+Queue jobs contain the organization ID and event, never the destination secret.++Set `SIEM_LOGGING_ENCRYPTION_KEY` in API and index-worker environments. If+partner egress is required, also set `PARTNER_EGRESS_PROXY_URL`.++Use `POST /v2/team/siem/test` to validate credentials and the destination+schema after deploying the Azure artifacts.diff --git a/apps/siem/azure-sentinel/azuredeploy.json b/apps/siem/azure-sentinel/azuredeploy.jsonnew file mode 100644index 000000000..955d7a188--- /dev/null+++ b/apps/siem/azure-sentinel/azuredeploy.json@@ -0,0 +1,178 @@+{+ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",+ "contentVersion": "1.0.0.0",+ "parameters": {+ "location": {+ "type": "string",+ "defaultValue": "[resourceGroup().location]"+ },+ "workspaceName": {+ "type": "string"+ },+ "dataCollectionEndpointName": {+ "type": "string",+ "defaultValue": "firecrawl-scrape-activity"+ },+ "dataCollectionRuleName": {+ "type": "string",+ "defaultValue": "FirecrawlScrapeActivityPushDCR"+ }+ },+ "variables": {+ "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspaceName'))]",+ "streamName": "Custom-FirecrawlScrapeActivity"+ },+ "resources": [+ {+ "type": "Microsoft.Insights/dataCollectionEndpoints",+ "apiVersion": "2022-06-01",+ "name": "[parameters('dataCollectionEndpointName')]",+ "location": "[parameters('location')]",+ "properties": {+ "networkAcls": {+ "publicNetworkAccess": "Enabled"+ }+ }+ },+ {+ "type": "Microsoft.Insights/dataCollectionRules",+ "apiVersion": "2021-09-01-preview",+ "name": "[parameters('dataCollectionRuleName')]",+ "location": "[parameters('location')]",+ "dependsOn": [+ "[resourceId('Microsoft.Insights/dataCollectionEndpoints', parameters('dataCollectionEndpointName'))]"+ ],+ "properties": {+ "streamDeclarations": {+ "Custom-FirecrawlScrapeActivity": {+ "columns": [+ {+ "name": "schema_version",+ "type": "int"+ },+ {+ "name": "event_type",+ "type": "string"+ },+ {+ "name": "scrape_id",+ "type": "string"+ },+ {+ "name": "request_id",+ "type": "string"+ },+ {+ "name": "endpoint",+ "type": "string"+ },+ {+ "name": "team_id",+ "type": "string"+ },+ {+ "name": "org_id",+ "type": "string"+ },+ {+ "name": "api_key",+ "type": "dynamic"+ },+ {+ "name": "audit_metadata",+ "type": "dynamic"+ },+ {+ "name": "started_at",+ "type": "datetime"+ },+ {+ "name": "completed_at",+ "type": "datetime"+ },+ {+ "name": "url",+ "type": "string"+ },+ {+ "name": "domain",+ "type": "string"+ },+ {+ "name": "http_method",+ "type": "string"+ },+ {+ "name": "http_status",+ "type": "int"+ },+ {+ "name": "result",+ "type": "string"+ },+ {+ "name": "error",+ "type": "dynamic"+ },+ {+ "name": "origin",+ "type": "string"+ },+ {+ "name": "integration",+ "type": "string"+ },+ {+ "name": "zero_data_retention",+ "type": "bool"+ },+ {+ "name": "threat",+ "type": "dynamic"+ }+ ]+ }+ },+ "destinations": {+ "logAnalytics": [+ {+ "workspaceResourceId": "[variables('workspaceResourceId')]",+ "name": "firecrawlWorkspace"+ }+ ]+ },+ "dataFlows": [+ {+ "streams": ["Custom-FirecrawlScrapeActivity"],+ "destinations": ["firecrawlWorkspace"],+ "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tobool(threat.security_alert.detected), \"High\", tostring(threat.decision) == \"deny\", \"Low\", result == \"success\", \"Informational\", \"Low\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"audit_metadata\", audit_metadata, \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)",+ "outputStream": "Microsoft-ASimWebSessionLogs"+ }+ ],+ "dataCollectionEndpointId": "[resourceId('Microsoft.Insights/dataCollectionEndpoints', parameters('dataCollectionEndpointName'))]"+ }+ }+ ],+ "outputs": {+ "streamName": {+ "type": "string",+ "value": "[variables('streamName')]"+ },+ "dataCollectionEndpointResourceId": {+ "type": "string",+ "value": "[resourceId('Microsoft.Insights/dataCollectionEndpoints', parameters('dataCollectionEndpointName'))]"+ },+ "dataCollectionEndpoint": {+ "type": "string",+ "value": "[reference(resourceId('Microsoft.Insights/dataCollectionEndpoints', parameters('dataCollectionEndpointName')), '2022-06-01').properties.logsIngestion.endpoint]"+ },+ "dataCollectionRuleResourceId": {+ "type": "string",+ "value": "[resourceId('Microsoft.Insights/dataCollectionRules', parameters('dataCollectionRuleName'))]"+ },+ "dataCollectionRuleImmutableId": {+ "type": "string",+ "value": "[reference(resourceId('Microsoft.Insights/dataCollectionRules', parameters('dataCollectionRuleName')), '2021-09-01-preview').properties.immutableId]"+ }+ }+}diff --git a/apps/siem/azure-sentinel/connectorDefinition.json b/apps/siem/azure-sentinel/connectorDefinition.jsonnew file mode 100644index 000000000..bb503e2ee--- /dev/null+++ b/apps/siem/azure-sentinel/connectorDefinition.json@@ -0,0 +1,143 @@+{+ "name": "FirecrawlScrapeActivityPush",+ "apiVersion": "2022-09-01-preview",+ "type": "Microsoft.SecurityInsights/dataConnectorDefinitions",+ "location": "[parameters('workspace-location')]",+ "kind": "Customizable",+ "properties": {+ "connectorUiConfig": {+ "id": "FirecrawlScrapeActivityPush",+ "title": "Firecrawl Scrape Activity (Push)",+ "publisher": "Firecrawl",+ "descriptionMarkdown": "Pushes Firecrawl scrape activity into Microsoft Sentinel through the Azure Monitor Logs Ingestion API. The customer-owned DCR normalizes native events into ASimWebSessionLogs.",+ "graphQueries": [+ {+ "metricName": "Scrape activity",+ "legend": "Firecrawl",+ "baseQuery": "ASimWebSessionLogs | where EventVendor == \"Firecrawl\""+ }+ ],+ "sampleQueries": [+ {+ "description": "Recent scrape activity",+ "query": "ASimWebSessionLogs | where EventVendor == \"Firecrawl\" | sort by TimeGenerated desc"+ },+ {+ "description": "Blocked activity and security alerts",+ "query": "ASimWebSessionLogs | where EventVendor == \"Firecrawl\" and DvcAction == \"Deny\" | project TimeGenerated, SrcUsername, Url, RuleName, ThreatCategory, EventSeverity"+ }+ ],+ "dataTypes": [+ {+ "name": "ASimWebSessionLogs",+ "lastDataReceivedQuery": "ASimWebSessionLogs | where EventVendor == \"Firecrawl\" | summarize Time = max(TimeGenerated) | where isnotempty(Time)"+ }+ ],+ "connectivityCriteria": [+ {+ "type": "IsConnectedQuery",+ "value": [+ "ASimWebSessionLogs | where EventVendor == \"Firecrawl\" | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(7d)"+ ]+ }+ ],+ "availability": {+ "status": 1+ },+ "permissions": {+ "resourceProvider": [+ {+ "provider": "Microsoft.OperationalInsights/workspaces",+ "permissionsDisplayText": "Read and write permissions are required.",+ "providerDisplayName": "Workspace",+ "scope": "Workspace",+ "requiredPermissions": {+ "write": true,+ "read": true,+ "delete": true+ }+ }+ ],+ "customs": [+ {+ "name": "Microsoft Entra",+ "description": "Permission to create an app registration and client secret."+ },+ {+ "name": "Microsoft Azure",+ "description": "Permission to assign Monitoring Metrics Publisher on the data collection rule."+ }+ ]+ },+ "instructionSteps": [+ {+ "title": "1. Deploy the push connector resources",+ "description": "Create the data collection endpoint, data collection rule, Entra application, and role assignment.",+ "instructions": [+ {+ "type": "Markdown",+ "parameters": {+ "content": "The DCR transforms Firecrawl's native scrape activity schema into ASimWebSessionLogs. Review the transform before deployment because it is controlled by your workspace."+ }+ },+ {+ "type": "DeployPushConnectorButton",+ "parameters": {+ "label": "Deploy Firecrawl push connector resources",+ "applicationDisplayName": "Firecrawl Scrape Activity Push Connector"+ }+ }+ ]+ },+ {+ "title": "2. Configure Firecrawl",+ "description": "Copy the generated values into the Firecrawl Enterprise Controls SIEM form.",+ "instructions": [+ {+ "type": "CopyableLabel",+ "parameters": {+ "label": "Tenant ID",+ "fillWith": ["TenantId"]+ }+ },+ {+ "type": "CopyableLabel",+ "parameters": {+ "label": "Application ID",+ "fillWith": ["ApplicationId"]+ }+ },+ {+ "type": "CopyableLabel",+ "parameters": {+ "label": "Application Secret",+ "fillWith": ["ApplicationSecret"]+ }+ },+ {+ "type": "CopyableLabel",+ "parameters": {+ "label": "Data Collection Endpoint",+ "fillWith": ["DataCollectionEndpoint"]+ }+ },+ {+ "type": "CopyableLabel",+ "parameters": {+ "label": "DCR Immutable ID",+ "fillWith": ["DataCollectionRuleId"]+ }+ },+ {+ "type": "CopyableLabel",+ "parameters": {+ "label": "Stream Name",+ "value": "Custom-FirecrawlScrapeActivity"+ }+ }+ ]+ }+ ]+ }+ }+}diff --git a/apps/siem/azure-sentinel/dataConnector.json b/apps/siem/azure-sentinel/dataConnector.jsonnew file mode 100644index 000000000..95ccffa73--- /dev/null+++ b/apps/siem/azure-sentinel/dataConnector.json@@ -0,0 +1,25 @@+{+ "name": "FirecrawlScrapeActivityPushConnector",+ "apiVersion": "2024-09-01",+ "type": "Microsoft.SecurityInsights/dataConnectors",+ "kind": "Push",+ "properties": {+ "connectorDefinitionName": "FirecrawlScrapeActivityPush",+ "dcrConfig": {+ "streamName": "Custom-FirecrawlScrapeActivity",+ "dataCollectionEndpoint": "[[parameters('dcrConfig').dataCollectionEndpoint]]",+ "dataCollectionRuleImmutableId": "[[parameters('dcrConfig').dataCollectionRuleImmutableId]]"+ },+ "auth": {+ "type": "Push",+ "AppId": "[[parameters('auth').appId]]",+ "ServicePrincipalId": "[[parameters('auth').servicePrincipalId]]"+ },+ "request": {+ "RetryCount": 1+ },+ "response": {+ "eventsJsonPaths": ["$"]+ }+ }+}diff --git a/apps/siem/azure-sentinel/sample-event.json b/apps/siem/azure-sentinel/sample-event.jsonnew file mode 100644index 000000000..e790e77a4--- /dev/null+++ b/apps/siem/azure-sentinel/sample-event.json@@ -0,0 +1,39 @@+[+ {+ "schema_version": 1,+ "event_type": "scrape_activity",+ "scrape_id": "00000000-0000-0000-0000-000000000001",+ "request_id": "00000000-0000-0000-0000-000000000002",+ "endpoint": "scrape",+ "team_id": "00000000-0000-0000-0000-000000000003",+ "org_id": "00000000-0000-0000-0000-000000000004",+ "api_key": {+ "id": "123",+ "name": "production"+ },+ "audit_metadata": {+ "username": "[email protected]"+ },+ "started_at": "2026-07-24T10:00:00.000Z",+ "completed_at": "2026-07-24T10:00:01.500Z",+ "url": "https://example.com/path?q=1",+ "domain": "example.com",+ "http_method": "GET",+ "http_status": 200,+ "result": "success",+ "error": null,+ "origin": "api",+ "integration": "node-sdk",+ "zero_data_retention": true,+ "threat": {+ "decision": "allow",+ "rule": "default-allow",+ "provider": "google-web-risk",+ "categories": [],+ "security_alert": {+ "detected": false,+ "category": null+ }+ }+ }+]