Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(siem): batch audit delivery over RabbitMQ and scope threat severity (#4145)
GergΕ MΓ³ricz882f64cbd3e1
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 177b7f7af..3b39e6b74 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -11,7 +11,6 @@ import {getDeepResearchQueue,getBillingQueue,getPrecrawlQueue,- getSiemLoggingQueue,} from "./services/queue-service";import { v0Router } from "./routes/v0";import os from "os";@@ -104,7 +103,8 @@ const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({new BullMQAdapter(getDeepResearchQueue()),new BullMQAdapter(getBillingQueue()),new BullMQAdapter(getPrecrawlQueue()),- new BullMQAdapter(getSiemLoggingQueue()),+ // SIEM audit delivery runs on RabbitMQ; inspect it in the broker's+ // management UI, not here.],serverAdapter: serverAdapter,});diff --git a/apps/api/src/lib/siem-logging/azure-sentinel.test.ts b/apps/api/src/lib/siem-logging/azure-sentinel.test.tsindex 63e6145d3..cfcd79554 100644--- a/apps/api/src/lib/siem-logging/azure-sentinel.test.ts+++ b/apps/api/src/lib/siem-logging/azure-sentinel.test.ts@@ -319,13 +319,39 @@ describe("Azure Sentinel sender", () => {const compressedBodies = [...batches];expect(compressedBodies.length).toBeGreaterThan(1);- expect(compressedBodies.every(batch => batch.byteLength <= 1200)).toBe(+ expect(compressedBodies.every(batch => batch.body.byteLength <= 1200)).toBe(true,);expect(compressedBodies.flatMap(batch =>- JSON.parse(gunzipSync(batch).toString("utf8")),+ JSON.parse(gunzipSync(batch.body).toString("utf8")),),).toHaveLength(noisyEvents.length);+ // Each batch reports its own size so the caller can count what it delivered.+ expect(+ compressedBodies.reduce((total, batch) => total + batch.eventCount, 0),+ ).toBe(noisyEvents.length);+ });++ // An event too large to ever deliver must not take its batch down with it: the+ // sender POSTs chunk by chunk, so failing the call would both lose the good+ // events and duplicate any chunk already accepted when they were replayed.+ it("skips an undeliverable event and still delivers the rest of the batch", () => {+ const oversized = event(+ Array.from({ length: 400 }, () => crypto.randomUUID()).join(""),+ );+ const events = [event("small-1"), oversized, event("small-2")];++ const batches = [...buildCompressedBatches(events, 1200)];+ const delivered = batches.flatMap(batch =>+ JSON.parse(gunzipSync(batch.body).toString("utf8")),+ ) as ScrapeActivityEvent[];++ expect(delivered.map(e => e.scrape_id)).toEqual(["small-1", "small-2"]);+ // The skipped event must not be counted as delivered β it is already+ // counted as dropped, and double-counting it corrupts both metrics.+ expect(batches.reduce((total, batch) => total + batch.eventCount, 0)).toBe(+ 2,+ );});});diff --git a/apps/api/src/lib/siem-logging/azure-sentinel.ts b/apps/api/src/lib/siem-logging/azure-sentinel.tsindex 8bef81489..ae9b8a74d 100644--- a/apps/api/src/lib/siem-logging/azure-sentinel.ts+++ b/apps/api/src/lib/siem-logging/azure-sentinel.ts@@ -7,10 +7,16 @@ import {type Response,} from "undici";import { config } from "../../config";-import { siemLoggingDeliveryBatchesTotal } from "./metrics";+import { logger as _logger } from "../logger";+import {+ siemLoggingDeliveryBatchesTotal,+ siemLoggingEventsTotal,+} from "./metrics";import type { AzureSentinelDestination, ScrapeActivityEvent } from "./types";import { SiemDeliveryError } from "./types";+const logger = _logger.child({ module: "siem-logging-azure-sentinel" });+const MAX_COMPRESSED_BODY_BYTES = 1_000_000;const MAX_ATTEMPTS = 3;const MAX_RETRY_DELAY_MS = 30_000;@@ -213,40 +219,61 @@ async function getAccessToken();}+function gzipEvents(events: ScrapeActivityEvent[]): Buffer {+ return gzipSync(Buffer.from(JSON.stringify(events), "utf8"));+}++/**+ * A gzipped request body plus how many events it carries, so the caller can+ * report what was actually delivered rather than what it was handed β a batch+ * can contain fewer events than the input if any were skipped as undeliverable.+ */+interface CompressedBatch {+ body: Buffer;+ eventCount: number;+}++function compressedBatch(events: ScrapeActivityEvent[]): CompressedBatch {+ return { body: gzipEvents(events), eventCount: events.length };+}+export function* buildCompressedBatches(events: ScrapeActivityEvent[],maxBytes = MAX_COMPRESSED_BODY_BYTES,-): Generator<Buffer> {+): Generator<CompressedBatch> {let pending: ScrapeActivityEvent[] = [];for (const event of events) {const candidate = [...pending, event];- const compressed = gzipSync(Buffer.from(JSON.stringify(candidate), "utf8"));- if (compressed.byteLength <= maxBytes) {+ if (gzipEvents(candidate).byteLength <= maxBytes) {pending = candidate;continue;}- if (pending.length === 0) {- throw new SiemDeliveryError(- "payload_too_large",- `A single SIEM event exceeds the ${maxBytes}-byte compressed payload limit`,- );+ // The event doesn't fit alongside what's pending β close that batch out and+ // consider the event on its own.+ if (pending.length > 0) {+ yield compressedBatch(pending);+ pending = [];}- yield gzipSync(Buffer.from(JSON.stringify(pending), "utf8"));- pending = [event];- const single = gzipSync(Buffer.from(JSON.stringify(pending), "utf8"));- if (single.byteLength > maxBytes) {- throw new SiemDeliveryError(- "payload_too_large",- `A single SIEM event exceeds the ${maxBytes}-byte compressed payload limit`,- );++ if (gzipEvents([event]).byteLength > maxBytes) {+ // Undeliverable at any batch size, and no retry will change that. Skip it+ // and keep going: failing the call here would take down every other event+ // in the batch, and dead-lettering the batch would duplicate the ones+ // already POSTed in earlier chunks.+ siemLoggingEventsTotal.inc({ result: "dropped_oversized" });+ logger.error("Dropping a SIEM event larger than the payload limit", {+ maxBytes,+ scrapeId: event.scrape_id,+ orgId: event.org_id,+ });+ continue;}+ pending = [event];}- if (pending.length > 0) {- yield gzipSync(Buffer.from(JSON.stringify(pending), "utf8"));- }+ if (pending.length > 0) yield compressedBatch(pending);}async function sendBatch(@@ -321,14 +348,21 @@ async function sendBatch(}}+/**+ * Returns how many events were actually accepted by the destination, which is+ * fewer than `events.length` when any were skipped as undeliverable. Callers+ * must attribute their `delivered` metric to this count, not to what they+ * passed in, or a skipped event gets counted as both dropped and delivered.+ */export async function sendAzureSentinelEvents(destination: AzureSentinelDestination,events: ScrapeActivityEvent[],deps: SenderDependencies = {},-): Promise<void> {- if (events.length === 0) return;+): Promise<number> {+ if (events.length === 0) return 0;let token: string | undefined;- for (const body of buildCompressedBatches(events)) {+ let delivered = 0;+ for (const { body, eventCount } of buildCompressedBatches(events)) {try {token ??= await getAccessToken(destination, deps);try {@@ -345,18 +379,22 @@ export async function sendAzureSentinelEvents(throw error;}}+ delivered += eventCount;siemLoggingDeliveryBatchesTotal.inc({ result: "success" });} catch (error) {- if (- error instanceof SiemDeliveryError &&- (error.statusCode === 401 || error.statusCode === 403)- ) {- invalidateToken(destination);+ if (error instanceof SiemDeliveryError) {+ if (error.statusCode === 401 || error.statusCode === 403) {+ invalidateToken(destination);+ }+ // Tell the caller what already landed, so the chunks Azure accepted+ // aren't counted as failures on the way out.+ error.deliveredEvents = delivered;}siemLoggingDeliveryBatchesTotal.inc({ result: "failure" });throw error;}}+ return delivered;}export function clearAzureTokenCache(): void {diff --git a/apps/api/src/lib/siem-logging/event.test.ts b/apps/api/src/lib/siem-logging/event.test.tsindex 373de6081..f059bc88c 100644--- a/apps/api/src/lib/siem-logging/event.test.ts+++ b/apps/api/src/lib/siem-logging/event.test.ts@@ -126,11 +126,47 @@ describe("ScrapeActivityEvent", () => {expect(event.result).toBe("blocked");expect(event.error?.code).toBe("unsafe_domain_blocked");- expect(event.threat).toMatchObject({+ expect(event.threat).toEqual({decision: "deny",rule: "risk-score",+ provider: "google-web-risk",categories: ["MALWARE"],- security_alert: { detected: true, category: "MALWARE" },+ });+ });++ // A local-rule deny never calls a provider, so it must not inherit the+ // categories of a redirect hop that did β downstream that would score a+ // policy block as a provider-confirmed threat.+ it("scopes the threat block to the deciding decision", () => {+ const provider: ThreatDecision = {+ ...allowedDecision,+ url: "https://redirect.example.com/",+ verdict: {+ ...allowedDecision.verdict!,+ riskScore: 100,+ categories: ["MALWARE"],+ },+ };+ const localDeny: ThreatDecision = {+ ...allowedDecision,+ allowed: false,+ rule: "blacklist",+ providerConsulted: false,+ verdict: null,+ };+ const event = buildScrapeActivityEvent("scrape-id", job(), "org-id", null, {+ success: false,+ error: new UnsafeDomainBlockedError(localDeny.url, localDeny),+ threatDecisions: [provider, localDeny],+ startedAt: 1_000,+ completedAt: 2_000,+ });++ expect(event.threat).toEqual({+ decision: "deny",+ rule: "blacklist",+ provider: null,+ categories: [],});});diff --git a/apps/api/src/lib/siem-logging/event.ts b/apps/api/src/lib/siem-logging/event.tsindex 4651ee7de..33984e953 100644--- a/apps/api/src/lib/siem-logging/event.ts+++ b/apps/api/src/lib/siem-logging/event.ts@@ -78,6 +78,12 @@ function resultForOutcome(return "failure";}+// One scrape can take several decisions (the initial URL plus a re-check per+// redirect hop). Report the deciding one β the first deny, else the last allow β+// and take its categories and provider from that same decision. Unioning+// categories across decisions would let a local-rule deny (which never calls a+// provider, so carries no categories) inherit another hop's threat categories+// and be scored as a provider-confirmed threat downstream.function threatForDecisions(decisions: ThreatDecision[] | undefined,): ScrapeActivityThreat | undefined {@@ -85,20 +91,11 @@ function threatForDecisions(const decision =decisions.find(candidate => !candidate.allowed) ??decisions[decisions.length - 1];- const categories = [- ...new Set(- decisions.flatMap(candidate => candidate.verdict?.categories ?? []),- ),- ];return {decision: decision.allowed ? "allow" : "deny",rule: decision.rule,provider: decision.verdict?.provider ?? null,- categories,- security_alert: {- detected: categories.length > 0,- category: categories[0] ?? null,- },+ categories: [...new Set(decision.verdict?.categories ?? [])],};}diff --git a/apps/api/src/lib/siem-logging/index.test.ts b/apps/api/src/lib/siem-logging/index.test.tsindex 78631474a..0b9462411 100644--- a/apps/api/src/lib/siem-logging/index.test.ts+++ b/apps/api/src/lib/siem-logging/index.test.ts@@ -17,7 +17,7 @@ vi.mock("./store", () => ({getApiKeyName: mocks.getApiKeyName,}));-vi.mock("./queue", () => ({+vi.mock("./transport", () => ({enqueueSiemLoggingEvent: mocks.enqueue,enqueueSiemLoggingEvents: vi.fn().mockResolvedValue(undefined),}));diff --git a/apps/api/src/lib/siem-logging/index.ts b/apps/api/src/lib/siem-logging/index.tsindex e400fdae9..672370fb3 100644--- a/apps/api/src/lib/siem-logging/index.ts+++ b/apps/api/src/lib/siem-logging/index.ts@@ -7,7 +7,7 @@ import {type RejectedScrapeActivity,type ScrapeActivityOutcome,} from "./event";-import { enqueueSiemLoggingEvent, enqueueSiemLoggingEvents } from "./queue";+import { enqueueSiemLoggingEvent, enqueueSiemLoggingEvents } from "./transport";import { getApiKeyName, isOrgSiemLoggingEnabled } from "./store";const logger = _logger.child({ module: "siem-logging" });diff --git a/apps/api/src/lib/siem-logging/queue.test.ts b/apps/api/src/lib/siem-logging/queue.test.tsdeleted file mode 100644index 533c7ea61..000000000--- a/apps/api/src/lib/siem-logging/queue.test.ts+++ /dev/null@@ -1,57 +0,0 @@-import { beforeEach, describe, expect, it, vi } from "vitest";-import type { ScrapeActivityEvent } from "./types";--const mocks = vi.hoisted(() => ({- addBulk: vi.fn().mockResolvedValue([]),- inc: vi.fn(),-}));--vi.mock("../../services/queue-service", () => ({- getSiemLoggingQueue: () => ({ addBulk: mocks.addBulk }),-}));--vi.mock("./metrics", () => ({- siemLoggingEventsTotal: { inc: mocks.inc },-}));--import { enqueueSiemLoggingEvent } from "./queue";--const event: ScrapeActivityEvent = {- schema_version: 1,- event_type: "scrape_activity",- scrape_id: "scrape-id",- request_id: "request-id",- endpoint: "scrape",- team_id: "team-id",- org_id: "org-id",- api_key: { id: null, name: null },- started_at: "2026-07-27T00:00:00.000Z",- completed_at: "2026-07-27T00:00:01.000Z",- url: "https://example.com",- domain: "example.com",- http_method: "GET",- http_status: 200,- result: "success",- error: null,- origin: "api",- integration: null,- zero_data_retention: false,-};--describe("SIEM logging queue", () => {- beforeEach(() => {- vi.clearAllMocks();- });-- it("deduplicates repeat terminal events regardless of completion time", async () => {- await enqueueSiemLoggingEvent("org-id", event);- await enqueueSiemLoggingEvent("org-id", {- ...event,- completed_at: "2026-07-27T00:01:00.000Z",- });-- const firstJobId = mocks.addBulk.mock.calls[0][0][0].opts.jobId;- const secondJobId = mocks.addBulk.mock.calls[1][0][0].opts.jobId;- expect(firstJobId).toBe(secondJobId);- });-});diff --git a/apps/api/src/lib/siem-logging/queue.ts b/apps/api/src/lib/siem-logging/queue.tsdeleted file mode 100644index 45e343400..000000000--- a/apps/api/src/lib/siem-logging/queue.ts+++ /dev/null@@ -1,46 +0,0 @@-import { createHash } from "node:crypto";-import { getSiemLoggingQueue } from "../../services/queue-service";-import { siemLoggingEventsTotal } from "./metrics";-import type { ScrapeActivityEvent, SiemLoggingJobData } from "./types";--function jobIdFor(orgId: string, event: ScrapeActivityEvent): string {- return createHash("sha256")- .update(- JSON.stringify([- orgId,- event.schema_version,- event.event_type,- event.scrape_id,- event.request_id,- event.result,- ]),- )- .digest("hex");-}--export async function enqueueSiemLoggingEvent(- orgId: string,- event: ScrapeActivityEvent,-): Promise<void> {- await enqueueSiemLoggingEvents(orgId, [event]);-}--export async function enqueueSiemLoggingEvents(- orgId: string,- events: ScrapeActivityEvent[],-): Promise<void> {- if (events.length === 0) return;- try {- await getSiemLoggingQueue().addBulk(- events.map(event => ({- name: "deliver",- data: { orgId, event } satisfies SiemLoggingJobData,- opts: { jobId: jobIdFor(orgId, event) },- })),- );- siemLoggingEventsTotal.inc({ result: "queued" }, events.length);- } catch (error) {- siemLoggingEventsTotal.inc({ result: "enqueue_failed" }, events.length);- throw error;- }-}diff --git a/apps/api/src/lib/siem-logging/transport.test.ts b/apps/api/src/lib/siem-logging/transport.test.tsnew file mode 100644index 000000000..601dc8691--- /dev/null+++ b/apps/api/src/lib/siem-logging/transport.test.ts@@ -0,0 +1,506 @@+import type { Mock } from "vitest";+import type { ScrapeActivityEvent } from "./types";++const { connectMock, channelMock, connectionMock } = vi.hoisted(() => {+ const channelMock = {+ assertExchange: vi.fn(),+ assertQueue: vi.fn(),+ bindQueue: vi.fn(),+ prefetch: vi.fn(),+ consume: vi.fn(),+ ack: vi.fn(),+ nack: vi.fn(),+ // Confirm-channel shape: the last argument is the broker's ack callback.+ publish: vi.fn((...args: unknown[]) => {+ const callback = args[4];+ if (typeof callback === "function") {+ (callback as (error: Error | null) => void)(null);+ }+ return true;+ }),+ close: vi.fn(),+ on: vi.fn(),+ };+ // Async like the real client, so a test can hold a connection open mid-connect.+ const connectionMock = {+ createChannel: vi.fn(async () => channelMock),+ createConfirmChannel: vi.fn(async () => channelMock),+ on: vi.fn(),+ close: vi.fn(async () => {}),+ };+ const connectMock = vi.fn(() => connectionMock);+ return { connectMock, channelMock, connectionMock };+});++vi.mock("amqplib", () => ({ default: { connect: connectMock } }));+vi.mock("../../config", () => ({+ config: { NUQ_RABBITMQ_URL: "amqp://test" },+}));+// Each test re-imports the module under test, and prom-client rejects a second+// registration of the same counter name in one registry.+const { metricsMock } = vi.hoisted(() => ({ metricsMock: { inc: vi.fn() } }));+vi.mock("./metrics", () => ({ siemLoggingEventsTotal: metricsMock }));+vi.mock("../logger", () => {+ const mk = (): Record<string, unknown> => ({+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ child: () => mk(),+ });+ return { logger: mk() };+});++function event(scrapeId: string): ScrapeActivityEvent {+ return {+ schema_version: 1,+ event_type: "scrape_activity",+ scrape_id: scrapeId,+ request_id: "request-id",+ endpoint: "scrape",+ team_id: "team-id",+ org_id: "org-id",+ api_key: { id: null, name: null },+ started_at: "2026-07-27T00:00:00.000Z",+ completed_at: "2026-07-27T00:00:01.000Z",+ url: "https://example.com",+ domain: "example.com",+ http_method: "GET",+ http_status: 200,+ result: "success",+ error: null,+ origin: "api",+ integration: null,+ zero_data_retention: false,+ };+}++function message(orgId: string, scrapeId: string, attempt = 0) {+ return {+ content: Buffer.from(JSON.stringify({ orgId, event: event(scrapeId) })),+ properties: {+ messageId: scrapeId,+ headers: { "x-siem-attempt": attempt },+ },+ };+}++async function freshTransport() {+ vi.resetModules();+ return import("./transport.js");+}++function assertedQueue(name: string) {+ const call = (channelMock.assertQueue as Mock).mock.calls.find(+ c => c[0] === name,+ );+ return call?.[1] as { arguments?: Record<string, unknown> } | undefined;+}++describe("SIEM logging transport", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ connectMock.mockReturnValue(connectionMock);+ connectionMock.createChannel.mockResolvedValue(channelMock);+ connectionMock.createConfirmChannel.mockResolvedValue(channelMock);+ });++ it("bounds the event queue in the broker instead of in memory", async () => {+ const transport = await freshTransport();+ await transport.enqueueSiemLoggingEvent("org-a", event("scrape-1"));++ const main = assertedQueue("siem.logging.events");+ expect(main?.arguments).toMatchObject({+ "x-max-length": 100_000,+ "x-overflow": "drop-head",+ "x-dead-letter-exchange": "siem.logging.dlx",+ });+ // Every retry rung is bounded too, so a wedged destination can't turn the+ // ladder into unbounded storage.+ for (const delayMs of [5_000, 30_000, 300_000, 1_800_000]) {+ expect(+ assertedQueue(`siem.logging.events.retry.${delayMs}`)?.arguments,+ ).toMatchObject({+ "x-message-ttl": delayMs,+ "x-max-length": 100_000,+ "x-overflow": "drop-head",+ "x-dead-letter-exchange": "siem.logging",+ });+ }+ });++ it("batches an organization's events into one handler call on the linger", async () => {+ vi.useFakeTimers();+ try {+ const transport = await freshTransport();+ const handler = vi.fn().mockResolvedValue({ disposition: "done" });+ await transport.consumeSiemLoggingEvents(handler, {+ batchSize: 10,+ batchLingerMs: 1000,+ });+ const onMessage = channelMock.consume.mock.calls[0][1];++ const first = message("org-a", "scrape-1");+ const second = message("org-a", "scrape-2");+ await onMessage(first);+ await onMessage(second);+ expect(handler).not.toHaveBeenCalled();++ await vi.advanceTimersByTimeAsync(1000);++ expect(handler).toHaveBeenCalledOnce();+ const [orgId, events] = handler.mock.calls[0];+ expect(orgId).toBe("org-a");+ expect(events.map((e: ScrapeActivityEvent) => e.scrape_id)).toEqual([+ "scrape-1",+ "scrape-2",+ ]);+ expect(channelMock.ack).toHaveBeenCalledWith(first);+ expect(channelMock.ack).toHaveBeenCalledWith(second);+ } finally {+ vi.useRealTimers();+ }+ });++ it("never mixes organizations in one batch", async () => {+ vi.useFakeTimers();+ try {+ const transport = await freshTransport();+ const handler = vi.fn().mockResolvedValue({ disposition: "done" });+ await transport.consumeSiemLoggingEvents(handler, {+ batchSize: 10,+ batchLingerMs: 1000,+ });+ const onMessage = channelMock.consume.mock.calls[0][1];++ await onMessage(message("org-a", "scrape-1"));+ await onMessage(message("org-b", "scrape-2"));+ await vi.advanceTimersByTimeAsync(1000);++ expect(handler).toHaveBeenCalledTimes(2);+ expect(handler.mock.calls.map(c => c[0]).sort()).toEqual([+ "org-a",+ "org-b",+ ]);+ for (const [, events] of handler.mock.calls) {+ expect(events).toHaveLength(1);+ }+ } finally {+ vi.useRealTimers();+ }+ });++ it("flushes early once a batch reaches its size ceiling", async () => {+ vi.useFakeTimers();+ try {+ const transport = await freshTransport();+ const handler = vi.fn().mockResolvedValue({ disposition: "done" });+ await transport.consumeSiemLoggingEvents(handler, {+ batchSize: 2,+ batchLingerMs: 60_000,+ });+ const onMessage = channelMock.consume.mock.calls[0][1];++ await onMessage(message("org-a", "scrape-1"));+ await onMessage(message("org-a", "scrape-2"));++ // No timer advance: the size ceiling closed the batch.+ expect(handler).toHaveBeenCalledOnce();+ expect(handler.mock.calls[0][1]).toHaveLength(2);+ } finally {+ vi.useRealTimers();+ }+ });++ it("leaves prefetch large enough to fill a batch", async () => {+ const transport = await freshTransport();+ await transport.consumeSiemLoggingEvents(vi.fn(), { batchSize: 50 });++ const [prefetch, global] = channelMock.prefetch.mock.calls[0];+ expect(prefetch).toBeGreaterThanOrEqual(50);+ expect(global).toBe(false);+ });++ it("republishes a retry onto the next ladder rung and acks the original", async () => {+ const transport = await freshTransport();+ await transport.consumeSiemLoggingEvents(+ vi.fn().mockResolvedValue({ disposition: "retry" }),+ { batchSize: 1 },+ );+ const onMessage = channelMock.consume.mock.calls[0][1];++ const msg = message("org-a", "scrape-1", 1);+ await onMessage(msg);++ const retryPublish = channelMock.publish.mock.calls.find(+ c => c[0] === "siem.logging.retry",+ );+ expect(retryPublish?.[1]).toBe("siem.logging.events.retry.30000");+ expect(retryPublish?.[3]).toMatchObject({+ headers: { "x-siem-attempt": 2 },+ });+ // Acked, not nacked: a requeue would redeliver immediately and spin against+ // a destination that is still down.+ expect(channelMock.ack).toHaveBeenCalledWith(msg);+ expect(channelMock.nack).not.toHaveBeenCalled();+ });++ it("rounds a Retry-After up to the smallest rung that satisfies it", async () => {+ const transport = await freshTransport();+ await transport.consumeSiemLoggingEvents(+ vi.fn().mockResolvedValue({ disposition: "retry", retryAfterMs: 20_000 }),+ { batchSize: 1 },+ );+ const onMessage = channelMock.consume.mock.calls[0][1];++ await onMessage(message("org-a", "scrape-1"));++ const retryPublish = channelMock.publish.mock.calls.find(+ c => c[0] === "siem.logging.retry",+ );+ expect(retryPublish?.[1]).toBe("siem.logging.events.retry.30000");+ });++ it("dead-letters once the ladder is exhausted", async () => {+ const transport = await freshTransport();+ await transport.consumeSiemLoggingEvents(+ vi.fn().mockResolvedValue({ disposition: "retry" }),+ { batchSize: 1 },+ );+ const onMessage = channelMock.consume.mock.calls[0][1];++ const msg = message("org-a", "scrape-1", 4);+ await onMessage(msg);++ expect(+ channelMock.publish.mock.calls.some(c => c[0] === "siem.logging.retry"),+ ).toBe(false);+ expect(channelMock.nack).toHaveBeenCalledWith(msg, false, false);+ });++ it("dead-letters a permanent failure without touching the ladder", async () => {+ const transport = await freshTransport();+ await transport.consumeSiemLoggingEvents(+ vi.fn().mockResolvedValue({ disposition: "drop" }),+ { batchSize: 1 },+ );+ const onMessage = channelMock.consume.mock.calls[0][1];++ const msg = message("org-a", "scrape-1");+ await onMessage(msg);++ expect(+ channelMock.publish.mock.calls.some(c => c[0] === "siem.logging.retry"),+ ).toBe(false);+ expect(channelMock.nack).toHaveBeenCalledWith(msg, false, false);+ });++ it("retries the batch when the handler throws", async () => {+ const transport = await freshTransport();+ await transport.consumeSiemLoggingEvents(+ vi.fn().mockRejectedValue(new Error("handler boom")),+ { batchSize: 1 },+ );+ const onMessage = channelMock.consume.mock.calls[0][1];++ await onMessage(message("org-a", "scrape-1"));++ expect(+ channelMock.publish.mock.calls.some(c => c[0] === "siem.logging.retry"),+ ).toBe(true);+ });++ it("fails the enqueue when the broker nacks the publish", async () => {+ const transport = await freshTransport();+ channelMock.publish.mockImplementationOnce((...args: unknown[]) => {+ (args[4] as (error: Error | null) => void)(new Error("nacked"));+ return true;+ });++ await expect(+ transport.enqueueSiemLoggingEvent("org-a", event("scrape-1")),+ ).rejects.toThrow("Failed to publish 1 of 1");+ expect(metricsMock.inc).toHaveBeenCalledWith(+ { result: "enqueue_failed" },+ 1,+ );+ });++ it("leaves a message unsettled when its retry republish is not confirmed", async () => {+ const transport = await freshTransport();+ await transport.consumeSiemLoggingEvents(+ vi.fn().mockResolvedValue({ disposition: "retry" }),+ { batchSize: 1 },+ );+ const onMessage = channelMock.consume.mock.calls[0][1];+ channelMock.publish.mockImplementationOnce((...args: unknown[]) => {+ (args[4] as (error: Error | null) => void)(new Error("nacked"));+ return true;+ });++ const msg = message("org-a", "scrape-1");+ await onMessage(msg);++ // Neither acked nor dead-lettered: the broker redelivers it instead of the+ // event vanishing between a lost republish and an ack.+ expect(channelMock.ack).not.toHaveBeenCalled();+ expect(channelMock.nack).not.toHaveBeenCalled();+ });++ it("dead-letters an unparseable message instead of hot-looping it", async () => {+ const transport = await freshTransport();+ await transport.consumeSiemLoggingEvents(vi.fn(), { batchSize: 1 });+ const onMessage = channelMock.consume.mock.calls[0][1];++ const msg = { content: Buffer.from("not json"), properties: {} };+ await onMessage(msg);++ expect(channelMock.nack).toHaveBeenCalledWith(msg, false, false);+ });++ it("dead-letters well-formed JSON that is not a SIEM message", async () => {+ const transport = await freshTransport();+ const handler = vi.fn();+ await transport.consumeSiemLoggingEvents(handler, { batchSize: 1 });+ const onMessage = channelMock.consume.mock.calls[0][1];++ const msg = {+ content: Buffer.from(JSON.stringify({ event: event("scrape-1") })),+ properties: {},+ };+ await onMessage(msg);++ expect(handler).not.toHaveBeenCalled();+ expect(channelMock.nack).toHaveBeenCalledWith(msg, false, false);+ });++ it("counts the accepted events when only some publishes are nacked", async () => {+ const transport = await freshTransport();+ channelMock.publish.mockImplementationOnce((...args: unknown[]) => {+ (args[4] as (error: Error | null) => void)(new Error("nacked"));+ return true;+ });++ // Rejects, but the sibling event was accepted β the aggregate must not+ // report the whole call as failed.+ await expect(+ transport.enqueueSiemLoggingEvents("org-a", [+ event("scrape-1"),+ event("scrape-2"),+ ]),+ ).rejects.toThrow("Failed to publish 1 of 2");+ expect(metricsMock.inc).toHaveBeenCalledWith({ result: "queued" }, 1);+ expect(metricsMock.inc).toHaveBeenCalledWith(+ { result: "enqueue_failed" },+ 1,+ );+ });++ it("publishes as mandatory and reports what the broker cannot route", async () => {+ const transport = await freshTransport();+ await transport.enqueueSiemLoggingEvent("org-a", event("scrape-1"));++ expect(channelMock.publish.mock.calls[0][3]).toMatchObject({+ mandatory: true,+ });++ const onReturn = (channelMock.on as Mock).mock.calls.find(+ c => c[0] === "return",+ )?.[1] as (msg: unknown) => void;+ expect(onReturn).toBeTypeOf("function");+ onReturn({+ fields: { exchange: "siem.logging", routingKey: "siem.logging.events" },+ properties: { messageId: "scrape-1" },+ });+ expect(metricsMock.inc).toHaveBeenCalledWith({ result: "unroutable" });+ });++ it("retries the subscribe when the broker is down at startup", async () => {+ vi.useFakeTimers();+ try {+ const transport = await freshTransport();+ connectMock.mockImplementationOnce(() => {+ throw new Error("broker down");+ });++ // Must resolve, not reject: an unavailable broker at boot has to enter the+ // reconnect loop instead of leaving the replica permanently deaf.+ await expect(+ transport.consumeSiemLoggingEvents(vi.fn(), { batchSize: 1 }),+ ).resolves.toBeUndefined();+ expect(channelMock.consume).not.toHaveBeenCalled();++ await vi.advanceTimersByTimeAsync(1500);++ expect(channelMock.consume).toHaveBeenCalledTimes(1);+ } finally {+ vi.useRealTimers();+ }+ });++ it("does not subscribe when shutdown races the initial connection", async () => {+ const transport = await freshTransport();+ let releaseChannel!: () => void;+ connectionMock.createChannel.mockImplementationOnce(+ () =>+ new Promise(resolve => {+ releaseChannel = () => resolve(channelMock);+ }),+ );++ const subscribing = transport.consumeSiemLoggingEvents(vi.fn(), {+ batchSize: 1,+ });+ await vi.waitFor(() =>+ expect(connectionMock.createChannel).toHaveBeenCalled(),+ );++ // Shutdown lands while that first channel is still opening.+ await transport.closeSiemLoggingTransport();+ releaseChannel();+ await subscribing;++ expect(channelMock.consume).not.toHaveBeenCalled();+ expect(connectionMock.close).toHaveBeenCalled();+ });++ it("does not reopen the transport when shutdown races a pending reconnect", async () => {+ vi.useFakeTimers();+ try {+ const transport = await freshTransport();+ connectMock.mockImplementationOnce(() => {+ throw new Error("broker down");+ });+ await transport.consumeSiemLoggingEvents(vi.fn(), { batchSize: 1 });++ await transport.closeSiemLoggingTransport();+ await vi.advanceTimersByTimeAsync(5000);++ // The pending reconnect timer must not resurrect a connection that+ // shutdown already tore down.+ expect(channelMock.consume).not.toHaveBeenCalled();+ } finally {+ vi.useRealTimers();+ }+ });++ it("re-subscribes the consumer after a channel drop (does not go deaf)", async () => {+ vi.useFakeTimers();+ try {+ const transport = await freshTransport();+ await transport.consumeSiemLoggingEvents(vi.fn(), { batchSize: 1 });+ expect(channelMock.consume).toHaveBeenCalledTimes(1);++ const onClose = (channelMock.on as Mock).mock.calls.find(+ c => c[0] === "close",+ )?.[1] as () => void;+ expect(onClose).toBeTypeOf("function");+ onClose();++ await vi.advanceTimersByTimeAsync(1500);++ expect(channelMock.consume.mock.calls.length).toBeGreaterThanOrEqual(2);+ } finally {+ vi.useRealTimers();+ }+ });+});diff --git a/apps/api/src/lib/siem-logging/transport.ts b/apps/api/src/lib/siem-logging/transport.tsnew file mode 100644index 000000000..2fa0f657d--- /dev/null+++ b/apps/api/src/lib/siem-logging/transport.ts@@ -0,0 +1,678 @@+// RabbitMQ transport for SIEM audit delivery.+//+// Why RabbitMQ and not the BullMQ queues next door: this queue turns one message+// per scrape into an outbound HTTPS call, so it needs two things Redis/BullMQ+// cannot express. First a hard bound β x-max-length with drop-head caps the+// backlog in the broker, so a destination outage can never grow it without limit+// and starve the workers. Second real batching β prefetch plus manual ack lets a+// consumer hold many messages, group them by organization, and deliver one gzip+// batch per org instead of one authenticated POST per scrape.+//+// Retries ride a fixed-delay ladder (x-message-ttl queues that dead-letter back+// onto the main queue) because the cluster does not load+// rabbitmq_delayed_message_exchange. Fixed per-queue TTLs also avoid the+// head-of-line blocking that per-message TTLs would introduce.++import amqp from "amqplib";+import { config } from "../../config";+import { logger as _logger } from "../logger";+import { siemLoggingEventsTotal } from "./metrics";+import type { ScrapeActivityEvent, SiemLoggingMessage } from "./types";++const EXCHANGE = "siem.logging";+const QUEUE = "siem.logging.events";+const RETRY_EXCHANGE = "siem.logging.retry";+const DLX = "siem.logging.dlx";+const DLQ = "siem.logging.dlq";++// ~36 minutes of destination downtime before an event dead-letters. The sender+// already makes three in-process attempts per call, so these rungs cover+// outages, not blips.+const RETRY_DELAYS_MS = [5_000, 30_000, 300_000, 1_800_000] as const;++// Hard backlog bound, enforced by the broker. An outage costs the oldest slice+// of the audit trail instead of the workers' memory; queue depth and the drop+// counters are scraped from rabbitmq_prometheus for alerting.+const MAX_QUEUED_MESSAGES = 100_000;++// The dead-letter queue holds what an operator needs to diagnose, so it gets a+// smaller cap and a retention window rather than growing forever.+const MAX_DEAD_LETTERED_MESSAGES = 50_000;+const DLQ_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;++// A batch closes on whichever comes first. The linger is what turns per-scrape+// messages into per-org batches; keep it short enough that a quiet org still+// sees its audit events land promptly.+const DEFAULT_BATCH_SIZE = 200;+const DEFAULT_BATCH_LINGER_MS = 1_000;++const logger = _logger.child({ module: "siem-logging-transport" });++export type SiemLoggingBatchOutcome = {+ /** "done" acks the batch, "retry" moves it to the next rung, "drop" dead-letters it. */+ disposition: "done" | "retry" | "drop";+ /** Earliest acceptable retry delay in ms β honors a destination's Retry-After. */+ retryAfterMs?: number;+};++/**+ * Deliver one organization's events. The batch shares a single fate because it+ * maps to one authenticated call against that org's destination.+ */+export type SiemLoggingBatchHandler = (+ orgId: string,+ events: ScrapeActivityEvent[],+) => Promise<SiemLoggingBatchOutcome>;++export interface SiemLoggingConsumerOptions {+ /** Unacked message budget; also the ceiling on how large a batch can grow. */+ prefetch?: number;+ batchSize?: number;+ batchLingerMs?: number;+}++let connection: amqp.ChannelModel | null = null;+// Memoized so concurrent callers share one connect() instead of racing to open+// (and orphan) duplicate connections.+let connectionPromise: Promise<amqp.ChannelModel> | null = null;+// Separate publish/consume channels so an exception on one can't tear down the other.+// Publishing goes through a confirm channel: emission is already fire-and-forget+// off the scrape path, so waiting for the broker to acknowledge each event costs+// the caller nothing and is the difference between a complete audit record and+// one with silent holes in it.+let publishChannel: amqp.ConfirmChannel | null = null;+let consumeChannel: amqp.Channel | null = null;+let publishChannelPromise: Promise<amqp.ConfirmChannel> | null = null;+let consumeChannelPromise: Promise<amqp.Channel> | null = null;++// Registered so a reconnect can re-attach the consumer; index-worker replicas+// never publish, so without this they go permanently deaf after a broker blip.+let registeredHandler: SiemLoggingBatchHandler | null = null;+let registeredOptions: SiemLoggingConsumerOptions = {};+let subscribed = false;+let reconnectInFlight = false;+let reconnectTimer: NodeJS.Timeout | null = null;+// Set by close() so a reconnect already in flight can't resurrect the transport+// after shutdown.+let closed = false;+const RECONNECT_BASE_DELAY_MS = 1000;+const RECONNECT_MAX_DELAY_MS = 30_000;++// Ceiling on publishes awaiting a broker confirm. Emission is fire-and-forget,+// so nothing upstream throttles it: without this, a broker that accepts+// connections but stops confirming would let unconfirmed events pile up in the+// producer until the pod dies β the exact failure the queue bound exists to+// prevent, just moved into the API process.+const MAX_IN_FLIGHT_CONFIRMS = 10_000;+let inFlightConfirms = 0;++function retryQueueName(delayMs: number): string {+ return `${QUEUE}.retry.${delayMs}`;+}++async function assertTopology(ch: amqp.Channel): Promise<void> {+ await ch.assertExchange(EXCHANGE, "direct", { durable: true });+ await ch.assertExchange(RETRY_EXCHANGE, "direct", { durable: true });+ await ch.assertExchange(DLX, "direct", { durable: true });++ // Bounded like everything else: a standing bad credential or rejected schema+ // pushes every event here, and an unbounded durable queue would eventually+ // fill the broker's disk. Retention is for an operator to read, not to replay+ // indefinitely.+ await ch.assertQueue(DLQ, {+ durable: true,+ arguments: {+ "x-max-length": MAX_DEAD_LETTERED_MESSAGES,+ "x-overflow": "drop-head",+ "x-message-ttl": DLQ_RETENTION_MS,+ },+ });+ await ch.bindQueue(DLQ, DLX, QUEUE);++ // Classic, not quorum like the monitoring queues: x-max-length with drop-head+ // is the bound this queue exists to have, and classic queues support that+ // combination on every broker version we might be running. The cost is that a+ // node loss drops this queue's backlog β acceptable for a buffer that already+ // sheds its oldest events at the cap, and cheaper than a declaration that+ // fails with PRECONDITION_FAILED and takes all delivery down with it.+ await ch.assertQueue(QUEUE, {+ durable: true,+ arguments: {+ "x-max-length": MAX_QUEUED_MESSAGES,+ "x-overflow": "drop-head",+ "x-dead-letter-exchange": DLX,+ "x-dead-letter-routing-key": QUEUE,+ },+ });+ await ch.bindQueue(QUEUE, EXCHANGE, QUEUE);++ // Each rung parks messages for a fixed TTL, then dead-letters them back onto+ // the main queue. The same bound applies, so a wedged destination can't turn+ // the retry ladder into unbounded storage either.+ for (const delayMs of RETRY_DELAYS_MS) {+ const queue = retryQueueName(delayMs);+ await ch.assertQueue(queue, {+ durable: true,+ arguments: {+ "x-message-ttl": delayMs,+ "x-max-length": MAX_QUEUED_MESSAGES,+ "x-overflow": "drop-head",+ "x-dead-letter-exchange": EXCHANGE,+ "x-dead-letter-routing-key": QUEUE,+ },+ });+ await ch.bindQueue(queue, RETRY_EXCHANGE, queue);+ }+}++function resetCachedState(): void {+ connection = null;+ connectionPromise = null;+ publishChannel = null;+ publishChannelPromise = null;+ consumeChannel = null;+ consumeChannelPromise = null;+ subscribed = false;+ abandonPendingBatches();+}++function handleConnectionDrop(reason: string, error?: unknown): void {+ if (connection || publishChannel || consumeChannel) {+ logger.warn("SIEM logging transport dropped β will reconnect", {+ reason,+ error,+ });+ }+ resetCachedState();+ if (registeredHandler) scheduleReconnect(RECONNECT_BASE_DELAY_MS);+}++function scheduleReconnect(delayMs: number): void {+ if (closed || reconnectInFlight) return;+ reconnectInFlight = true;++ reconnectTimer = setTimeout(async () => {+ reconnectTimer = null;+ try {+ const ch = await getConsumeChannel();+ // Shutdown can land while the channel was opening. Without this the+ // connection we just opened outlives close() with nothing consuming it.+ if (closed) {+ reconnectInFlight = false;+ await closeConnection();+ return;+ }+ if (registeredHandler) {+ await subscribe(ch, registeredHandler, registeredOptions);+ }+ reconnectInFlight = false;+ logger.info("SIEM logging transport reconnected; consumer re-subscribed");+ } catch (error) {+ reconnectInFlight = false;+ const nextDelay = Math.min(delayMs * 2, RECONNECT_MAX_DELAY_MS);+ logger.error("SIEM logging transport reconnect failed; retrying", {+ error,+ nextDelayMs: nextDelay,+ });+ // A mid-subscribe failure on a healthy channel never fires "close", so+ // reset everything or the next attempt short-circuits and stays deaf.+ await consumeChannel?.close().catch(() => {});+ resetCachedState();+ scheduleReconnect(nextDelay);+ }+ }, delayMs);+ reconnectTimer.unref?.();+}++async function openConnection(): Promise<amqp.ChannelModel> {+ const url = config.NUQ_RABBITMQ_URL;+ if (!url) throw new Error("NUQ_RABBITMQ_URL is not configured");++ const conn = await amqp.connect(url, {+ clientProperties: { connection_name: "siem-logging" },+ });+ conn.on("close", () => handleConnectionDrop("connection closed"));+ conn.on("error", error =>+ logger.error("SIEM logging transport connection error", { error }),+ );+ return conn;+}++async function getConnection(): Promise<amqp.ChannelModel> {+ if (connection) return connection;+ if (!connectionPromise) {+ connectionPromise = openConnection()+ .then(conn => {+ connection = conn;+ return conn;+ })+ .catch(error => {+ connectionPromise = null;+ throw error;+ });+ }+ return connectionPromise;+}++async function createChannel(label: "publish"): Promise<amqp.ConfirmChannel>;+async function createChannel(label: "consume"): Promise<amqp.Channel>;+async function createChannel(+ label: "publish" | "consume",+): Promise<amqp.Channel | amqp.ConfirmChannel> {+ const conn = await getConnection();+ const ch =+ label === "publish"+ ? await conn.createConfirmChannel()+ : await conn.createChannel();+ await assertTopology(ch);+ // Channels can close independently of the connection (e.g. PRECONDITION_FAILED);+ // without this the cached channel stays non-null and every call throws forever.+ ch.on("close", () => {+ if (label === "publish" && publishChannel === ch) {+ publishChannel = null;+ publishChannelPromise = null;+ }+ if (label === "consume" && consumeChannel === ch) {+ consumeChannel = null;+ consumeChannelPromise = null;+ subscribed = false;+ handleConnectionDrop("consume channel closed");+ }+ });+ ch.on("error", error =>+ logger.error("SIEM logging transport channel error", { label, error }),+ );+ // Only reachable if the bindings asserted above are removed out of band, but+ // the failure mode is a silently discarded audit event, so make it loud.+ if (label === "publish") {+ ch.on("return", msg => {+ siemLoggingEventsTotal.inc({ result: "unroutable" });+ logger.error("Broker could not route a SIEM logging event", {+ exchange: msg.fields.exchange,+ routingKey: msg.fields.routingKey,+ messageId: msg.properties.messageId,+ });+ });+ }+ return ch;+}++async function getPublishChannel(): Promise<amqp.ConfirmChannel> {+ if (publishChannel) return publishChannel;+ if (!publishChannelPromise) {+ publishChannelPromise = createChannel("publish")+ .then(ch => {+ publishChannel = ch;+ return ch;+ })+ .catch(error => {+ publishChannelPromise = null;+ throw error;+ });+ }+ return publishChannelPromise;+}++async function getConsumeChannel(): Promise<amqp.Channel> {+ if (consumeChannel) return consumeChannel;+ if (!consumeChannelPromise) {+ consumeChannelPromise = createChannel("consume")+ .then(ch => {+ consumeChannel = ch;+ return ch;+ })+ .catch(error => {+ consumeChannelPromise = null;+ throw error;+ });+ }+ return consumeChannelPromise;+}++// Resolves once the broker has taken responsibility for the message, and rejects+// if it nacks instead. A confirm says nothing about routing β an unroutable+// publish is still confirmed β so publishes are `mandatory` and the channel's+// "return" handler counts anything the exchange couldn't route.+function publishConfirmed(+ ch: amqp.ConfirmChannel,+ exchange: string,+ routingKey: string,+ content: Buffer,+ options: amqp.Options.Publish,+): Promise<void> {+ if (inFlightConfirms >= MAX_IN_FLIGHT_CONFIRMS) {+ siemLoggingEventsTotal.inc({ result: "dropped_producer_full" });+ return Promise.reject(+ new Error(+ `Refusing to publish: ${inFlightConfirms} SIEM events are already awaiting a broker confirm`,+ ),+ );+ }++ return new Promise((resolve, reject) => {+ inFlightConfirms++;+ const settle = (error: unknown): void => {+ inFlightConfirms--;+ if (error) reject(error);+ else resolve();+ };+ const sent = ch.publish(+ exchange,+ routingKey,+ content,+ { ...options, mandatory: true },+ settle,+ );+ // A false return only means the socket write buffer is full; the message is+ // still queued client-side and the confirm callback still fires. The+ // in-flight ceiling above is what actually bounds producer memory.+ if (!sent) siemLoggingEventsTotal.inc({ result: "publish_backpressure" });+ });+}++export async function enqueueSiemLoggingEvent(+ orgId: string,+ event: ScrapeActivityEvent,+): Promise<void> {+ await enqueueSiemLoggingEvents(orgId, [event]);+}++export async function enqueueSiemLoggingEvents(+ orgId: string,+ events: ScrapeActivityEvent[],+): Promise<void> {+ if (events.length === 0) return;++ let ch: amqp.ConfirmChannel;+ try {+ ch = await getPublishChannel();+ } catch (error) {+ // No channel means nothing was published, so the whole call failed.+ siemLoggingEventsTotal.inc({ result: "enqueue_failed" }, events.length);+ throw error;+ }++ // allSettled, not all: these events are independent, and a single rejected+ // confirm must not mislabel the ones the broker did accept. There is nowhere+ // durable to park a rejected event β Firecrawl holds no scrape-derived state β+ // so a failure here is a counted, logged loss rather than a retry.+ const results = await Promise.allSettled(+ events.map(event =>+ publishConfirmed(+ ch,+ EXCHANGE,+ QUEUE,+ Buffer.from(+ JSON.stringify({ orgId, event } satisfies SiemLoggingMessage),+ ),+ {+ persistent: true,+ contentType: "application/json",+ messageId: event.scrape_id,+ headers: { "x-siem-attempt": 0 },+ },+ ),+ ),+ );++ const rejected = results.filter(+ (result): result is PromiseRejectedResult => result.status === "rejected",+ );+ const queued = results.length - rejected.length;+ if (queued > 0) siemLoggingEventsTotal.inc({ result: "queued" }, queued);+ if (rejected.length > 0) {+ siemLoggingEventsTotal.inc({ result: "enqueue_failed" }, rejected.length);+ throw new AggregateError(+ rejected.map(result => result.reason),+ `Failed to publish ${rejected.length} of ${results.length} SIEM logging events`,+ );+ }+}++/** Smallest ladder rung that is at least `minDelayMs`, else the largest rung. */+function rungForDelay(minDelayMs: number | undefined): number {+ if (minDelayMs === undefined || minDelayMs <= 0) return RETRY_DELAYS_MS[0];+ return (+ RETRY_DELAYS_MS.find(delay => delay >= minDelayMs) ??+ RETRY_DELAYS_MS[RETRY_DELAYS_MS.length - 1]+ );+}++function attemptOf(msg: amqp.ConsumeMessage): number {+ const raw = msg.properties.headers?.["x-siem-attempt"];+ const attempt = typeof raw === "number" ? raw : Number(raw ?? 0);+ return Number.isFinite(attempt) && attempt > 0 ? Math.floor(attempt) : 0;+}++// Republish onto a delay rung, then ack the original: a plain nack-with-requeue+// redelivers immediately and would spin against a destination that is still down.+// The republish is confirmed before the caller acks, so the window where the+// original is gone and the retry never landed stays closed.+async function scheduleRetry(+ msg: amqp.ConsumeMessage,+ retryAfterMs: number | undefined,+): Promise<boolean> {+ const attempt = attemptOf(msg);+ if (attempt >= RETRY_DELAYS_MS.length) return false;+ // Never step backwards down the ladder on a repeat failure.+ const delayMs = Math.max(+ rungForDelay(retryAfterMs),+ RETRY_DELAYS_MS[attempt],+ );+ const ch = await getPublishChannel();+ await publishConfirmed(+ ch,+ RETRY_EXCHANGE,+ retryQueueName(delayMs),+ msg.content,+ {+ persistent: true,+ contentType: "application/json",+ messageId: msg.properties.messageId,+ headers: { "x-siem-attempt": attempt + 1 },+ },+ );+ return true;+}++interface PendingBatch {+ messages: amqp.ConsumeMessage[];+ events: ScrapeActivityEvent[];+ timer: NodeJS.Timeout;+}++// Valid JSON is not a valid message: without this, a body missing orgId would+// open a batch keyed on undefined and send the handler looking up a destination+// for an organization that doesn't exist.+function isSiemLoggingMessage(value: unknown): value is SiemLoggingMessage {+ if (typeof value !== "object" || value === null) return false;+ const candidate = value as Partial<SiemLoggingMessage>;+ return (+ typeof candidate.orgId === "string" &&+ candidate.orgId.length > 0 &&+ typeof candidate.event === "object" &&+ candidate.event !== null+ );+}++// One open batch per organization: each org has its own destination, credentials+// and token, so batches can never span orgs.+const pendingBatches = new Map<string, PendingBatch>();++function abandonPendingBatches(): void {+ for (const batch of pendingBatches.values()) clearTimeout(batch.timer);+ // Left unacked on purpose β the broker redelivers them to a live consumer.+ pendingBatches.clear();+}++async function applyOutcome(+ ch: amqp.Channel,+ batch: PendingBatch,+ outcome: SiemLoggingBatchOutcome,+): Promise<void> {+ if (outcome.disposition === "done") {+ for (const msg of batch.messages) ch.ack(msg);+ return;+ }+ if (outcome.disposition === "drop") {+ for (const msg of batch.messages) ch.nack(msg, false, false);+ siemLoggingEventsTotal.inc(+ { result: "dead_lettered" },+ batch.messages.length,+ );+ return;+ }+ let retried = 0;+ let deadLettered = 0;+ for (const msg of batch.messages) {+ if (await scheduleRetry(msg, outcome.retryAfterMs)) {+ ch.ack(msg);+ retried++;+ } else {+ ch.nack(msg, false, false);+ deadLettered++;+ }+ }+ if (retried > 0) siemLoggingEventsTotal.inc({ result: "retried" }, retried);+ if (deadLettered > 0) {+ siemLoggingEventsTotal.inc({ result: "dead_lettered" }, deadLettered);+ }+}++async function flushBatch(+ ch: amqp.Channel,+ handler: SiemLoggingBatchHandler,+ orgId: string,+): Promise<void> {+ const batch = pendingBatches.get(orgId);+ if (!batch) return;+ pendingBatches.delete(orgId);+ clearTimeout(batch.timer);++ let outcome: SiemLoggingBatchOutcome;+ try {+ outcome = await handler(orgId, batch.events);+ } catch (error) {+ logger.error("SIEM logging batch handler threw", {+ error,+ orgId,+ events: batch.events.length,+ });+ outcome = { disposition: "retry" };+ }++ try {+ await applyOutcome(ch, batch, outcome);+ } catch (error) {+ // The channel died mid-batch, or a retry republish was nacked; either way+ // the broker redelivers everything still unacked.+ logger.warn("Failed to settle a SIEM logging batch", { error, orgId });+ }+}++async function subscribe(+ ch: amqp.Channel,+ handler: SiemLoggingBatchHandler,+ options: SiemLoggingConsumerOptions,+): Promise<void> {+ if (subscribed) return;+ const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;+ const lingerMs = options.batchLingerMs ?? DEFAULT_BATCH_LINGER_MS;+ // A batch can only ever hold messages the broker has handed us, so prefetch+ // has to leave room for a full batch or it caps the batch below batchSize.+ await ch.prefetch(options.prefetch ?? batchSize * 2, false);++ await ch.consume(+ QUEUE,+ async msg => {+ if (!msg) return;++ let message: SiemLoggingMessage;+ try {+ const parsed: unknown = JSON.parse(msg.content.toString());+ if (!isSiemLoggingMessage(parsed)) {+ throw new Error("Message is not a SIEM logging event");+ }+ message = parsed;+ } catch (error) {+ logger.error("Failed to parse SIEM logging message", { error });+ siemLoggingEventsTotal.inc({ result: "unparseable" });+ ch.nack(msg, false, false);+ return;+ }++ const orgId = message.orgId;+ let batch = pendingBatches.get(orgId);+ if (!batch) {+ batch = {+ messages: [],+ events: [],+ timer: setTimeout(() => {+ void flushBatch(ch, handler, orgId);+ }, lingerMs),+ };+ batch.timer.unref?.();+ pendingBatches.set(orgId, batch);+ }+ batch.messages.push(msg);+ batch.events.push(message.event);++ if (batch.events.length >= batchSize) {+ await flushBatch(ch, handler, orgId);+ }+ },+ { noAck: false },+ );++ subscribed = true;+ logger.info("Started consuming SIEM logging events", {+ queue: QUEUE,+ batchSize,+ lingerMs,+ });+}++export async function consumeSiemLoggingEvents(+ handler: SiemLoggingBatchHandler,+ options: SiemLoggingConsumerOptions = {},+): Promise<void> {+ closed = false;+ registeredHandler = handler;+ registeredOptions = options;+ try {+ const ch = await getConsumeChannel();+ // Same race the reconnect path guards: a SIGTERM during a redeploy can land+ // while the initial connection is still opening, and subscribing after that+ // would leave a consumer attached to a transport shutdown already released.+ if (closed) {+ await closeConnection();+ return;+ }+ await subscribe(ch, handler, options);+ } catch (error) {+ // A broker that is unavailable at worker startup has to enter the same retry+ // loop as one that drops later. Throwing here instead left the replica+ // permanently deaf, because only an already-established channel closing+ // scheduled a reconnect.+ logger.error("Failed to subscribe to SIEM logging events; will retry", {+ error,+ });+ scheduleReconnect(RECONNECT_BASE_DELAY_MS);+ }+}++async function closeConnection(): Promise<void> {+ const conn = connection;+ resetCachedState();+ await conn?.close().catch(() => {});+}++export async function closeSiemLoggingTransport(): Promise<void> {+ closed = true;+ registeredHandler = null;+ if (reconnectTimer) {+ clearTimeout(reconnectTimer);+ reconnectTimer = null;+ }+ await closeConnection();+}diff --git a/apps/api/src/lib/siem-logging/types.ts b/apps/api/src/lib/siem-logging/types.tsindex efb2c91ec..688f57e0a 100644--- a/apps/api/src/lib/siem-logging/types.ts+++ b/apps/api/src/lib/siem-logging/types.ts@@ -76,13 +76,17 @@ export type ScrapeActivityResult =export interface ScrapeActivityThreat {decision: "allow" | "deny";+ /**+ * The rule that produced {@link decision}. Local-policy rules (blacklist,+ * blocked-tld) deny without consulting a provider; only "risk-score" with a+ * non-null {@link provider} is a provider-confirmed threat. The destination+ * DCR transform keys ASim EventSeverity off exactly that distinction, so the+ * three fields below always describe one decision β never a union across the+ * decisions taken for one scrape.+ */rule: ThreatDecisionRule;provider: ThreatProvider | null;categories: string[];- security_alert: {- detected: boolean;- category: string | null;- };}export interface ScrapeActivityEvent {@@ -123,7 +127,7 @@ export interface ScrapeActivityEvent {threat?: ScrapeActivityThreat;}-export interface SiemLoggingJobData {+export interface SiemLoggingMessage {orgId: string;event: ScrapeActivityEvent;}@@ -132,10 +136,17 @@ type SiemDeliveryErrorKind =| "invalid_credentials"| "schema_rejection"| "rate_limited"- | "delivery_error"- | "payload_too_large";+ | "delivery_error";export class SiemDeliveryError extends Error {+ /**+ * Events the destination accepted before this failure. A batch is sent as+ * several compressed chunks, so a late chunk failing does not undo the earlier+ * ones β without this the caller would count accepted events as failed and+ * then count them again as delivered when the batch retries.+ */+ public deliveredEvents = 0;+constructor(public readonly kind: SiemDeliveryErrorKind,message: string,diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 5bb2efa9a..5e74be8be 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -9,7 +9,6 @@ import {getRedisConnection,getBillingQueue,getPrecrawlQueue,- getSiemLoggingQueue,precrawlQueueName,} from "../queue-service";import {@@ -43,7 +42,8 @@ import { crawlGroup, resolveNewGroupBackend } from "../worker/nuq-router";import { getACUCTeam } from "../../controllers/auth";import { processEngpickerJob } from "../../lib/engpicker";import { logRequest } from "../logging/log_job";-import { processSiemLoggingJob } from "../siem-logging/worker";+import { startSiemLoggingConsumer } from "../siem-logging/worker";+import { closeSiemLoggingTransport } from "../../lib/siem-logging/transport";const workerLockDuration = config.WORKER_LOCK_DURATION;const workerStalledCheckInterval = config.WORKER_STALLED_CHECK_INTERVAL;@@ -589,15 +589,27 @@ const processPrecrawlJob = async (token: string, job: Job) => {let isShuttingDown = false;+// The SIEM consumer is a subscription rather than a loop, so it has to be torn+// down from the signal path: workerFun exits the process once its own jobs+// drain, so anything after the worker Promise.all never runs. Unacked batches go+// back to the broker for another replica.+function shutdownSiemLoggingTransport(): void {+ closeSiemLoggingTransport().catch(error => {+ logger.warn("Failed to close the SIEM logging transport", { error });+ });+}+if (require.main === module) {process.on("SIGINT", () => {logger.info("Received SIGINT. Shutting down gracefully...");isShuttingDown = true;+ shutdownSiemLoggingTransport();});process.on("SIGTERM", () => {logger.info("Received SIGTERM. Shutting down gracefully...");isShuttingDown = true;+ shutdownSiemLoggingTransport();});}@@ -700,10 +712,11 @@ const BROWSER_ACTIVITY_INSERT_INTERVAL = 10000;: (async () => {logger.warn("PRECRAWL_TEAM_ID not set, skipping precrawl worker");})();- const siemLoggingWorkerPromise = workerFun(- getSiemLoggingQueue(),- processSiemLoggingJob,- );+ // A RabbitMQ consumer registration, not a polling loop: it returns once+ // subscribed and the transport reconnects itself on a broker drop.+ startSiemLoggingConsumer().catch(error => {+ logger.error("Failed to start the SIEM logging consumer", { error });+ });const indexInserterInterval = setInterval(async () => {if (isShuttingDown) {@@ -785,7 +798,6 @@ const BROWSER_ACTIVITY_INSERT_INTERVAL = 10000;await Promise.all([billingWorkerPromise,precrawlWorkerPromise,- siemLoggingWorkerPromise,engpickerPromise,]);diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 1c5bf0ccd..04dc0b263 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -3,7 +3,6 @@ import { config } from "../config";import { logger } from "../lib/logger";import IORedis from "ioredis";import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";-import type { SiemLoggingJobData } from "../lib/siem-logging/types";import { addExtractJob, ExtractJobData } from "./extract-queue";let loggingQueue: Queue;@@ -12,7 +11,6 @@ let deepResearchQueue: Queue;let generateLlmsTxtQueue: Queue;let billingQueue: Queue;let precrawlQueue: Queue;-let siemLoggingQueue: Queue<SiemLoggingJobData>;let redisConnection: IORedis;export function getRedisConnection(): IORedis {@@ -31,7 +29,6 @@ const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";const deepResearchQueueName = "{deepResearchQueue}";const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";-const siemLoggingQueueName = "{siemLoggingQueue}";export async function addExtractJobToQueue(extractId: string,@@ -110,25 +107,3 @@ export function getPrecrawlQueue() {}return precrawlQueue;}--export function getSiemLoggingQueue(): Queue<SiemLoggingJobData> {- if (!siemLoggingQueue) {- siemLoggingQueue = new Queue<SiemLoggingJobData>(siemLoggingQueueName, {- connection: getRedisConnection(),- defaultJobOptions: {- attempts: 8,- backoff: {- type: "exponential",- delay: 1000,- },- removeOnComplete: {- age: 60,- },- removeOnFail: {- age: 24 * 60 * 60,- },- },- });- }- return siemLoggingQueue;-}diff --git a/apps/api/src/services/siem-logging/worker.test.ts b/apps/api/src/services/siem-logging/worker.test.tsindex 4e8fac631..7b8a6b348 100644--- a/apps/api/src/services/siem-logging/worker.test.ts+++ b/apps/api/src/services/siem-logging/worker.test.ts@@ -1,34 +1,47 @@-import type { Job } from "bullmq";-import { describe, expect, it, vi } from "vitest";+import { beforeEach, describe, expect, it, vi } from "vitest";++const { eventsTotal, failuresTotal } = vi.hoisted(() => ({+ eventsTotal: { inc: vi.fn() },+ failuresTotal: { inc: vi.fn() },+}));+vi.mock("../../lib/siem-logging/metrics", () => ({+ siemLoggingEventsTotal: eventsTotal,+ siemLoggingDeliveryFailuresTotal: failuresTotal,+ siemLoggingDeliveryBatchesTotal: { inc: vi.fn() },+}));+import {SiemDeliveryError,type OrgSiemLoggingConfig,type ScrapeActivityEvent,- type SiemLoggingJobData,} from "../../lib/siem-logging/types";-import { processSiemLoggingJob } from "./worker";--const event: ScrapeActivityEvent = {- schema_version: 1,- event_type: "scrape_activity",- scrape_id: "scrape-id",- request_id: "request-id",- endpoint: "scrape",- team_id: "team-id",- org_id: "org-id",- api_key: { id: null, name: null },- started_at: "2026-07-27T00:00:00.000Z",- completed_at: "2026-07-27T00:00:01.000Z",- url: "https://example.com",- domain: "example.com",- http_method: "GET",- http_status: 200,- result: "success",- error: null,- origin: "api",- integration: null,- zero_data_retention: false,-};+import { deliverSiemLoggingBatch } from "./worker";++function event(scrapeId: string): ScrapeActivityEvent {+ return {+ schema_version: 1,+ event_type: "scrape_activity",+ scrape_id: scrapeId,+ request_id: "request-id",+ endpoint: "scrape",+ team_id: "team-id",+ org_id: "org-id",+ api_key: { id: null, name: null },+ started_at: "2026-07-27T00:00:00.000Z",+ completed_at: "2026-07-27T00:00:01.000Z",+ url: "https://example.com",+ domain: "example.com",+ http_method: "GET",+ http_status: 200,+ result: "success",+ error: null,+ origin: "api",+ integration: null,+ zero_data_retention: false,+ };+}++const events = [event("scrape-1"), event("scrape-2")];const config: OrgSiemLoggingConfig = {orgId: "org-id",@@ -46,76 +59,89 @@ const config: OrgSiemLoggingConfig = {updatedAt: null,};-function createJob() {- return {- id: "job-id",- data: { orgId: "org-id", event },- extendLock: vi.fn().mockResolvedValue(1),- moveToCompleted: vi.fn().mockResolvedValue(undefined),- moveToFailed: vi.fn().mockResolvedValue(undefined),- discard: vi.fn(),- attemptsMade: 0,- opts: {- attempts: 8,- backoff: { type: "exponential", delay: 1000 },- },- } as unknown as Job<SiemLoggingJobData>;-}+describe("SIEM logging batch delivery", () => {+ beforeEach(() => {+ eventsTotal.inc.mockClear();+ failuresTotal.inc.mockClear();+ });-describe("SIEM logging worker", () => {- it("loads the current secret at delivery time and completes the job", async () => {- const job = createJob();- const deliver = vi.fn().mockResolvedValue(undefined);+ it("counts only the events a partly-delivered batch actually landed", async () => {+ const error = new SiemDeliveryError("delivery_error", "chunk 2 failed");+ // Azure accepted the first chunk before the call blew up.+ error.deliveredEvents = 1;- await processSiemLoggingJob("token", job, {+ const outcome = await deliverSiemLoggingBatch("org-id", events, {getConfig: vi.fn().mockResolvedValue(config),- deliver,+ deliver: vi.fn().mockRejectedValue(error),});- expect(deliver).toHaveBeenCalledWith(config.destination, [event]);- expect(job.moveToCompleted).toHaveBeenCalledWith(- { delivered: true },- "token",- false,+ expect(outcome).toEqual({+ disposition: "retry",+ retryAfterMs: undefined,+ });+ expect(eventsTotal.inc).toHaveBeenCalledWith({ result: "delivered" }, 1);+ expect(eventsTotal.inc).toHaveBeenCalledWith(+ { result: "delivery_failed" },+ 1,+ );+ expect(failuresTotal.inc).toHaveBeenCalledWith(+ { reason: "delivery_error" },+ 1,);- expect(job.moveToFailed).not.toHaveBeenCalled();});- it("skips queued events after logging is disabled", async () => {- const job = createJob();+ it("loads the current secret at delivery time and sends one batch", async () => {+ const deliver = vi.fn().mockResolvedValue(2);+ const getConfig = vi.fn().mockResolvedValue(config);++ const outcome = await deliverSiemLoggingBatch("org-id", events, {+ getConfig,+ deliver,+ });++ expect(getConfig).toHaveBeenCalledWith("org-id");+ expect(deliver).toHaveBeenCalledOnce();+ expect(deliver).toHaveBeenCalledWith(config.destination, events);+ expect(outcome).toEqual({ disposition: "done" });+ });++ it("acks queued events after logging is disabled", async () => {const deliver = vi.fn();- await processSiemLoggingJob("token", job, {+ const outcome = await deliverSiemLoggingBatch("org-id", events, {getConfig: vi.fn().mockResolvedValue({ ...config, enabled: false }),deliver,});expect(deliver).not.toHaveBeenCalled();- expect(job.moveToCompleted).toHaveBeenCalledWith(- { skipped: true },- "token",- false,- );+ expect(outcome).toEqual({ disposition: "done" });});- it("leaves transient delivery failures retryable", async () => {- const job = createJob();+ it("retries when the configuration lookup fails", async () => {+ const outcome = await deliverSiemLoggingBatch("org-id", events, {+ getConfig: vi.fn().mockRejectedValue(new Error("db down")),+ deliver: vi.fn(),+ });++ expect(outcome).toEqual({ disposition: "retry" });+ });- await processSiemLoggingJob("token", job, {+ it("retries transient delivery failures", async () => {+ const outcome = await deliverSiemLoggingBatch("org-id", events, {getConfig: vi.fn().mockResolvedValue(config),deliver: vi.fn()- .mockRejectedValue(new SiemDeliveryError("rate_limited", "retry")),+ .mockRejectedValue(new SiemDeliveryError("delivery_error", "boom")),});- expect(job.discard).not.toHaveBeenCalled();- expect(job.moveToFailed).toHaveBeenCalledOnce();+ expect(outcome).toEqual({+ disposition: "retry",+ retryAfterMs: undefined,+ });});- it("does not retry before the provider's Retry-After delay", async () => {- const job = createJob();-- await processSiemLoggingJob("token", job, {+ it("surfaces the provider's Retry-After to the retry ladder", async () => {+ const outcome = await deliverSiemLoggingBatch("org-id", events, {getConfig: vi.fn().mockResolvedValue(config),deliver: vi.fn()@@ -124,23 +150,17 @@ describe("SIEM logging worker", () => {),});- expect(job.opts.backoff).toEqual({ type: "fixed", delay: 15_000 });- expect(job.moveToFailed).toHaveBeenCalledOnce();+ expect(outcome).toEqual({ disposition: "retry", retryAfterMs: 15_000 });});- it("discards permanent destination failures", async () => {- const job = createJob();-- await processSiemLoggingJob("token", job, {- getConfig: vi.fn().mockResolvedValue(config),- deliver: vi- .fn()- .mockRejectedValue(- new SiemDeliveryError("invalid_credentials", "invalid"),- ),- });+ it("dead-letters permanent destination failures", async () => {+ for (const kind of ["invalid_credentials", "schema_rejection"] as const) {+ const outcome = await deliverSiemLoggingBatch("org-id", events, {+ getConfig: vi.fn().mockResolvedValue(config),+ deliver: vi.fn().mockRejectedValue(new SiemDeliveryError(kind, kind)),+ });- expect(job.discard).toHaveBeenCalledOnce();- expect(job.moveToFailed).toHaveBeenCalledOnce();+ expect(outcome).toEqual({ disposition: "drop" });+ }});});diff --git a/apps/api/src/services/siem-logging/worker.ts b/apps/api/src/services/siem-logging/worker.tsindex c3075779e..3a8edca5c 100644--- a/apps/api/src/services/siem-logging/worker.ts+++ b/apps/api/src/services/siem-logging/worker.ts@@ -1,29 +1,34 @@-import type { Job } from "bullmq";-import { config } from "../../config";import { sendAzureSentinelEvents } from "../../lib/siem-logging/azure-sentinel";import {siemLoggingDeliveryFailuresTotal,siemLoggingEventsTotal,} from "../../lib/siem-logging/metrics";import { getOrgSiemLoggingConfig } from "../../lib/siem-logging/store";+import {+ consumeSiemLoggingEvents,+ type SiemLoggingBatchOutcome,+} from "../../lib/siem-logging/transport";import {SiemDeliveryError,type AzureSentinelDestination,type OrgSiemLoggingConfig,type ScrapeActivityEvent,- type SiemLoggingJobData,} from "../../lib/siem-logging/types";import { logger as _logger } from "../../lib/logger";-interface WorkerDependencies {+interface HandlerDependencies {getConfig: (orgId: string) => Promise<OrgSiemLoggingConfig | null>;+ /** Resolves with the number of events the destination actually accepted. */deliver: (destination: AzureSentinelDestination,events: ScrapeActivityEvent[],- ) => Promise<void>;+ ) => Promise<number>;}-const defaultDependencies: WorkerDependencies = {+const defaultDependencies: HandlerDependencies = {+ // Read the destination at delivery time from the primary, never from the+ // queued message: a rotated secret or a disabled config has to take effect on+ // events that were already in flight.getConfig: orgId =>getOrgSiemLoggingConfig(orgId, { fresh: true, primary: true }),deliver: sendAzureSentinelEvents,@@ -32,90 +37,77 @@ const defaultDependencies: WorkerDependencies = {const permanentFailureKinds = new Set(["invalid_credentials","schema_rejection",- "payload_too_large",]);-function asError(error: unknown): Error {- return error instanceof Error ? error : new Error(String(error));-}+export async function deliverSiemLoggingBatch(+ orgId: string,+ events: ScrapeActivityEvent[],+ deps: HandlerDependencies = defaultDependencies,+): Promise<SiemLoggingBatchOutcome> {+ const logger = _logger.child({ module: "siem-logging-worker", orgId });-function configuredRetryDelay(job: Job<SiemLoggingJobData>): number {- const backoff = job.opts.backoff;- if (typeof backoff === "number") return backoff;- if (!backoff) return 0;- const delay = backoff.delay ?? 0;- if (backoff.type === "exponential") {- return delay * 2 ** job.attemptsMade;+ let config: OrgSiemLoggingConfig | null;+ try {+ config = await deps.getConfig(orgId);+ } catch (error) {+ logger.error("Failed to load SIEM logging configuration", { error });+ return { disposition: "retry" };}- return delay;-}-function applyRetryAfter(- job: Job<SiemLoggingJobData>,- error: SiemDeliveryError,-): void {- if (- error.kind !== "rate_limited" ||- error.retryAfterMs === undefined ||- error.retryAfterMs <= 0- ) {- return;+ if (!config?.enabled) {+ siemLoggingEventsTotal.inc({ result: "skipped_disabled" }, events.length);+ return { disposition: "done" };}- job.opts.backoff = {- type: "fixed",- delay: Math.max(configuredRetryDelay(job), error.retryAfterMs),- };-}--export async function processSiemLoggingJob(- token: string,- job: Job<SiemLoggingJobData>,- deps: WorkerDependencies = defaultDependencies,-): Promise<void> {- const logger = _logger.child({- module: "siem-logging-worker",- jobId: job.id,- orgId: job.data.orgId,- });- const extendLockInterval = setInterval(() => {- void job.extendLock(token, config.JOB_LOCK_EXTENSION_TIME).catch(error => {- logger.warn("Failed to extend SIEM logging job lock", { error });- });- }, config.JOB_LOCK_EXTEND_INTERVAL);- extendLockInterval.unref?.();try {- const currentConfig = await deps.getConfig(job.data.orgId);- if (!currentConfig?.enabled) {- siemLoggingEventsTotal.inc({ result: "skipped_disabled" });- await job.moveToCompleted({ skipped: true }, token, false);- return;+ // Count what the destination took, not what we handed over: the sender skips+ // events it can never deliver, and those are already counted as dropped.+ const delivered = await deps.deliver(config.destination, events);+ if (delivered > 0) {+ siemLoggingEventsTotal.inc({ result: "delivered" }, delivered);}-- await deps.deliver(currentConfig.destination, [job.data.event]);- siemLoggingEventsTotal.inc({ result: "delivered" });- await job.moveToCompleted({ delivered: true }, token, false);+ return { disposition: "done" };} catch (error) {const reason =error instanceof SiemDeliveryError ? error.kind : "delivery_error";- siemLoggingEventsTotal.inc({ result: "delivery_failed" });- siemLoggingDeliveryFailuresTotal.inc({ reason });- logger.error("SIEM logging event delivery failed", {+ // A batch goes out as several chunks, so a failure part-way through still+ // left events with the destination. Count those as delivered and only the+ // remainder as failed, or the retry double-counts them.+ const delivered =+ error instanceof SiemDeliveryError ? error.deliveredEvents : 0;+ const failed = Math.max(0, events.length - delivered);+ if (delivered > 0) {+ siemLoggingEventsTotal.inc({ result: "delivered" }, delivered);+ }+ if (failed > 0) {+ siemLoggingEventsTotal.inc({ result: "delivery_failed" }, failed);+ siemLoggingDeliveryFailuresTotal.inc({ reason }, failed);+ }+ logger.error("SIEM logging batch delivery failed", {error,reason,- scrapeId: job.data.event.scrape_id,+ events: events.length,+ deliveredBeforeFailure: delivered,});+if (error instanceof SiemDeliveryError &&permanentFailureKinds.has(error.kind)) {- job.discard();+ // Bad credentials or a rejected schema will fail identically forever;+ // dead-letter so an operator sees it instead of burning the ladder.+ return { disposition: "drop" };}- if (error instanceof SiemDeliveryError) {- applyRetryAfter(job, error);- }- await job.moveToFailed(asError(error), token, false);- } finally {- clearInterval(extendLockInterval);+ return {+ disposition: "retry",+ retryAfterMs:+ error instanceof SiemDeliveryError ? error.retryAfterMs : undefined,+ };}}++export async function startSiemLoggingConsumer(): Promise<void> {+ await consumeSiemLoggingEvents((orgId, events) =>+ deliverSiemLoggingBatch(orgId, events),+ );+}diff --git a/apps/siem/azure-sentinel/DCR.json b/apps/siem/azure-sentinel/DCR.jsonindex ad2a61002..102168ead 100644--- a/apps/siem/azure-sentinel/DCR.json+++ b/apps/siem/azure-sentinel/DCR.json@@ -106,7 +106,7 @@{"streams": ["Custom-FirecrawlScrapeActivity"],"destinations": ["firecrawlWorkspace"],- "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tobool(threat.security_alert.detected), \"High\", tostring(threat.decision) == \"deny\", \"Low\", result == \"success\", \"Informational\", \"Low\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)",+ "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tostring(threat.rule) == \"risk-score\" and isnotempty(tostring(threat.provider)), \"High\", tostring(threat.decision) == \"deny\", \"Low\", \"Informational\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)","outputStream": "Microsoft-ASimWebSessionLogs"}],diff --git a/apps/siem/azure-sentinel/README.md b/apps/siem/azure-sentinel/README.mdindex 7bb0bf43d..6b1efe23b 100644--- a/apps/siem/azure-sentinel/README.md+++ b/apps/siem/azure-sentinel/README.md@@ -14,15 +14,58 @@ Sentinel through the Azure Monitor Logs Ingestion API.The database schema is maintained as a migration in the Firecrawl databaserepository, not in this package.+## Severity++The DCR transform derives ASim `EventSeverity` from how a block was decided,+not from whether a category happens to be present:++- `High` β denied by `risk-score` with a classifier verdict, i.e. a+ provider-confirmed threat.+- `Low` β denied by any other rule (local blacklist, blocked TLD, classifier+ unavailable under a closed failure policy).+- `Informational` β everything else, including scrapes that simply failed.++`Medium` is left unused so a workspace can claim it without colliding.+## Runtime`PUT /v2/team/siem` stores an organization destination and its encrypted,-write-only client secret. Scrape activity is placed on a shared BullMQ queue;-index-worker replicas consume that queue and retry transient delivery failures.-Queue jobs contain the organization ID and event, never the destination secret.+write-only client secret. Scrape activity is published to the RabbitMQ queue+`siem.logging.events`; index-worker replicas consume it, group events by+organization, and deliver one gzip batch per org. Messages carry the+organization ID and event, never the destination secret β the destination and+its secret are read fresh at delivery time, so disabling logging or rotating a+secret takes effect on events already in flight.++Operational shape of that queue:++- Bounded at 100k messages with `drop-head`, so a destination outage sheds the+ oldest events instead of growing without limit. The retry queues carry the+ same cap, and `siem.logging.dlq` is capped at 50k with a 7-day TTL β it exists+ for an operator to read, not as indefinite storage.+- Transient failures ride a fixed-delay retry ladder (5s, 30s, 5m, 30m) built+ from TTL queues that dead-letter back onto the main queue; a destination's+ `Retry-After` rounds up to the smallest rung that satisfies it.+- Bad credentials or a rejected DCR schema dead-letter straight to+ `siem.logging.dlq` rather than burning the ladder. An event too large to+ compress under 1 MB on its own is dropped and counted, not dead-lettered β+ failing its batch would lose up to 199 deliverable events alongside it.+- Retries are at-least-once: a batch that fails partway through its β€1 MB+ sub-batches will resend the sub-batches that already landed.+- Publishes are confirmed and `mandatory`, and at most 10k may await a confirm+ at once. Beyond that the producer sheds events rather than accumulating them+ in the API process, which is the same bound the queue has, applied upstream.++Queue depth comes from `rabbitmq_prometheus`.+`firecrawl_siem_logging_events_total` breaks out per-event outcomes: `queued`,+`delivered`, `retried`, `dead_lettered`, `skipped_disabled`, `enqueue_failed`,+`delivery_failed`, `unparseable`, `unroutable`, `dropped_oversized` and+`dropped_producer_full`. The last three should normally be flat at zero; any of+them moving means events are being lost rather than delayed.Set `SIEM_LOGGING_ENCRYPTION_KEY` in API and index-worker environments. If-partner egress is required, also set `PARTNER_EGRESS_PROXY_URL`.+partner egress is required, also set `PARTNER_EGRESS_PROXY_URL` and allowlist+`login.microsoftonline.com` plus `*.ingest.monitor.azure.com` on the proxy.Use `POST /v2/team/siem/test` to validate credentials and the destinationschema after deploying the Azure artifacts.diff --git a/apps/siem/azure-sentinel/azuredeploy.json b/apps/siem/azure-sentinel/azuredeploy.jsonindex 125a09ea8..186fd3354 100644--- a/apps/siem/azure-sentinel/azuredeploy.json+++ b/apps/siem/azure-sentinel/azuredeploy.json@@ -145,7 +145,7 @@{"streams": ["Custom-FirecrawlScrapeActivity"],"destinations": ["firecrawlWorkspace"],- "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tobool(threat.security_alert.detected), \"High\", tostring(threat.decision) == \"deny\", \"Low\", result == \"success\", \"Informational\", \"Low\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)",+ "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tostring(threat.rule) == \"risk-score\" and isnotempty(tostring(threat.provider)), \"High\", tostring(threat.decision) == \"deny\", \"Low\", \"Informational\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)","outputStream": "Microsoft-ASimWebSessionLogs"}],diff --git a/apps/siem/azure-sentinel/sample-event.json b/apps/siem/azure-sentinel/sample-event.jsonindex e790e77a4..13e24402f 100644--- a/apps/siem/azure-sentinel/sample-event.json+++ b/apps/siem/azure-sentinel/sample-event.json@@ -29,11 +29,7 @@"decision": "allow","rule": "default-allow","provider": "google-web-risk",- "categories": [],- "security_alert": {- "detected": false,- "category": null- }+ "categories": []}}]