Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(siem): batch audit delivery over RabbitMQ and scope threat severity (#4145)

GergΕ‘ MΓ³ricz882f64cbd3e1
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 177b7f7af..3b39e6b74 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -11,7 +11,6 @@ import {
getDeepResearchQueue,
getBillingQueue,
getPrecrawlQueue,
- getSiemLoggingQueue,
} from "./services/queue-service";
import { v0Router } from "./routes/v0";
import os from "os";
@@ -104,7 +103,8 @@ const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
new BullMQAdapter(getDeepResearchQueue()),
new BullMQAdapter(getBillingQueue()),
new BullMQAdapter(getPrecrawlQueue()),
- new BullMQAdapter(getSiemLoggingQueue()),
+ // SIEM audit delivery runs on RabbitMQ; inspect it in the broker's
+ // management UI, not here.
],
serverAdapter: serverAdapter,
});
diff --git a/apps/api/src/lib/siem-logging/azure-sentinel.test.ts b/apps/api/src/lib/siem-logging/azure-sentinel.test.ts
index 63e6145d3..cfcd79554 100644
--- a/apps/api/src/lib/siem-logging/azure-sentinel.test.ts
+++ b/apps/api/src/lib/siem-logging/azure-sentinel.test.ts
@@ -319,13 +319,39 @@ describe("Azure Sentinel sender", () => {
const compressedBodies = [...batches];
expect(compressedBodies.length).toBeGreaterThan(1);
- expect(compressedBodies.every(batch => batch.byteLength <= 1200)).toBe(
+ expect(compressedBodies.every(batch => batch.body.byteLength <= 1200)).toBe(
true,
);
expect(
compressedBodies.flatMap(batch =>
- JSON.parse(gunzipSync(batch).toString("utf8")),
+ JSON.parse(gunzipSync(batch.body).toString("utf8")),
),
).toHaveLength(noisyEvents.length);
+ // Each batch reports its own size so the caller can count what it delivered.
+ expect(
+ compressedBodies.reduce((total, batch) => total + batch.eventCount, 0),
+ ).toBe(noisyEvents.length);
+ });
+
+ // An event too large to ever deliver must not take its batch down with it: the
+ // sender POSTs chunk by chunk, so failing the call would both lose the good
+ // events and duplicate any chunk already accepted when they were replayed.
+ it("skips an undeliverable event and still delivers the rest of the batch", () => {
+ const oversized = event(
+ Array.from({ length: 400 }, () => crypto.randomUUID()).join(""),
+ );
+ const events = [event("small-1"), oversized, event("small-2")];
+
+ const batches = [...buildCompressedBatches(events, 1200)];
+ const delivered = batches.flatMap(batch =>
+ JSON.parse(gunzipSync(batch.body).toString("utf8")),
+ ) as ScrapeActivityEvent[];
+
+ expect(delivered.map(e => e.scrape_id)).toEqual(["small-1", "small-2"]);
+ // The skipped event must not be counted as delivered β€” it is already
+ // counted as dropped, and double-counting it corrupts both metrics.
+ expect(batches.reduce((total, batch) => total + batch.eventCount, 0)).toBe(
+ 2,
+ );
});
});
diff --git a/apps/api/src/lib/siem-logging/azure-sentinel.ts b/apps/api/src/lib/siem-logging/azure-sentinel.ts
index 8bef81489..ae9b8a74d 100644
--- a/apps/api/src/lib/siem-logging/azure-sentinel.ts
+++ b/apps/api/src/lib/siem-logging/azure-sentinel.ts
@@ -7,10 +7,16 @@ import {
type Response,
} from "undici";
import { config } from "../../config";
-import { siemLoggingDeliveryBatchesTotal } from "./metrics";
+import { logger as _logger } from "../logger";
+import {
+ siemLoggingDeliveryBatchesTotal,
+ siemLoggingEventsTotal,
+} from "./metrics";
import type { AzureSentinelDestination, ScrapeActivityEvent } from "./types";
import { SiemDeliveryError } from "./types";
+const logger = _logger.child({ module: "siem-logging-azure-sentinel" });
+
const MAX_COMPRESSED_BODY_BYTES = 1_000_000;
const MAX_ATTEMPTS = 3;
const MAX_RETRY_DELAY_MS = 30_000;
@@ -213,40 +219,61 @@ async function getAccessToken(
);
}
+function gzipEvents(events: ScrapeActivityEvent[]): Buffer {
+ return gzipSync(Buffer.from(JSON.stringify(events), "utf8"));
+}
+
+/**
+ * A gzipped request body plus how many events it carries, so the caller can
+ * report what was actually delivered rather than what it was handed β€” a batch
+ * can contain fewer events than the input if any were skipped as undeliverable.
+ */
+interface CompressedBatch {
+ body: Buffer;
+ eventCount: number;
+}
+
+function compressedBatch(events: ScrapeActivityEvent[]): CompressedBatch {
+ return { body: gzipEvents(events), eventCount: events.length };
+}
+
export function* buildCompressedBatches(
events: ScrapeActivityEvent[],
maxBytes = MAX_COMPRESSED_BODY_BYTES,
-): Generator<Buffer> {
+): Generator<CompressedBatch> {
let pending: ScrapeActivityEvent[] = [];
for (const event of events) {
const candidate = [...pending, event];
- const compressed = gzipSync(Buffer.from(JSON.stringify(candidate), "utf8"));
- if (compressed.byteLength <= maxBytes) {
+ if (gzipEvents(candidate).byteLength <= maxBytes) {
pending = candidate;
continue;
}
- if (pending.length === 0) {
- throw new SiemDeliveryError(
- "payload_too_large",
- `A single SIEM event exceeds the ${maxBytes}-byte compressed payload limit`,
- );
+ // The event doesn't fit alongside what's pending β€” close that batch out and
+ // consider the event on its own.
+ if (pending.length > 0) {
+ yield compressedBatch(pending);
+ pending = [];
}
- yield gzipSync(Buffer.from(JSON.stringify(pending), "utf8"));
- pending = [event];
- const single = gzipSync(Buffer.from(JSON.stringify(pending), "utf8"));
- if (single.byteLength > maxBytes) {
- throw new SiemDeliveryError(
- "payload_too_large",
- `A single SIEM event exceeds the ${maxBytes}-byte compressed payload limit`,
- );
+
+ if (gzipEvents([event]).byteLength > maxBytes) {
+ // Undeliverable at any batch size, and no retry will change that. Skip it
+ // and keep going: failing the call here would take down every other event
+ // in the batch, and dead-lettering the batch would duplicate the ones
+ // already POSTed in earlier chunks.
+ siemLoggingEventsTotal.inc({ result: "dropped_oversized" });
+ logger.error("Dropping a SIEM event larger than the payload limit", {
+ maxBytes,
+ scrapeId: event.scrape_id,
+ orgId: event.org_id,
+ });
+ continue;
}
+ pending = [event];
}
- if (pending.length > 0) {
- yield gzipSync(Buffer.from(JSON.stringify(pending), "utf8"));
- }
+ if (pending.length > 0) yield compressedBatch(pending);
}
async function sendBatch(
@@ -321,14 +348,21 @@ async function sendBatch(
}
}
+/**
+ * Returns how many events were actually accepted by the destination, which is
+ * fewer than `events.length` when any were skipped as undeliverable. Callers
+ * must attribute their `delivered` metric to this count, not to what they
+ * passed in, or a skipped event gets counted as both dropped and delivered.
+ */
export async function sendAzureSentinelEvents(
destination: AzureSentinelDestination,
events: ScrapeActivityEvent[],
deps: SenderDependencies = {},
-): Promise<void> {
- if (events.length === 0) return;
+): Promise<number> {
+ if (events.length === 0) return 0;
let token: string | undefined;
- for (const body of buildCompressedBatches(events)) {
+ let delivered = 0;
+ for (const { body, eventCount } of buildCompressedBatches(events)) {
try {
token ??= await getAccessToken(destination, deps);
try {
@@ -345,18 +379,22 @@ export async function sendAzureSentinelEvents(
throw error;
}
}
+ delivered += eventCount;
siemLoggingDeliveryBatchesTotal.inc({ result: "success" });
} catch (error) {
- if (
- error instanceof SiemDeliveryError &&
- (error.statusCode === 401 || error.statusCode === 403)
- ) {
- invalidateToken(destination);
+ if (error instanceof SiemDeliveryError) {
+ if (error.statusCode === 401 || error.statusCode === 403) {
+ invalidateToken(destination);
+ }
+ // Tell the caller what already landed, so the chunks Azure accepted
+ // aren't counted as failures on the way out.
+ error.deliveredEvents = delivered;
}
siemLoggingDeliveryBatchesTotal.inc({ result: "failure" });
throw error;
}
}
+ return delivered;
}
export function clearAzureTokenCache(): void {
diff --git a/apps/api/src/lib/siem-logging/event.test.ts b/apps/api/src/lib/siem-logging/event.test.ts
index 373de6081..f059bc88c 100644
--- a/apps/api/src/lib/siem-logging/event.test.ts
+++ b/apps/api/src/lib/siem-logging/event.test.ts
@@ -126,11 +126,47 @@ describe("ScrapeActivityEvent", () => {
expect(event.result).toBe("blocked");
expect(event.error?.code).toBe("unsafe_domain_blocked");
- expect(event.threat).toMatchObject({
+ expect(event.threat).toEqual({
decision: "deny",
rule: "risk-score",
+ provider: "google-web-risk",
categories: ["MALWARE"],
- security_alert: { detected: true, category: "MALWARE" },
+ });
+ });
+
+ // A local-rule deny never calls a provider, so it must not inherit the
+ // categories of a redirect hop that did β€” downstream that would score a
+ // policy block as a provider-confirmed threat.
+ it("scopes the threat block to the deciding decision", () => {
+ const provider: ThreatDecision = {
+ ...allowedDecision,
+ url: "https://redirect.example.com/",
+ verdict: {
+ ...allowedDecision.verdict!,
+ riskScore: 100,
+ categories: ["MALWARE"],
+ },
+ };
+ const localDeny: ThreatDecision = {
+ ...allowedDecision,
+ allowed: false,
+ rule: "blacklist",
+ providerConsulted: false,
+ verdict: null,
+ };
+ const event = buildScrapeActivityEvent("scrape-id", job(), "org-id", null, {
+ success: false,
+ error: new UnsafeDomainBlockedError(localDeny.url, localDeny),
+ threatDecisions: [provider, localDeny],
+ startedAt: 1_000,
+ completedAt: 2_000,
+ });
+
+ expect(event.threat).toEqual({
+ decision: "deny",
+ rule: "blacklist",
+ provider: null,
+ categories: [],
});
});
diff --git a/apps/api/src/lib/siem-logging/event.ts b/apps/api/src/lib/siem-logging/event.ts
index 4651ee7de..33984e953 100644
--- a/apps/api/src/lib/siem-logging/event.ts
+++ b/apps/api/src/lib/siem-logging/event.ts
@@ -78,6 +78,12 @@ function resultForOutcome(
return "failure";
}
+// One scrape can take several decisions (the initial URL plus a re-check per
+// redirect hop). Report the deciding one β€” the first deny, else the last allow β€”
+// and take its categories and provider from that same decision. Unioning
+// categories across decisions would let a local-rule deny (which never calls a
+// provider, so carries no categories) inherit another hop's threat categories
+// and be scored as a provider-confirmed threat downstream.
function threatForDecisions(
decisions: ThreatDecision[] | undefined,
): ScrapeActivityThreat | undefined {
@@ -85,20 +91,11 @@ function threatForDecisions(
const decision =
decisions.find(candidate => !candidate.allowed) ??
decisions[decisions.length - 1];
- const categories = [
- ...new Set(
- decisions.flatMap(candidate => candidate.verdict?.categories ?? []),
- ),
- ];
return {
decision: decision.allowed ? "allow" : "deny",
rule: decision.rule,
provider: decision.verdict?.provider ?? null,
- categories,
- security_alert: {
- detected: categories.length > 0,
- category: categories[0] ?? null,
- },
+ categories: [...new Set(decision.verdict?.categories ?? [])],
};
}
diff --git a/apps/api/src/lib/siem-logging/index.test.ts b/apps/api/src/lib/siem-logging/index.test.ts
index 78631474a..0b9462411 100644
--- a/apps/api/src/lib/siem-logging/index.test.ts
+++ b/apps/api/src/lib/siem-logging/index.test.ts
@@ -17,7 +17,7 @@ vi.mock("./store", () => ({
getApiKeyName: mocks.getApiKeyName,
}));
-vi.mock("./queue", () => ({
+vi.mock("./transport", () => ({
enqueueSiemLoggingEvent: mocks.enqueue,
enqueueSiemLoggingEvents: vi.fn().mockResolvedValue(undefined),
}));
diff --git a/apps/api/src/lib/siem-logging/index.ts b/apps/api/src/lib/siem-logging/index.ts
index e400fdae9..672370fb3 100644
--- a/apps/api/src/lib/siem-logging/index.ts
+++ b/apps/api/src/lib/siem-logging/index.ts
@@ -7,7 +7,7 @@ import {
type RejectedScrapeActivity,
type ScrapeActivityOutcome,
} from "./event";
-import { enqueueSiemLoggingEvent, enqueueSiemLoggingEvents } from "./queue";
+import { enqueueSiemLoggingEvent, enqueueSiemLoggingEvents } from "./transport";
import { getApiKeyName, isOrgSiemLoggingEnabled } from "./store";
const logger = _logger.child({ module: "siem-logging" });
diff --git a/apps/api/src/lib/siem-logging/queue.test.ts b/apps/api/src/lib/siem-logging/queue.test.ts
deleted file mode 100644
index 533c7ea61..000000000
--- a/apps/api/src/lib/siem-logging/queue.test.ts
+++ /dev/null
@@ -1,57 +0,0 @@
-import { beforeEach, describe, expect, it, vi } from "vitest";
-import type { ScrapeActivityEvent } from "./types";
-
-const mocks = vi.hoisted(() => ({
- addBulk: vi.fn().mockResolvedValue([]),
- inc: vi.fn(),
-}));
-
-vi.mock("../../services/queue-service", () => ({
- getSiemLoggingQueue: () => ({ addBulk: mocks.addBulk }),
-}));
-
-vi.mock("./metrics", () => ({
- siemLoggingEventsTotal: { inc: mocks.inc },
-}));
-
-import { enqueueSiemLoggingEvent } from "./queue";
-
-const event: ScrapeActivityEvent = {
- schema_version: 1,
- event_type: "scrape_activity",
- scrape_id: "scrape-id",
- request_id: "request-id",
- endpoint: "scrape",
- team_id: "team-id",
- org_id: "org-id",
- api_key: { id: null, name: null },
- started_at: "2026-07-27T00:00:00.000Z",
- completed_at: "2026-07-27T00:00:01.000Z",
- url: "https://example.com",
- domain: "example.com",
- http_method: "GET",
- http_status: 200,
- result: "success",
- error: null,
- origin: "api",
- integration: null,
- zero_data_retention: false,
-};
-
-describe("SIEM logging queue", () => {
- beforeEach(() => {
- vi.clearAllMocks();
- });
-
- it("deduplicates repeat terminal events regardless of completion time", async () => {
- await enqueueSiemLoggingEvent("org-id", event);
- await enqueueSiemLoggingEvent("org-id", {
- ...event,
- completed_at: "2026-07-27T00:01:00.000Z",
- });
-
- const firstJobId = mocks.addBulk.mock.calls[0][0][0].opts.jobId;
- const secondJobId = mocks.addBulk.mock.calls[1][0][0].opts.jobId;
- expect(firstJobId).toBe(secondJobId);
- });
-});
diff --git a/apps/api/src/lib/siem-logging/queue.ts b/apps/api/src/lib/siem-logging/queue.ts
deleted file mode 100644
index 45e343400..000000000
--- a/apps/api/src/lib/siem-logging/queue.ts
+++ /dev/null
@@ -1,46 +0,0 @@
-import { createHash } from "node:crypto";
-import { getSiemLoggingQueue } from "../../services/queue-service";
-import { siemLoggingEventsTotal } from "./metrics";
-import type { ScrapeActivityEvent, SiemLoggingJobData } from "./types";
-
-function jobIdFor(orgId: string, event: ScrapeActivityEvent): string {
- return createHash("sha256")
- .update(
- JSON.stringify([
- orgId,
- event.schema_version,
- event.event_type,
- event.scrape_id,
- event.request_id,
- event.result,
- ]),
- )
- .digest("hex");
-}
-
-export async function enqueueSiemLoggingEvent(
- orgId: string,
- event: ScrapeActivityEvent,
-): Promise<void> {
- await enqueueSiemLoggingEvents(orgId, [event]);
-}
-
-export async function enqueueSiemLoggingEvents(
- orgId: string,
- events: ScrapeActivityEvent[],
-): Promise<void> {
- if (events.length === 0) return;
- try {
- await getSiemLoggingQueue().addBulk(
- events.map(event => ({
- name: "deliver",
- data: { orgId, event } satisfies SiemLoggingJobData,
- opts: { jobId: jobIdFor(orgId, event) },
- })),
- );
- siemLoggingEventsTotal.inc({ result: "queued" }, events.length);
- } catch (error) {
- siemLoggingEventsTotal.inc({ result: "enqueue_failed" }, events.length);
- throw error;
- }
-}
diff --git a/apps/api/src/lib/siem-logging/transport.test.ts b/apps/api/src/lib/siem-logging/transport.test.ts
new file mode 100644
index 000000000..601dc8691
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/transport.test.ts
@@ -0,0 +1,506 @@
+import type { Mock } from "vitest";
+import type { ScrapeActivityEvent } from "./types";
+
+const { connectMock, channelMock, connectionMock } = vi.hoisted(() => {
+ const channelMock = {
+ assertExchange: vi.fn(),
+ assertQueue: vi.fn(),
+ bindQueue: vi.fn(),
+ prefetch: vi.fn(),
+ consume: vi.fn(),
+ ack: vi.fn(),
+ nack: vi.fn(),
+ // Confirm-channel shape: the last argument is the broker's ack callback.
+ publish: vi.fn((...args: unknown[]) => {
+ const callback = args[4];
+ if (typeof callback === "function") {
+ (callback as (error: Error | null) => void)(null);
+ }
+ return true;
+ }),
+ close: vi.fn(),
+ on: vi.fn(),
+ };
+ // Async like the real client, so a test can hold a connection open mid-connect.
+ const connectionMock = {
+ createChannel: vi.fn(async () => channelMock),
+ createConfirmChannel: vi.fn(async () => channelMock),
+ on: vi.fn(),
+ close: vi.fn(async () => {}),
+ };
+ const connectMock = vi.fn(() => connectionMock);
+ return { connectMock, channelMock, connectionMock };
+});
+
+vi.mock("amqplib", () => ({ default: { connect: connectMock } }));
+vi.mock("../../config", () => ({
+ config: { NUQ_RABBITMQ_URL: "amqp://test" },
+}));
+// Each test re-imports the module under test, and prom-client rejects a second
+// registration of the same counter name in one registry.
+const { metricsMock } = vi.hoisted(() => ({ metricsMock: { inc: vi.fn() } }));
+vi.mock("./metrics", () => ({ siemLoggingEventsTotal: metricsMock }));
+vi.mock("../logger", () => {
+ const mk = (): Record<string, unknown> => ({
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ child: () => mk(),
+ });
+ return { logger: mk() };
+});
+
+function event(scrapeId: string): ScrapeActivityEvent {
+ return {
+ schema_version: 1,
+ event_type: "scrape_activity",
+ scrape_id: scrapeId,
+ request_id: "request-id",
+ endpoint: "scrape",
+ team_id: "team-id",
+ org_id: "org-id",
+ api_key: { id: null, name: null },
+ started_at: "2026-07-27T00:00:00.000Z",
+ completed_at: "2026-07-27T00:00:01.000Z",
+ url: "https://example.com",
+ domain: "example.com",
+ http_method: "GET",
+ http_status: 200,
+ result: "success",
+ error: null,
+ origin: "api",
+ integration: null,
+ zero_data_retention: false,
+ };
+}
+
+function message(orgId: string, scrapeId: string, attempt = 0) {
+ return {
+ content: Buffer.from(JSON.stringify({ orgId, event: event(scrapeId) })),
+ properties: {
+ messageId: scrapeId,
+ headers: { "x-siem-attempt": attempt },
+ },
+ };
+}
+
+async function freshTransport() {
+ vi.resetModules();
+ return import("./transport.js");
+}
+
+function assertedQueue(name: string) {
+ const call = (channelMock.assertQueue as Mock).mock.calls.find(
+ c => c[0] === name,
+ );
+ return call?.[1] as { arguments?: Record<string, unknown> } | undefined;
+}
+
+describe("SIEM logging transport", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ connectMock.mockReturnValue(connectionMock);
+ connectionMock.createChannel.mockResolvedValue(channelMock);
+ connectionMock.createConfirmChannel.mockResolvedValue(channelMock);
+ });
+
+ it("bounds the event queue in the broker instead of in memory", async () => {
+ const transport = await freshTransport();
+ await transport.enqueueSiemLoggingEvent("org-a", event("scrape-1"));
+
+ const main = assertedQueue("siem.logging.events");
+ expect(main?.arguments).toMatchObject({
+ "x-max-length": 100_000,
+ "x-overflow": "drop-head",
+ "x-dead-letter-exchange": "siem.logging.dlx",
+ });
+ // Every retry rung is bounded too, so a wedged destination can't turn the
+ // ladder into unbounded storage.
+ for (const delayMs of [5_000, 30_000, 300_000, 1_800_000]) {
+ expect(
+ assertedQueue(`siem.logging.events.retry.${delayMs}`)?.arguments,
+ ).toMatchObject({
+ "x-message-ttl": delayMs,
+ "x-max-length": 100_000,
+ "x-overflow": "drop-head",
+ "x-dead-letter-exchange": "siem.logging",
+ });
+ }
+ });
+
+ it("batches an organization's events into one handler call on the linger", async () => {
+ vi.useFakeTimers();
+ try {
+ const transport = await freshTransport();
+ const handler = vi.fn().mockResolvedValue({ disposition: "done" });
+ await transport.consumeSiemLoggingEvents(handler, {
+ batchSize: 10,
+ batchLingerMs: 1000,
+ });
+ const onMessage = channelMock.consume.mock.calls[0][1];
+
+ const first = message("org-a", "scrape-1");
+ const second = message("org-a", "scrape-2");
+ await onMessage(first);
+ await onMessage(second);
+ expect(handler).not.toHaveBeenCalled();
+
+ await vi.advanceTimersByTimeAsync(1000);
+
+ expect(handler).toHaveBeenCalledOnce();
+ const [orgId, events] = handler.mock.calls[0];
+ expect(orgId).toBe("org-a");
+ expect(events.map((e: ScrapeActivityEvent) => e.scrape_id)).toEqual([
+ "scrape-1",
+ "scrape-2",
+ ]);
+ expect(channelMock.ack).toHaveBeenCalledWith(first);
+ expect(channelMock.ack).toHaveBeenCalledWith(second);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("never mixes organizations in one batch", async () => {
+ vi.useFakeTimers();
+ try {
+ const transport = await freshTransport();
+ const handler = vi.fn().mockResolvedValue({ disposition: "done" });
+ await transport.consumeSiemLoggingEvents(handler, {
+ batchSize: 10,
+ batchLingerMs: 1000,
+ });
+ const onMessage = channelMock.consume.mock.calls[0][1];
+
+ await onMessage(message("org-a", "scrape-1"));
+ await onMessage(message("org-b", "scrape-2"));
+ await vi.advanceTimersByTimeAsync(1000);
+
+ expect(handler).toHaveBeenCalledTimes(2);
+ expect(handler.mock.calls.map(c => c[0]).sort()).toEqual([
+ "org-a",
+ "org-b",
+ ]);
+ for (const [, events] of handler.mock.calls) {
+ expect(events).toHaveLength(1);
+ }
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("flushes early once a batch reaches its size ceiling", async () => {
+ vi.useFakeTimers();
+ try {
+ const transport = await freshTransport();
+ const handler = vi.fn().mockResolvedValue({ disposition: "done" });
+ await transport.consumeSiemLoggingEvents(handler, {
+ batchSize: 2,
+ batchLingerMs: 60_000,
+ });
+ const onMessage = channelMock.consume.mock.calls[0][1];
+
+ await onMessage(message("org-a", "scrape-1"));
+ await onMessage(message("org-a", "scrape-2"));
+
+ // No timer advance: the size ceiling closed the batch.
+ expect(handler).toHaveBeenCalledOnce();
+ expect(handler.mock.calls[0][1]).toHaveLength(2);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("leaves prefetch large enough to fill a batch", async () => {
+ const transport = await freshTransport();
+ await transport.consumeSiemLoggingEvents(vi.fn(), { batchSize: 50 });
+
+ const [prefetch, global] = channelMock.prefetch.mock.calls[0];
+ expect(prefetch).toBeGreaterThanOrEqual(50);
+ expect(global).toBe(false);
+ });
+
+ it("republishes a retry onto the next ladder rung and acks the original", async () => {
+ const transport = await freshTransport();
+ await transport.consumeSiemLoggingEvents(
+ vi.fn().mockResolvedValue({ disposition: "retry" }),
+ { batchSize: 1 },
+ );
+ const onMessage = channelMock.consume.mock.calls[0][1];
+
+ const msg = message("org-a", "scrape-1", 1);
+ await onMessage(msg);
+
+ const retryPublish = channelMock.publish.mock.calls.find(
+ c => c[0] === "siem.logging.retry",
+ );
+ expect(retryPublish?.[1]).toBe("siem.logging.events.retry.30000");
+ expect(retryPublish?.[3]).toMatchObject({
+ headers: { "x-siem-attempt": 2 },
+ });
+ // Acked, not nacked: a requeue would redeliver immediately and spin against
+ // a destination that is still down.
+ expect(channelMock.ack).toHaveBeenCalledWith(msg);
+ expect(channelMock.nack).not.toHaveBeenCalled();
+ });
+
+ it("rounds a Retry-After up to the smallest rung that satisfies it", async () => {
+ const transport = await freshTransport();
+ await transport.consumeSiemLoggingEvents(
+ vi.fn().mockResolvedValue({ disposition: "retry", retryAfterMs: 20_000 }),
+ { batchSize: 1 },
+ );
+ const onMessage = channelMock.consume.mock.calls[0][1];
+
+ await onMessage(message("org-a", "scrape-1"));
+
+ const retryPublish = channelMock.publish.mock.calls.find(
+ c => c[0] === "siem.logging.retry",
+ );
+ expect(retryPublish?.[1]).toBe("siem.logging.events.retry.30000");
+ });
+
+ it("dead-letters once the ladder is exhausted", async () => {
+ const transport = await freshTransport();
+ await transport.consumeSiemLoggingEvents(
+ vi.fn().mockResolvedValue({ disposition: "retry" }),
+ { batchSize: 1 },
+ );
+ const onMessage = channelMock.consume.mock.calls[0][1];
+
+ const msg = message("org-a", "scrape-1", 4);
+ await onMessage(msg);
+
+ expect(
+ channelMock.publish.mock.calls.some(c => c[0] === "siem.logging.retry"),
+ ).toBe(false);
+ expect(channelMock.nack).toHaveBeenCalledWith(msg, false, false);
+ });
+
+ it("dead-letters a permanent failure without touching the ladder", async () => {
+ const transport = await freshTransport();
+ await transport.consumeSiemLoggingEvents(
+ vi.fn().mockResolvedValue({ disposition: "drop" }),
+ { batchSize: 1 },
+ );
+ const onMessage = channelMock.consume.mock.calls[0][1];
+
+ const msg = message("org-a", "scrape-1");
+ await onMessage(msg);
+
+ expect(
+ channelMock.publish.mock.calls.some(c => c[0] === "siem.logging.retry"),
+ ).toBe(false);
+ expect(channelMock.nack).toHaveBeenCalledWith(msg, false, false);
+ });
+
+ it("retries the batch when the handler throws", async () => {
+ const transport = await freshTransport();
+ await transport.consumeSiemLoggingEvents(
+ vi.fn().mockRejectedValue(new Error("handler boom")),
+ { batchSize: 1 },
+ );
+ const onMessage = channelMock.consume.mock.calls[0][1];
+
+ await onMessage(message("org-a", "scrape-1"));
+
+ expect(
+ channelMock.publish.mock.calls.some(c => c[0] === "siem.logging.retry"),
+ ).toBe(true);
+ });
+
+ it("fails the enqueue when the broker nacks the publish", async () => {
+ const transport = await freshTransport();
+ channelMock.publish.mockImplementationOnce((...args: unknown[]) => {
+ (args[4] as (error: Error | null) => void)(new Error("nacked"));
+ return true;
+ });
+
+ await expect(
+ transport.enqueueSiemLoggingEvent("org-a", event("scrape-1")),
+ ).rejects.toThrow("Failed to publish 1 of 1");
+ expect(metricsMock.inc).toHaveBeenCalledWith(
+ { result: "enqueue_failed" },
+ 1,
+ );
+ });
+
+ it("leaves a message unsettled when its retry republish is not confirmed", async () => {
+ const transport = await freshTransport();
+ await transport.consumeSiemLoggingEvents(
+ vi.fn().mockResolvedValue({ disposition: "retry" }),
+ { batchSize: 1 },
+ );
+ const onMessage = channelMock.consume.mock.calls[0][1];
+ channelMock.publish.mockImplementationOnce((...args: unknown[]) => {
+ (args[4] as (error: Error | null) => void)(new Error("nacked"));
+ return true;
+ });
+
+ const msg = message("org-a", "scrape-1");
+ await onMessage(msg);
+
+ // Neither acked nor dead-lettered: the broker redelivers it instead of the
+ // event vanishing between a lost republish and an ack.
+ expect(channelMock.ack).not.toHaveBeenCalled();
+ expect(channelMock.nack).not.toHaveBeenCalled();
+ });
+
+ it("dead-letters an unparseable message instead of hot-looping it", async () => {
+ const transport = await freshTransport();
+ await transport.consumeSiemLoggingEvents(vi.fn(), { batchSize: 1 });
+ const onMessage = channelMock.consume.mock.calls[0][1];
+
+ const msg = { content: Buffer.from("not json"), properties: {} };
+ await onMessage(msg);
+
+ expect(channelMock.nack).toHaveBeenCalledWith(msg, false, false);
+ });
+
+ it("dead-letters well-formed JSON that is not a SIEM message", async () => {
+ const transport = await freshTransport();
+ const handler = vi.fn();
+ await transport.consumeSiemLoggingEvents(handler, { batchSize: 1 });
+ const onMessage = channelMock.consume.mock.calls[0][1];
+
+ const msg = {
+ content: Buffer.from(JSON.stringify({ event: event("scrape-1") })),
+ properties: {},
+ };
+ await onMessage(msg);
+
+ expect(handler).not.toHaveBeenCalled();
+ expect(channelMock.nack).toHaveBeenCalledWith(msg, false, false);
+ });
+
+ it("counts the accepted events when only some publishes are nacked", async () => {
+ const transport = await freshTransport();
+ channelMock.publish.mockImplementationOnce((...args: unknown[]) => {
+ (args[4] as (error: Error | null) => void)(new Error("nacked"));
+ return true;
+ });
+
+ // Rejects, but the sibling event was accepted β€” the aggregate must not
+ // report the whole call as failed.
+ await expect(
+ transport.enqueueSiemLoggingEvents("org-a", [
+ event("scrape-1"),
+ event("scrape-2"),
+ ]),
+ ).rejects.toThrow("Failed to publish 1 of 2");
+ expect(metricsMock.inc).toHaveBeenCalledWith({ result: "queued" }, 1);
+ expect(metricsMock.inc).toHaveBeenCalledWith(
+ { result: "enqueue_failed" },
+ 1,
+ );
+ });
+
+ it("publishes as mandatory and reports what the broker cannot route", async () => {
+ const transport = await freshTransport();
+ await transport.enqueueSiemLoggingEvent("org-a", event("scrape-1"));
+
+ expect(channelMock.publish.mock.calls[0][3]).toMatchObject({
+ mandatory: true,
+ });
+
+ const onReturn = (channelMock.on as Mock).mock.calls.find(
+ c => c[0] === "return",
+ )?.[1] as (msg: unknown) => void;
+ expect(onReturn).toBeTypeOf("function");
+ onReturn({
+ fields: { exchange: "siem.logging", routingKey: "siem.logging.events" },
+ properties: { messageId: "scrape-1" },
+ });
+ expect(metricsMock.inc).toHaveBeenCalledWith({ result: "unroutable" });
+ });
+
+ it("retries the subscribe when the broker is down at startup", async () => {
+ vi.useFakeTimers();
+ try {
+ const transport = await freshTransport();
+ connectMock.mockImplementationOnce(() => {
+ throw new Error("broker down");
+ });
+
+ // Must resolve, not reject: an unavailable broker at boot has to enter the
+ // reconnect loop instead of leaving the replica permanently deaf.
+ await expect(
+ transport.consumeSiemLoggingEvents(vi.fn(), { batchSize: 1 }),
+ ).resolves.toBeUndefined();
+ expect(channelMock.consume).not.toHaveBeenCalled();
+
+ await vi.advanceTimersByTimeAsync(1500);
+
+ expect(channelMock.consume).toHaveBeenCalledTimes(1);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("does not subscribe when shutdown races the initial connection", async () => {
+ const transport = await freshTransport();
+ let releaseChannel!: () => void;
+ connectionMock.createChannel.mockImplementationOnce(
+ () =>
+ new Promise(resolve => {
+ releaseChannel = () => resolve(channelMock);
+ }),
+ );
+
+ const subscribing = transport.consumeSiemLoggingEvents(vi.fn(), {
+ batchSize: 1,
+ });
+ await vi.waitFor(() =>
+ expect(connectionMock.createChannel).toHaveBeenCalled(),
+ );
+
+ // Shutdown lands while that first channel is still opening.
+ await transport.closeSiemLoggingTransport();
+ releaseChannel();
+ await subscribing;
+
+ expect(channelMock.consume).not.toHaveBeenCalled();
+ expect(connectionMock.close).toHaveBeenCalled();
+ });
+
+ it("does not reopen the transport when shutdown races a pending reconnect", async () => {
+ vi.useFakeTimers();
+ try {
+ const transport = await freshTransport();
+ connectMock.mockImplementationOnce(() => {
+ throw new Error("broker down");
+ });
+ await transport.consumeSiemLoggingEvents(vi.fn(), { batchSize: 1 });
+
+ await transport.closeSiemLoggingTransport();
+ await vi.advanceTimersByTimeAsync(5000);
+
+ // The pending reconnect timer must not resurrect a connection that
+ // shutdown already tore down.
+ expect(channelMock.consume).not.toHaveBeenCalled();
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("re-subscribes the consumer after a channel drop (does not go deaf)", async () => {
+ vi.useFakeTimers();
+ try {
+ const transport = await freshTransport();
+ await transport.consumeSiemLoggingEvents(vi.fn(), { batchSize: 1 });
+ expect(channelMock.consume).toHaveBeenCalledTimes(1);
+
+ const onClose = (channelMock.on as Mock).mock.calls.find(
+ c => c[0] === "close",
+ )?.[1] as () => void;
+ expect(onClose).toBeTypeOf("function");
+ onClose();
+
+ await vi.advanceTimersByTimeAsync(1500);
+
+ expect(channelMock.consume.mock.calls.length).toBeGreaterThanOrEqual(2);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+});
diff --git a/apps/api/src/lib/siem-logging/transport.ts b/apps/api/src/lib/siem-logging/transport.ts
new file mode 100644
index 000000000..2fa0f657d
--- /dev/null
+++ b/apps/api/src/lib/siem-logging/transport.ts
@@ -0,0 +1,678 @@
+// RabbitMQ transport for SIEM audit delivery.
+//
+// Why RabbitMQ and not the BullMQ queues next door: this queue turns one message
+// per scrape into an outbound HTTPS call, so it needs two things Redis/BullMQ
+// cannot express. First a hard bound β€” x-max-length with drop-head caps the
+// backlog in the broker, so a destination outage can never grow it without limit
+// and starve the workers. Second real batching β€” prefetch plus manual ack lets a
+// consumer hold many messages, group them by organization, and deliver one gzip
+// batch per org instead of one authenticated POST per scrape.
+//
+// Retries ride a fixed-delay ladder (x-message-ttl queues that dead-letter back
+// onto the main queue) because the cluster does not load
+// rabbitmq_delayed_message_exchange. Fixed per-queue TTLs also avoid the
+// head-of-line blocking that per-message TTLs would introduce.
+
+import amqp from "amqplib";
+import { config } from "../../config";
+import { logger as _logger } from "../logger";
+import { siemLoggingEventsTotal } from "./metrics";
+import type { ScrapeActivityEvent, SiemLoggingMessage } from "./types";
+
+const EXCHANGE = "siem.logging";
+const QUEUE = "siem.logging.events";
+const RETRY_EXCHANGE = "siem.logging.retry";
+const DLX = "siem.logging.dlx";
+const DLQ = "siem.logging.dlq";
+
+// ~36 minutes of destination downtime before an event dead-letters. The sender
+// already makes three in-process attempts per call, so these rungs cover
+// outages, not blips.
+const RETRY_DELAYS_MS = [5_000, 30_000, 300_000, 1_800_000] as const;
+
+// Hard backlog bound, enforced by the broker. An outage costs the oldest slice
+// of the audit trail instead of the workers' memory; queue depth and the drop
+// counters are scraped from rabbitmq_prometheus for alerting.
+const MAX_QUEUED_MESSAGES = 100_000;
+
+// The dead-letter queue holds what an operator needs to diagnose, so it gets a
+// smaller cap and a retention window rather than growing forever.
+const MAX_DEAD_LETTERED_MESSAGES = 50_000;
+const DLQ_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+
+// A batch closes on whichever comes first. The linger is what turns per-scrape
+// messages into per-org batches; keep it short enough that a quiet org still
+// sees its audit events land promptly.
+const DEFAULT_BATCH_SIZE = 200;
+const DEFAULT_BATCH_LINGER_MS = 1_000;
+
+const logger = _logger.child({ module: "siem-logging-transport" });
+
+export type SiemLoggingBatchOutcome = {
+ /** "done" acks the batch, "retry" moves it to the next rung, "drop" dead-letters it. */
+ disposition: "done" | "retry" | "drop";
+ /** Earliest acceptable retry delay in ms β€” honors a destination's Retry-After. */
+ retryAfterMs?: number;
+};
+
+/**
+ * Deliver one organization's events. The batch shares a single fate because it
+ * maps to one authenticated call against that org's destination.
+ */
+export type SiemLoggingBatchHandler = (
+ orgId: string,
+ events: ScrapeActivityEvent[],
+) => Promise<SiemLoggingBatchOutcome>;
+
+export interface SiemLoggingConsumerOptions {
+ /** Unacked message budget; also the ceiling on how large a batch can grow. */
+ prefetch?: number;
+ batchSize?: number;
+ batchLingerMs?: number;
+}
+
+let connection: amqp.ChannelModel | null = null;
+// Memoized so concurrent callers share one connect() instead of racing to open
+// (and orphan) duplicate connections.
+let connectionPromise: Promise<amqp.ChannelModel> | null = null;
+// Separate publish/consume channels so an exception on one can't tear down the other.
+// Publishing goes through a confirm channel: emission is already fire-and-forget
+// off the scrape path, so waiting for the broker to acknowledge each event costs
+// the caller nothing and is the difference between a complete audit record and
+// one with silent holes in it.
+let publishChannel: amqp.ConfirmChannel | null = null;
+let consumeChannel: amqp.Channel | null = null;
+let publishChannelPromise: Promise<amqp.ConfirmChannel> | null = null;
+let consumeChannelPromise: Promise<amqp.Channel> | null = null;
+
+// Registered so a reconnect can re-attach the consumer; index-worker replicas
+// never publish, so without this they go permanently deaf after a broker blip.
+let registeredHandler: SiemLoggingBatchHandler | null = null;
+let registeredOptions: SiemLoggingConsumerOptions = {};
+let subscribed = false;
+let reconnectInFlight = false;
+let reconnectTimer: NodeJS.Timeout | null = null;
+// Set by close() so a reconnect already in flight can't resurrect the transport
+// after shutdown.
+let closed = false;
+const RECONNECT_BASE_DELAY_MS = 1000;
+const RECONNECT_MAX_DELAY_MS = 30_000;
+
+// Ceiling on publishes awaiting a broker confirm. Emission is fire-and-forget,
+// so nothing upstream throttles it: without this, a broker that accepts
+// connections but stops confirming would let unconfirmed events pile up in the
+// producer until the pod dies β€” the exact failure the queue bound exists to
+// prevent, just moved into the API process.
+const MAX_IN_FLIGHT_CONFIRMS = 10_000;
+let inFlightConfirms = 0;
+
+function retryQueueName(delayMs: number): string {
+ return `${QUEUE}.retry.${delayMs}`;
+}
+
+async function assertTopology(ch: amqp.Channel): Promise<void> {
+ await ch.assertExchange(EXCHANGE, "direct", { durable: true });
+ await ch.assertExchange(RETRY_EXCHANGE, "direct", { durable: true });
+ await ch.assertExchange(DLX, "direct", { durable: true });
+
+ // Bounded like everything else: a standing bad credential or rejected schema
+ // pushes every event here, and an unbounded durable queue would eventually
+ // fill the broker's disk. Retention is for an operator to read, not to replay
+ // indefinitely.
+ await ch.assertQueue(DLQ, {
+ durable: true,
+ arguments: {
+ "x-max-length": MAX_DEAD_LETTERED_MESSAGES,
+ "x-overflow": "drop-head",
+ "x-message-ttl": DLQ_RETENTION_MS,
+ },
+ });
+ await ch.bindQueue(DLQ, DLX, QUEUE);
+
+ // Classic, not quorum like the monitoring queues: x-max-length with drop-head
+ // is the bound this queue exists to have, and classic queues support that
+ // combination on every broker version we might be running. The cost is that a
+ // node loss drops this queue's backlog β€” acceptable for a buffer that already
+ // sheds its oldest events at the cap, and cheaper than a declaration that
+ // fails with PRECONDITION_FAILED and takes all delivery down with it.
+ await ch.assertQueue(QUEUE, {
+ durable: true,
+ arguments: {
+ "x-max-length": MAX_QUEUED_MESSAGES,
+ "x-overflow": "drop-head",
+ "x-dead-letter-exchange": DLX,
+ "x-dead-letter-routing-key": QUEUE,
+ },
+ });
+ await ch.bindQueue(QUEUE, EXCHANGE, QUEUE);
+
+ // Each rung parks messages for a fixed TTL, then dead-letters them back onto
+ // the main queue. The same bound applies, so a wedged destination can't turn
+ // the retry ladder into unbounded storage either.
+ for (const delayMs of RETRY_DELAYS_MS) {
+ const queue = retryQueueName(delayMs);
+ await ch.assertQueue(queue, {
+ durable: true,
+ arguments: {
+ "x-message-ttl": delayMs,
+ "x-max-length": MAX_QUEUED_MESSAGES,
+ "x-overflow": "drop-head",
+ "x-dead-letter-exchange": EXCHANGE,
+ "x-dead-letter-routing-key": QUEUE,
+ },
+ });
+ await ch.bindQueue(queue, RETRY_EXCHANGE, queue);
+ }
+}
+
+function resetCachedState(): void {
+ connection = null;
+ connectionPromise = null;
+ publishChannel = null;
+ publishChannelPromise = null;
+ consumeChannel = null;
+ consumeChannelPromise = null;
+ subscribed = false;
+ abandonPendingBatches();
+}
+
+function handleConnectionDrop(reason: string, error?: unknown): void {
+ if (connection || publishChannel || consumeChannel) {
+ logger.warn("SIEM logging transport dropped β€” will reconnect", {
+ reason,
+ error,
+ });
+ }
+ resetCachedState();
+ if (registeredHandler) scheduleReconnect(RECONNECT_BASE_DELAY_MS);
+}
+
+function scheduleReconnect(delayMs: number): void {
+ if (closed || reconnectInFlight) return;
+ reconnectInFlight = true;
+
+ reconnectTimer = setTimeout(async () => {
+ reconnectTimer = null;
+ try {
+ const ch = await getConsumeChannel();
+ // Shutdown can land while the channel was opening. Without this the
+ // connection we just opened outlives close() with nothing consuming it.
+ if (closed) {
+ reconnectInFlight = false;
+ await closeConnection();
+ return;
+ }
+ if (registeredHandler) {
+ await subscribe(ch, registeredHandler, registeredOptions);
+ }
+ reconnectInFlight = false;
+ logger.info("SIEM logging transport reconnected; consumer re-subscribed");
+ } catch (error) {
+ reconnectInFlight = false;
+ const nextDelay = Math.min(delayMs * 2, RECONNECT_MAX_DELAY_MS);
+ logger.error("SIEM logging transport reconnect failed; retrying", {
+ error,
+ nextDelayMs: nextDelay,
+ });
+ // A mid-subscribe failure on a healthy channel never fires "close", so
+ // reset everything or the next attempt short-circuits and stays deaf.
+ await consumeChannel?.close().catch(() => {});
+ resetCachedState();
+ scheduleReconnect(nextDelay);
+ }
+ }, delayMs);
+ reconnectTimer.unref?.();
+}
+
+async function openConnection(): Promise<amqp.ChannelModel> {
+ const url = config.NUQ_RABBITMQ_URL;
+ if (!url) throw new Error("NUQ_RABBITMQ_URL is not configured");
+
+ const conn = await amqp.connect(url, {
+ clientProperties: { connection_name: "siem-logging" },
+ });
+ conn.on("close", () => handleConnectionDrop("connection closed"));
+ conn.on("error", error =>
+ logger.error("SIEM logging transport connection error", { error }),
+ );
+ return conn;
+}
+
+async function getConnection(): Promise<amqp.ChannelModel> {
+ if (connection) return connection;
+ if (!connectionPromise) {
+ connectionPromise = openConnection()
+ .then(conn => {
+ connection = conn;
+ return conn;
+ })
+ .catch(error => {
+ connectionPromise = null;
+ throw error;
+ });
+ }
+ return connectionPromise;
+}
+
+async function createChannel(label: "publish"): Promise<amqp.ConfirmChannel>;
+async function createChannel(label: "consume"): Promise<amqp.Channel>;
+async function createChannel(
+ label: "publish" | "consume",
+): Promise<amqp.Channel | amqp.ConfirmChannel> {
+ const conn = await getConnection();
+ const ch =
+ label === "publish"
+ ? await conn.createConfirmChannel()
+ : await conn.createChannel();
+ await assertTopology(ch);
+ // Channels can close independently of the connection (e.g. PRECONDITION_FAILED);
+ // without this the cached channel stays non-null and every call throws forever.
+ ch.on("close", () => {
+ if (label === "publish" && publishChannel === ch) {
+ publishChannel = null;
+ publishChannelPromise = null;
+ }
+ if (label === "consume" && consumeChannel === ch) {
+ consumeChannel = null;
+ consumeChannelPromise = null;
+ subscribed = false;
+ handleConnectionDrop("consume channel closed");
+ }
+ });
+ ch.on("error", error =>
+ logger.error("SIEM logging transport channel error", { label, error }),
+ );
+ // Only reachable if the bindings asserted above are removed out of band, but
+ // the failure mode is a silently discarded audit event, so make it loud.
+ if (label === "publish") {
+ ch.on("return", msg => {
+ siemLoggingEventsTotal.inc({ result: "unroutable" });
+ logger.error("Broker could not route a SIEM logging event", {
+ exchange: msg.fields.exchange,
+ routingKey: msg.fields.routingKey,
+ messageId: msg.properties.messageId,
+ });
+ });
+ }
+ return ch;
+}
+
+async function getPublishChannel(): Promise<amqp.ConfirmChannel> {
+ if (publishChannel) return publishChannel;
+ if (!publishChannelPromise) {
+ publishChannelPromise = createChannel("publish")
+ .then(ch => {
+ publishChannel = ch;
+ return ch;
+ })
+ .catch(error => {
+ publishChannelPromise = null;
+ throw error;
+ });
+ }
+ return publishChannelPromise;
+}
+
+async function getConsumeChannel(): Promise<amqp.Channel> {
+ if (consumeChannel) return consumeChannel;
+ if (!consumeChannelPromise) {
+ consumeChannelPromise = createChannel("consume")
+ .then(ch => {
+ consumeChannel = ch;
+ return ch;
+ })
+ .catch(error => {
+ consumeChannelPromise = null;
+ throw error;
+ });
+ }
+ return consumeChannelPromise;
+}
+
+// Resolves once the broker has taken responsibility for the message, and rejects
+// if it nacks instead. A confirm says nothing about routing β€” an unroutable
+// publish is still confirmed β€” so publishes are `mandatory` and the channel's
+// "return" handler counts anything the exchange couldn't route.
+function publishConfirmed(
+ ch: amqp.ConfirmChannel,
+ exchange: string,
+ routingKey: string,
+ content: Buffer,
+ options: amqp.Options.Publish,
+): Promise<void> {
+ if (inFlightConfirms >= MAX_IN_FLIGHT_CONFIRMS) {
+ siemLoggingEventsTotal.inc({ result: "dropped_producer_full" });
+ return Promise.reject(
+ new Error(
+ `Refusing to publish: ${inFlightConfirms} SIEM events are already awaiting a broker confirm`,
+ ),
+ );
+ }
+
+ return new Promise((resolve, reject) => {
+ inFlightConfirms++;
+ const settle = (error: unknown): void => {
+ inFlightConfirms--;
+ if (error) reject(error);
+ else resolve();
+ };
+ const sent = ch.publish(
+ exchange,
+ routingKey,
+ content,
+ { ...options, mandatory: true },
+ settle,
+ );
+ // A false return only means the socket write buffer is full; the message is
+ // still queued client-side and the confirm callback still fires. The
+ // in-flight ceiling above is what actually bounds producer memory.
+ if (!sent) siemLoggingEventsTotal.inc({ result: "publish_backpressure" });
+ });
+}
+
+export async function enqueueSiemLoggingEvent(
+ orgId: string,
+ event: ScrapeActivityEvent,
+): Promise<void> {
+ await enqueueSiemLoggingEvents(orgId, [event]);
+}
+
+export async function enqueueSiemLoggingEvents(
+ orgId: string,
+ events: ScrapeActivityEvent[],
+): Promise<void> {
+ if (events.length === 0) return;
+
+ let ch: amqp.ConfirmChannel;
+ try {
+ ch = await getPublishChannel();
+ } catch (error) {
+ // No channel means nothing was published, so the whole call failed.
+ siemLoggingEventsTotal.inc({ result: "enqueue_failed" }, events.length);
+ throw error;
+ }
+
+ // allSettled, not all: these events are independent, and a single rejected
+ // confirm must not mislabel the ones the broker did accept. There is nowhere
+ // durable to park a rejected event β€” Firecrawl holds no scrape-derived state β€”
+ // so a failure here is a counted, logged loss rather than a retry.
+ const results = await Promise.allSettled(
+ events.map(event =>
+ publishConfirmed(
+ ch,
+ EXCHANGE,
+ QUEUE,
+ Buffer.from(
+ JSON.stringify({ orgId, event } satisfies SiemLoggingMessage),
+ ),
+ {
+ persistent: true,
+ contentType: "application/json",
+ messageId: event.scrape_id,
+ headers: { "x-siem-attempt": 0 },
+ },
+ ),
+ ),
+ );
+
+ const rejected = results.filter(
+ (result): result is PromiseRejectedResult => result.status === "rejected",
+ );
+ const queued = results.length - rejected.length;
+ if (queued > 0) siemLoggingEventsTotal.inc({ result: "queued" }, queued);
+ if (rejected.length > 0) {
+ siemLoggingEventsTotal.inc({ result: "enqueue_failed" }, rejected.length);
+ throw new AggregateError(
+ rejected.map(result => result.reason),
+ `Failed to publish ${rejected.length} of ${results.length} SIEM logging events`,
+ );
+ }
+}
+
+/** Smallest ladder rung that is at least `minDelayMs`, else the largest rung. */
+function rungForDelay(minDelayMs: number | undefined): number {
+ if (minDelayMs === undefined || minDelayMs <= 0) return RETRY_DELAYS_MS[0];
+ return (
+ RETRY_DELAYS_MS.find(delay => delay >= minDelayMs) ??
+ RETRY_DELAYS_MS[RETRY_DELAYS_MS.length - 1]
+ );
+}
+
+function attemptOf(msg: amqp.ConsumeMessage): number {
+ const raw = msg.properties.headers?.["x-siem-attempt"];
+ const attempt = typeof raw === "number" ? raw : Number(raw ?? 0);
+ return Number.isFinite(attempt) && attempt > 0 ? Math.floor(attempt) : 0;
+}
+
+// Republish onto a delay rung, then ack the original: a plain nack-with-requeue
+// redelivers immediately and would spin against a destination that is still down.
+// The republish is confirmed before the caller acks, so the window where the
+// original is gone and the retry never landed stays closed.
+async function scheduleRetry(
+ msg: amqp.ConsumeMessage,
+ retryAfterMs: number | undefined,
+): Promise<boolean> {
+ const attempt = attemptOf(msg);
+ if (attempt >= RETRY_DELAYS_MS.length) return false;
+ // Never step backwards down the ladder on a repeat failure.
+ const delayMs = Math.max(
+ rungForDelay(retryAfterMs),
+ RETRY_DELAYS_MS[attempt],
+ );
+ const ch = await getPublishChannel();
+ await publishConfirmed(
+ ch,
+ RETRY_EXCHANGE,
+ retryQueueName(delayMs),
+ msg.content,
+ {
+ persistent: true,
+ contentType: "application/json",
+ messageId: msg.properties.messageId,
+ headers: { "x-siem-attempt": attempt + 1 },
+ },
+ );
+ return true;
+}
+
+interface PendingBatch {
+ messages: amqp.ConsumeMessage[];
+ events: ScrapeActivityEvent[];
+ timer: NodeJS.Timeout;
+}
+
+// Valid JSON is not a valid message: without this, a body missing orgId would
+// open a batch keyed on undefined and send the handler looking up a destination
+// for an organization that doesn't exist.
+function isSiemLoggingMessage(value: unknown): value is SiemLoggingMessage {
+ if (typeof value !== "object" || value === null) return false;
+ const candidate = value as Partial<SiemLoggingMessage>;
+ return (
+ typeof candidate.orgId === "string" &&
+ candidate.orgId.length > 0 &&
+ typeof candidate.event === "object" &&
+ candidate.event !== null
+ );
+}
+
+// One open batch per organization: each org has its own destination, credentials
+// and token, so batches can never span orgs.
+const pendingBatches = new Map<string, PendingBatch>();
+
+function abandonPendingBatches(): void {
+ for (const batch of pendingBatches.values()) clearTimeout(batch.timer);
+ // Left unacked on purpose β€” the broker redelivers them to a live consumer.
+ pendingBatches.clear();
+}
+
+async function applyOutcome(
+ ch: amqp.Channel,
+ batch: PendingBatch,
+ outcome: SiemLoggingBatchOutcome,
+): Promise<void> {
+ if (outcome.disposition === "done") {
+ for (const msg of batch.messages) ch.ack(msg);
+ return;
+ }
+ if (outcome.disposition === "drop") {
+ for (const msg of batch.messages) ch.nack(msg, false, false);
+ siemLoggingEventsTotal.inc(
+ { result: "dead_lettered" },
+ batch.messages.length,
+ );
+ return;
+ }
+ let retried = 0;
+ let deadLettered = 0;
+ for (const msg of batch.messages) {
+ if (await scheduleRetry(msg, outcome.retryAfterMs)) {
+ ch.ack(msg);
+ retried++;
+ } else {
+ ch.nack(msg, false, false);
+ deadLettered++;
+ }
+ }
+ if (retried > 0) siemLoggingEventsTotal.inc({ result: "retried" }, retried);
+ if (deadLettered > 0) {
+ siemLoggingEventsTotal.inc({ result: "dead_lettered" }, deadLettered);
+ }
+}
+
+async function flushBatch(
+ ch: amqp.Channel,
+ handler: SiemLoggingBatchHandler,
+ orgId: string,
+): Promise<void> {
+ const batch = pendingBatches.get(orgId);
+ if (!batch) return;
+ pendingBatches.delete(orgId);
+ clearTimeout(batch.timer);
+
+ let outcome: SiemLoggingBatchOutcome;
+ try {
+ outcome = await handler(orgId, batch.events);
+ } catch (error) {
+ logger.error("SIEM logging batch handler threw", {
+ error,
+ orgId,
+ events: batch.events.length,
+ });
+ outcome = { disposition: "retry" };
+ }
+
+ try {
+ await applyOutcome(ch, batch, outcome);
+ } catch (error) {
+ // The channel died mid-batch, or a retry republish was nacked; either way
+ // the broker redelivers everything still unacked.
+ logger.warn("Failed to settle a SIEM logging batch", { error, orgId });
+ }
+}
+
+async function subscribe(
+ ch: amqp.Channel,
+ handler: SiemLoggingBatchHandler,
+ options: SiemLoggingConsumerOptions,
+): Promise<void> {
+ if (subscribed) return;
+ const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
+ const lingerMs = options.batchLingerMs ?? DEFAULT_BATCH_LINGER_MS;
+ // A batch can only ever hold messages the broker has handed us, so prefetch
+ // has to leave room for a full batch or it caps the batch below batchSize.
+ await ch.prefetch(options.prefetch ?? batchSize * 2, false);
+
+ await ch.consume(
+ QUEUE,
+ async msg => {
+ if (!msg) return;
+
+ let message: SiemLoggingMessage;
+ try {
+ const parsed: unknown = JSON.parse(msg.content.toString());
+ if (!isSiemLoggingMessage(parsed)) {
+ throw new Error("Message is not a SIEM logging event");
+ }
+ message = parsed;
+ } catch (error) {
+ logger.error("Failed to parse SIEM logging message", { error });
+ siemLoggingEventsTotal.inc({ result: "unparseable" });
+ ch.nack(msg, false, false);
+ return;
+ }
+
+ const orgId = message.orgId;
+ let batch = pendingBatches.get(orgId);
+ if (!batch) {
+ batch = {
+ messages: [],
+ events: [],
+ timer: setTimeout(() => {
+ void flushBatch(ch, handler, orgId);
+ }, lingerMs),
+ };
+ batch.timer.unref?.();
+ pendingBatches.set(orgId, batch);
+ }
+ batch.messages.push(msg);
+ batch.events.push(message.event);
+
+ if (batch.events.length >= batchSize) {
+ await flushBatch(ch, handler, orgId);
+ }
+ },
+ { noAck: false },
+ );
+
+ subscribed = true;
+ logger.info("Started consuming SIEM logging events", {
+ queue: QUEUE,
+ batchSize,
+ lingerMs,
+ });
+}
+
+export async function consumeSiemLoggingEvents(
+ handler: SiemLoggingBatchHandler,
+ options: SiemLoggingConsumerOptions = {},
+): Promise<void> {
+ closed = false;
+ registeredHandler = handler;
+ registeredOptions = options;
+ try {
+ const ch = await getConsumeChannel();
+ // Same race the reconnect path guards: a SIGTERM during a redeploy can land
+ // while the initial connection is still opening, and subscribing after that
+ // would leave a consumer attached to a transport shutdown already released.
+ if (closed) {
+ await closeConnection();
+ return;
+ }
+ await subscribe(ch, handler, options);
+ } catch (error) {
+ // A broker that is unavailable at worker startup has to enter the same retry
+ // loop as one that drops later. Throwing here instead left the replica
+ // permanently deaf, because only an already-established channel closing
+ // scheduled a reconnect.
+ logger.error("Failed to subscribe to SIEM logging events; will retry", {
+ error,
+ });
+ scheduleReconnect(RECONNECT_BASE_DELAY_MS);
+ }
+}
+
+async function closeConnection(): Promise<void> {
+ const conn = connection;
+ resetCachedState();
+ await conn?.close().catch(() => {});
+}
+
+export async function closeSiemLoggingTransport(): Promise<void> {
+ closed = true;
+ registeredHandler = null;
+ if (reconnectTimer) {
+ clearTimeout(reconnectTimer);
+ reconnectTimer = null;
+ }
+ await closeConnection();
+}
diff --git a/apps/api/src/lib/siem-logging/types.ts b/apps/api/src/lib/siem-logging/types.ts
index efb2c91ec..688f57e0a 100644
--- a/apps/api/src/lib/siem-logging/types.ts
+++ b/apps/api/src/lib/siem-logging/types.ts
@@ -76,13 +76,17 @@ export type ScrapeActivityResult =
export interface ScrapeActivityThreat {
decision: "allow" | "deny";
+ /**
+ * The rule that produced {@link decision}. Local-policy rules (blacklist,
+ * blocked-tld) deny without consulting a provider; only "risk-score" with a
+ * non-null {@link provider} is a provider-confirmed threat. The destination
+ * DCR transform keys ASim EventSeverity off exactly that distinction, so the
+ * three fields below always describe one decision β€” never a union across the
+ * decisions taken for one scrape.
+ */
rule: ThreatDecisionRule;
provider: ThreatProvider | null;
categories: string[];
- security_alert: {
- detected: boolean;
- category: string | null;
- };
}
export interface ScrapeActivityEvent {
@@ -123,7 +127,7 @@ export interface ScrapeActivityEvent {
threat?: ScrapeActivityThreat;
}
-export interface SiemLoggingJobData {
+export interface SiemLoggingMessage {
orgId: string;
event: ScrapeActivityEvent;
}
@@ -132,10 +136,17 @@ type SiemDeliveryErrorKind =
| "invalid_credentials"
| "schema_rejection"
| "rate_limited"
- | "delivery_error"
- | "payload_too_large";
+ | "delivery_error";
export class SiemDeliveryError extends Error {
+ /**
+ * Events the destination accepted before this failure. A batch is sent as
+ * several compressed chunks, so a late chunk failing does not undo the earlier
+ * ones β€” without this the caller would count accepted events as failed and
+ * then count them again as delivered when the batch retries.
+ */
+ public deliveredEvents = 0;
+
constructor(
public readonly kind: SiemDeliveryErrorKind,
message: string,
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 5bb2efa9a..5e74be8be 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -9,7 +9,6 @@ import {
getRedisConnection,
getBillingQueue,
getPrecrawlQueue,
- getSiemLoggingQueue,
precrawlQueueName,
} from "../queue-service";
import {
@@ -43,7 +42,8 @@ import { crawlGroup, resolveNewGroupBackend } from "../worker/nuq-router";
import { getACUCTeam } from "../../controllers/auth";
import { processEngpickerJob } from "../../lib/engpicker";
import { logRequest } from "../logging/log_job";
-import { processSiemLoggingJob } from "../siem-logging/worker";
+import { startSiemLoggingConsumer } from "../siem-logging/worker";
+import { closeSiemLoggingTransport } from "../../lib/siem-logging/transport";
const workerLockDuration = config.WORKER_LOCK_DURATION;
const workerStalledCheckInterval = config.WORKER_STALLED_CHECK_INTERVAL;
@@ -589,15 +589,27 @@ const processPrecrawlJob = async (token: string, job: Job) => {
let isShuttingDown = false;
+// The SIEM consumer is a subscription rather than a loop, so it has to be torn
+// down from the signal path: workerFun exits the process once its own jobs
+// drain, so anything after the worker Promise.all never runs. Unacked batches go
+// back to the broker for another replica.
+function shutdownSiemLoggingTransport(): void {
+ closeSiemLoggingTransport().catch(error => {
+ logger.warn("Failed to close the SIEM logging transport", { error });
+ });
+}
+
if (require.main === module) {
process.on("SIGINT", () => {
logger.info("Received SIGINT. Shutting down gracefully...");
isShuttingDown = true;
+ shutdownSiemLoggingTransport();
});
process.on("SIGTERM", () => {
logger.info("Received SIGTERM. Shutting down gracefully...");
isShuttingDown = true;
+ shutdownSiemLoggingTransport();
});
}
@@ -700,10 +712,11 @@ const BROWSER_ACTIVITY_INSERT_INTERVAL = 10000;
: (async () => {
logger.warn("PRECRAWL_TEAM_ID not set, skipping precrawl worker");
})();
- const siemLoggingWorkerPromise = workerFun(
- getSiemLoggingQueue(),
- processSiemLoggingJob,
- );
+ // A RabbitMQ consumer registration, not a polling loop: it returns once
+ // subscribed and the transport reconnects itself on a broker drop.
+ startSiemLoggingConsumer().catch(error => {
+ logger.error("Failed to start the SIEM logging consumer", { error });
+ });
const indexInserterInterval = setInterval(async () => {
if (isShuttingDown) {
@@ -785,7 +798,6 @@ const BROWSER_ACTIVITY_INSERT_INTERVAL = 10000;
await Promise.all([
billingWorkerPromise,
precrawlWorkerPromise,
- siemLoggingWorkerPromise,
engpickerPromise,
]);
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index 1c5bf0ccd..04dc0b263 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -3,7 +3,6 @@ import { config } from "../config";
import { logger } from "../lib/logger";
import IORedis from "ioredis";
import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";
-import type { SiemLoggingJobData } from "../lib/siem-logging/types";
import { addExtractJob, ExtractJobData } from "./extract-queue";
let loggingQueue: Queue;
@@ -12,7 +11,6 @@ let deepResearchQueue: Queue;
let generateLlmsTxtQueue: Queue;
let billingQueue: Queue;
let precrawlQueue: Queue;
-let siemLoggingQueue: Queue<SiemLoggingJobData>;
let redisConnection: IORedis;
export function getRedisConnection(): IORedis {
@@ -31,7 +29,6 @@ const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";
const deepResearchQueueName = "{deepResearchQueue}";
const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
-const siemLoggingQueueName = "{siemLoggingQueue}";
export async function addExtractJobToQueue(
extractId: string,
@@ -110,25 +107,3 @@ export function getPrecrawlQueue() {
}
return precrawlQueue;
}
-
-export function getSiemLoggingQueue(): Queue<SiemLoggingJobData> {
- if (!siemLoggingQueue) {
- siemLoggingQueue = new Queue<SiemLoggingJobData>(siemLoggingQueueName, {
- connection: getRedisConnection(),
- defaultJobOptions: {
- attempts: 8,
- backoff: {
- type: "exponential",
- delay: 1000,
- },
- removeOnComplete: {
- age: 60,
- },
- removeOnFail: {
- age: 24 * 60 * 60,
- },
- },
- });
- }
- return siemLoggingQueue;
-}
diff --git a/apps/api/src/services/siem-logging/worker.test.ts b/apps/api/src/services/siem-logging/worker.test.ts
index 4e8fac631..7b8a6b348 100644
--- a/apps/api/src/services/siem-logging/worker.test.ts
+++ b/apps/api/src/services/siem-logging/worker.test.ts
@@ -1,34 +1,47 @@
-import type { Job } from "bullmq";
-import { describe, expect, it, vi } from "vitest";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const { eventsTotal, failuresTotal } = vi.hoisted(() => ({
+ eventsTotal: { inc: vi.fn() },
+ failuresTotal: { inc: vi.fn() },
+}));
+vi.mock("../../lib/siem-logging/metrics", () => ({
+ siemLoggingEventsTotal: eventsTotal,
+ siemLoggingDeliveryFailuresTotal: failuresTotal,
+ siemLoggingDeliveryBatchesTotal: { inc: vi.fn() },
+}));
+
import {
SiemDeliveryError,
type OrgSiemLoggingConfig,
type ScrapeActivityEvent,
- type SiemLoggingJobData,
} from "../../lib/siem-logging/types";
-import { processSiemLoggingJob } from "./worker";
-
-const event: ScrapeActivityEvent = {
- schema_version: 1,
- event_type: "scrape_activity",
- scrape_id: "scrape-id",
- request_id: "request-id",
- endpoint: "scrape",
- team_id: "team-id",
- org_id: "org-id",
- api_key: { id: null, name: null },
- started_at: "2026-07-27T00:00:00.000Z",
- completed_at: "2026-07-27T00:00:01.000Z",
- url: "https://example.com",
- domain: "example.com",
- http_method: "GET",
- http_status: 200,
- result: "success",
- error: null,
- origin: "api",
- integration: null,
- zero_data_retention: false,
-};
+import { deliverSiemLoggingBatch } from "./worker";
+
+function event(scrapeId: string): ScrapeActivityEvent {
+ return {
+ schema_version: 1,
+ event_type: "scrape_activity",
+ scrape_id: scrapeId,
+ request_id: "request-id",
+ endpoint: "scrape",
+ team_id: "team-id",
+ org_id: "org-id",
+ api_key: { id: null, name: null },
+ started_at: "2026-07-27T00:00:00.000Z",
+ completed_at: "2026-07-27T00:00:01.000Z",
+ url: "https://example.com",
+ domain: "example.com",
+ http_method: "GET",
+ http_status: 200,
+ result: "success",
+ error: null,
+ origin: "api",
+ integration: null,
+ zero_data_retention: false,
+ };
+}
+
+const events = [event("scrape-1"), event("scrape-2")];
const config: OrgSiemLoggingConfig = {
orgId: "org-id",
@@ -46,76 +59,89 @@ const config: OrgSiemLoggingConfig = {
updatedAt: null,
};
-function createJob() {
- return {
- id: "job-id",
- data: { orgId: "org-id", event },
- extendLock: vi.fn().mockResolvedValue(1),
- moveToCompleted: vi.fn().mockResolvedValue(undefined),
- moveToFailed: vi.fn().mockResolvedValue(undefined),
- discard: vi.fn(),
- attemptsMade: 0,
- opts: {
- attempts: 8,
- backoff: { type: "exponential", delay: 1000 },
- },
- } as unknown as Job<SiemLoggingJobData>;
-}
+describe("SIEM logging batch delivery", () => {
+ beforeEach(() => {
+ eventsTotal.inc.mockClear();
+ failuresTotal.inc.mockClear();
+ });
-describe("SIEM logging worker", () => {
- it("loads the current secret at delivery time and completes the job", async () => {
- const job = createJob();
- const deliver = vi.fn().mockResolvedValue(undefined);
+ it("counts only the events a partly-delivered batch actually landed", async () => {
+ const error = new SiemDeliveryError("delivery_error", "chunk 2 failed");
+ // Azure accepted the first chunk before the call blew up.
+ error.deliveredEvents = 1;
- await processSiemLoggingJob("token", job, {
+ const outcome = await deliverSiemLoggingBatch("org-id", events, {
getConfig: vi.fn().mockResolvedValue(config),
- deliver,
+ deliver: vi.fn().mockRejectedValue(error),
});
- expect(deliver).toHaveBeenCalledWith(config.destination, [event]);
- expect(job.moveToCompleted).toHaveBeenCalledWith(
- { delivered: true },
- "token",
- false,
+ expect(outcome).toEqual({
+ disposition: "retry",
+ retryAfterMs: undefined,
+ });
+ expect(eventsTotal.inc).toHaveBeenCalledWith({ result: "delivered" }, 1);
+ expect(eventsTotal.inc).toHaveBeenCalledWith(
+ { result: "delivery_failed" },
+ 1,
+ );
+ expect(failuresTotal.inc).toHaveBeenCalledWith(
+ { reason: "delivery_error" },
+ 1,
);
- expect(job.moveToFailed).not.toHaveBeenCalled();
});
- it("skips queued events after logging is disabled", async () => {
- const job = createJob();
+ it("loads the current secret at delivery time and sends one batch", async () => {
+ const deliver = vi.fn().mockResolvedValue(2);
+ const getConfig = vi.fn().mockResolvedValue(config);
+
+ const outcome = await deliverSiemLoggingBatch("org-id", events, {
+ getConfig,
+ deliver,
+ });
+
+ expect(getConfig).toHaveBeenCalledWith("org-id");
+ expect(deliver).toHaveBeenCalledOnce();
+ expect(deliver).toHaveBeenCalledWith(config.destination, events);
+ expect(outcome).toEqual({ disposition: "done" });
+ });
+
+ it("acks queued events after logging is disabled", async () => {
const deliver = vi.fn();
- await processSiemLoggingJob("token", job, {
+ const outcome = await deliverSiemLoggingBatch("org-id", events, {
getConfig: vi.fn().mockResolvedValue({ ...config, enabled: false }),
deliver,
});
expect(deliver).not.toHaveBeenCalled();
- expect(job.moveToCompleted).toHaveBeenCalledWith(
- { skipped: true },
- "token",
- false,
- );
+ expect(outcome).toEqual({ disposition: "done" });
});
- it("leaves transient delivery failures retryable", async () => {
- const job = createJob();
+ it("retries when the configuration lookup fails", async () => {
+ const outcome = await deliverSiemLoggingBatch("org-id", events, {
+ getConfig: vi.fn().mockRejectedValue(new Error("db down")),
+ deliver: vi.fn(),
+ });
+
+ expect(outcome).toEqual({ disposition: "retry" });
+ });
- await processSiemLoggingJob("token", job, {
+ it("retries transient delivery failures", async () => {
+ const outcome = await deliverSiemLoggingBatch("org-id", events, {
getConfig: vi.fn().mockResolvedValue(config),
deliver: vi
.fn()
- .mockRejectedValue(new SiemDeliveryError("rate_limited", "retry")),
+ .mockRejectedValue(new SiemDeliveryError("delivery_error", "boom")),
});
- expect(job.discard).not.toHaveBeenCalled();
- expect(job.moveToFailed).toHaveBeenCalledOnce();
+ expect(outcome).toEqual({
+ disposition: "retry",
+ retryAfterMs: undefined,
+ });
});
- it("does not retry before the provider's Retry-After delay", async () => {
- const job = createJob();
-
- await processSiemLoggingJob("token", job, {
+ it("surfaces the provider's Retry-After to the retry ladder", async () => {
+ const outcome = await deliverSiemLoggingBatch("org-id", events, {
getConfig: vi.fn().mockResolvedValue(config),
deliver: vi
.fn()
@@ -124,23 +150,17 @@ describe("SIEM logging worker", () => {
),
});
- expect(job.opts.backoff).toEqual({ type: "fixed", delay: 15_000 });
- expect(job.moveToFailed).toHaveBeenCalledOnce();
+ expect(outcome).toEqual({ disposition: "retry", retryAfterMs: 15_000 });
});
- it("discards permanent destination failures", async () => {
- const job = createJob();
-
- await processSiemLoggingJob("token", job, {
- getConfig: vi.fn().mockResolvedValue(config),
- deliver: vi
- .fn()
- .mockRejectedValue(
- new SiemDeliveryError("invalid_credentials", "invalid"),
- ),
- });
+ it("dead-letters permanent destination failures", async () => {
+ for (const kind of ["invalid_credentials", "schema_rejection"] as const) {
+ const outcome = await deliverSiemLoggingBatch("org-id", events, {
+ getConfig: vi.fn().mockResolvedValue(config),
+ deliver: vi.fn().mockRejectedValue(new SiemDeliveryError(kind, kind)),
+ });
- expect(job.discard).toHaveBeenCalledOnce();
- expect(job.moveToFailed).toHaveBeenCalledOnce();
+ expect(outcome).toEqual({ disposition: "drop" });
+ }
});
});
diff --git a/apps/api/src/services/siem-logging/worker.ts b/apps/api/src/services/siem-logging/worker.ts
index c3075779e..3a8edca5c 100644
--- a/apps/api/src/services/siem-logging/worker.ts
+++ b/apps/api/src/services/siem-logging/worker.ts
@@ -1,29 +1,34 @@
-import type { Job } from "bullmq";
-import { config } from "../../config";
import { sendAzureSentinelEvents } from "../../lib/siem-logging/azure-sentinel";
import {
siemLoggingDeliveryFailuresTotal,
siemLoggingEventsTotal,
} from "../../lib/siem-logging/metrics";
import { getOrgSiemLoggingConfig } from "../../lib/siem-logging/store";
+import {
+ consumeSiemLoggingEvents,
+ type SiemLoggingBatchOutcome,
+} from "../../lib/siem-logging/transport";
import {
SiemDeliveryError,
type AzureSentinelDestination,
type OrgSiemLoggingConfig,
type ScrapeActivityEvent,
- type SiemLoggingJobData,
} from "../../lib/siem-logging/types";
import { logger as _logger } from "../../lib/logger";
-interface WorkerDependencies {
+interface HandlerDependencies {
getConfig: (orgId: string) => Promise<OrgSiemLoggingConfig | null>;
+ /** Resolves with the number of events the destination actually accepted. */
deliver: (
destination: AzureSentinelDestination,
events: ScrapeActivityEvent[],
- ) => Promise<void>;
+ ) => Promise<number>;
}
-const defaultDependencies: WorkerDependencies = {
+const defaultDependencies: HandlerDependencies = {
+ // Read the destination at delivery time from the primary, never from the
+ // queued message: a rotated secret or a disabled config has to take effect on
+ // events that were already in flight.
getConfig: orgId =>
getOrgSiemLoggingConfig(orgId, { fresh: true, primary: true }),
deliver: sendAzureSentinelEvents,
@@ -32,90 +37,77 @@ const defaultDependencies: WorkerDependencies = {
const permanentFailureKinds = new Set([
"invalid_credentials",
"schema_rejection",
- "payload_too_large",
]);
-function asError(error: unknown): Error {
- return error instanceof Error ? error : new Error(String(error));
-}
+export async function deliverSiemLoggingBatch(
+ orgId: string,
+ events: ScrapeActivityEvent[],
+ deps: HandlerDependencies = defaultDependencies,
+): Promise<SiemLoggingBatchOutcome> {
+ const logger = _logger.child({ module: "siem-logging-worker", orgId });
-function configuredRetryDelay(job: Job<SiemLoggingJobData>): number {
- const backoff = job.opts.backoff;
- if (typeof backoff === "number") return backoff;
- if (!backoff) return 0;
- const delay = backoff.delay ?? 0;
- if (backoff.type === "exponential") {
- return delay * 2 ** job.attemptsMade;
+ let config: OrgSiemLoggingConfig | null;
+ try {
+ config = await deps.getConfig(orgId);
+ } catch (error) {
+ logger.error("Failed to load SIEM logging configuration", { error });
+ return { disposition: "retry" };
}
- return delay;
-}
-function applyRetryAfter(
- job: Job<SiemLoggingJobData>,
- error: SiemDeliveryError,
-): void {
- if (
- error.kind !== "rate_limited" ||
- error.retryAfterMs === undefined ||
- error.retryAfterMs <= 0
- ) {
- return;
+ if (!config?.enabled) {
+ siemLoggingEventsTotal.inc({ result: "skipped_disabled" }, events.length);
+ return { disposition: "done" };
}
- job.opts.backoff = {
- type: "fixed",
- delay: Math.max(configuredRetryDelay(job), error.retryAfterMs),
- };
-}
-
-export async function processSiemLoggingJob(
- token: string,
- job: Job<SiemLoggingJobData>,
- deps: WorkerDependencies = defaultDependencies,
-): Promise<void> {
- const logger = _logger.child({
- module: "siem-logging-worker",
- jobId: job.id,
- orgId: job.data.orgId,
- });
- const extendLockInterval = setInterval(() => {
- void job.extendLock(token, config.JOB_LOCK_EXTENSION_TIME).catch(error => {
- logger.warn("Failed to extend SIEM logging job lock", { error });
- });
- }, config.JOB_LOCK_EXTEND_INTERVAL);
- extendLockInterval.unref?.();
try {
- const currentConfig = await deps.getConfig(job.data.orgId);
- if (!currentConfig?.enabled) {
- siemLoggingEventsTotal.inc({ result: "skipped_disabled" });
- await job.moveToCompleted({ skipped: true }, token, false);
- return;
+ // Count what the destination took, not what we handed over: the sender skips
+ // events it can never deliver, and those are already counted as dropped.
+ const delivered = await deps.deliver(config.destination, events);
+ if (delivered > 0) {
+ siemLoggingEventsTotal.inc({ result: "delivered" }, delivered);
}
-
- await deps.deliver(currentConfig.destination, [job.data.event]);
- siemLoggingEventsTotal.inc({ result: "delivered" });
- await job.moveToCompleted({ delivered: true }, token, false);
+ return { disposition: "done" };
} catch (error) {
const reason =
error instanceof SiemDeliveryError ? error.kind : "delivery_error";
- siemLoggingEventsTotal.inc({ result: "delivery_failed" });
- siemLoggingDeliveryFailuresTotal.inc({ reason });
- logger.error("SIEM logging event delivery failed", {
+ // A batch goes out as several chunks, so a failure part-way through still
+ // left events with the destination. Count those as delivered and only the
+ // remainder as failed, or the retry double-counts them.
+ const delivered =
+ error instanceof SiemDeliveryError ? error.deliveredEvents : 0;
+ const failed = Math.max(0, events.length - delivered);
+ if (delivered > 0) {
+ siemLoggingEventsTotal.inc({ result: "delivered" }, delivered);
+ }
+ if (failed > 0) {
+ siemLoggingEventsTotal.inc({ result: "delivery_failed" }, failed);
+ siemLoggingDeliveryFailuresTotal.inc({ reason }, failed);
+ }
+ logger.error("SIEM logging batch delivery failed", {
error,
reason,
- scrapeId: job.data.event.scrape_id,
+ events: events.length,
+ deliveredBeforeFailure: delivered,
});
+
if (
error instanceof SiemDeliveryError &&
permanentFailureKinds.has(error.kind)
) {
- job.discard();
+ // Bad credentials or a rejected schema will fail identically forever;
+ // dead-letter so an operator sees it instead of burning the ladder.
+ return { disposition: "drop" };
}
- if (error instanceof SiemDeliveryError) {
- applyRetryAfter(job, error);
- }
- await job.moveToFailed(asError(error), token, false);
- } finally {
- clearInterval(extendLockInterval);
+ return {
+ disposition: "retry",
+ retryAfterMs:
+ error instanceof SiemDeliveryError ? error.retryAfterMs : undefined,
+ };
}
}
+
+export async function startSiemLoggingConsumer(): Promise<void> {
+ await consumeSiemLoggingEvents((orgId, events) =>
+ deliverSiemLoggingBatch(orgId, events),
+ );
+}
diff --git a/apps/siem/azure-sentinel/DCR.json b/apps/siem/azure-sentinel/DCR.json
index ad2a61002..102168ead 100644
--- a/apps/siem/azure-sentinel/DCR.json
+++ b/apps/siem/azure-sentinel/DCR.json
@@ -106,7 +106,7 @@
{
"streams": ["Custom-FirecrawlScrapeActivity"],
"destinations": ["firecrawlWorkspace"],
- "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tobool(threat.security_alert.detected), \"High\", tostring(threat.decision) == \"deny\", \"Low\", result == \"success\", \"Informational\", \"Low\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)",
+ "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tostring(threat.rule) == \"risk-score\" and isnotempty(tostring(threat.provider)), \"High\", tostring(threat.decision) == \"deny\", \"Low\", \"Informational\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)",
"outputStream": "Microsoft-ASimWebSessionLogs"
}
],
diff --git a/apps/siem/azure-sentinel/README.md b/apps/siem/azure-sentinel/README.md
index 7bb0bf43d..6b1efe23b 100644
--- a/apps/siem/azure-sentinel/README.md
+++ b/apps/siem/azure-sentinel/README.md
@@ -14,15 +14,58 @@ Sentinel through the Azure Monitor Logs Ingestion API.
The database schema is maintained as a migration in the Firecrawl database
repository, not in this package.
+## Severity
+
+The DCR transform derives ASim `EventSeverity` from how a block was decided,
+not from whether a category happens to be present:
+
+- `High` β€” denied by `risk-score` with a classifier verdict, i.e. a
+ provider-confirmed threat.
+- `Low` β€” denied by any other rule (local blacklist, blocked TLD, classifier
+ unavailable under a closed failure policy).
+- `Informational` β€” everything else, including scrapes that simply failed.
+
+`Medium` is left unused so a workspace can claim it without colliding.
+
## Runtime
`PUT /v2/team/siem` stores an organization destination and its encrypted,
-write-only client secret. Scrape activity is placed on a shared BullMQ queue;
-index-worker replicas consume that queue and retry transient delivery failures.
-Queue jobs contain the organization ID and event, never the destination secret.
+write-only client secret. Scrape activity is published to the RabbitMQ queue
+`siem.logging.events`; index-worker replicas consume it, group events by
+organization, and deliver one gzip batch per org. Messages carry the
+organization ID and event, never the destination secret β€” the destination and
+its secret are read fresh at delivery time, so disabling logging or rotating a
+secret takes effect on events already in flight.
+
+Operational shape of that queue:
+
+- Bounded at 100k messages with `drop-head`, so a destination outage sheds the
+ oldest events instead of growing without limit. The retry queues carry the
+ same cap, and `siem.logging.dlq` is capped at 50k with a 7-day TTL β€” it exists
+ for an operator to read, not as indefinite storage.
+- Transient failures ride a fixed-delay retry ladder (5s, 30s, 5m, 30m) built
+ from TTL queues that dead-letter back onto the main queue; a destination's
+ `Retry-After` rounds up to the smallest rung that satisfies it.
+- Bad credentials or a rejected DCR schema dead-letter straight to
+ `siem.logging.dlq` rather than burning the ladder. An event too large to
+ compress under 1 MB on its own is dropped and counted, not dead-lettered β€”
+ failing its batch would lose up to 199 deliverable events alongside it.
+- Retries are at-least-once: a batch that fails partway through its ≀1 MB
+ sub-batches will resend the sub-batches that already landed.
+- Publishes are confirmed and `mandatory`, and at most 10k may await a confirm
+ at once. Beyond that the producer sheds events rather than accumulating them
+ in the API process, which is the same bound the queue has, applied upstream.
+
+Queue depth comes from `rabbitmq_prometheus`.
+`firecrawl_siem_logging_events_total` breaks out per-event outcomes: `queued`,
+`delivered`, `retried`, `dead_lettered`, `skipped_disabled`, `enqueue_failed`,
+`delivery_failed`, `unparseable`, `unroutable`, `dropped_oversized` and
+`dropped_producer_full`. The last three should normally be flat at zero; any of
+them moving means events are being lost rather than delayed.
Set `SIEM_LOGGING_ENCRYPTION_KEY` in API and index-worker environments. If
-partner egress is required, also set `PARTNER_EGRESS_PROXY_URL`.
+partner egress is required, also set `PARTNER_EGRESS_PROXY_URL` and allowlist
+`login.microsoftonline.com` plus `*.ingest.monitor.azure.com` on the proxy.
Use `POST /v2/team/siem/test` to validate credentials and the destination
schema after deploying the Azure artifacts.
diff --git a/apps/siem/azure-sentinel/azuredeploy.json b/apps/siem/azure-sentinel/azuredeploy.json
index 125a09ea8..186fd3354 100644
--- a/apps/siem/azure-sentinel/azuredeploy.json
+++ b/apps/siem/azure-sentinel/azuredeploy.json
@@ -145,7 +145,7 @@
{
"streams": ["Custom-FirecrawlScrapeActivity"],
"destinations": ["firecrawlWorkspace"],
- "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tobool(threat.security_alert.detected), \"High\", tostring(threat.decision) == \"deny\", \"Low\", result == \"success\", \"Informational\", \"Low\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)",
+ "transformKql": "source | project TimeGenerated = completed_at, EventStartTime = started_at, EventEndTime = completed_at, EventOriginalUid = scrape_id, NetworkSessionId = request_id, EventVendor = \"Firecrawl\", EventProduct = \"Firecrawl\", EventSchema = \"WebSession\", EventSchemaVersion = \"0.2.7\", EventType = \"HTTPsession\", EventCount = 1, EventResult = iff(result == \"success\", \"Success\", \"Failure\"), EventResultDetails = iff(isnull(http_status), \"\", tostring(http_status)), EventOriginalResultDetails = coalesce(tostring(error.code), result), EventOriginalType = endpoint, EventMessage = tostring(error.message), EventSeverity = case(tostring(threat.decision) == \"deny\" and tostring(threat.rule) == \"risk-score\" and isnotempty(tostring(threat.provider)), \"High\", tostring(threat.decision) == \"deny\", \"Low\", \"Informational\"), Dvc = \"Firecrawl\", DvcAction = case(isnull(threat), \"\", tostring(threat.decision) == \"deny\", \"Deny\", \"Allow\"), RuleName = tostring(threat.rule), ThreatCategory = tostring(threat.categories[0]), Url = url, DstDomain = domain, DstFQDN = domain, HttpHost = domain, HttpRequestMethod = http_method, NetworkApplicationProtocol = iff(url startswith \"https://\", \"HTTPS\", \"HTTP\"), NetworkDuration = toint(datetime_diff(\"millisecond\", completed_at, started_at)), SrcUsername = tostring(audit_metadata.username), AdditionalFields = pack(\"schema_version\", schema_version, \"event_type\", event_type, \"team_id\", team_id, \"org_id\", org_id, \"api_key_id\", tostring(api_key.id), \"api_key_name\", tostring(api_key.name), \"origin\", origin, \"integration\", integration, \"zero_data_retention\", zero_data_retention, \"threat\", threat)",
"outputStream": "Microsoft-ASimWebSessionLogs"
}
],
diff --git a/apps/siem/azure-sentinel/sample-event.json b/apps/siem/azure-sentinel/sample-event.json
index e790e77a4..13e24402f 100644
--- a/apps/siem/azure-sentinel/sample-event.json
+++ b/apps/siem/azure-sentinel/sample-event.json
@@ -29,11 +29,7 @@
"decision": "allow",
"rule": "default-allow",
"provider": "google-web-risk",
- "categories": [],
- "security_alert": {
- "detected": false,
- "category": null
- }
+ "categories": []
}
}
]