Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

35.7 KB
Raw Blame
import express from 'express';
import cors from 'cors';
import crypto from 'crypto';
import http from 'http';
import { WebSocketServer } from 'ws';
import { Kafka } from 'kafkajs';
import Redis from 'ioredis';
import pino from 'pino';
import rateLimit from 'express-rate-limit';

import { PushManager } from './push';
import { resolveTicketSecret, verifyRealtimeTicket } from './auth';
import { setupTelemetry, type Shutdown } from './telemetry';
import { resolveKafkaTransport } from './kafkaTransport.js';

const log = pino({ level: process.env.LOG_LEVEL || 'info' });

const PORT = parseInt(process.env.PORT || '8086', 10);
const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379/4';
const KAFKA_BROKERS = (process.env.KAFKA_BOOTSTRAP_SERVERS || 'localhost:9092').split(',');
const KAFKA_TOPIC_FUSED = process.env.KAFKA_TOPIC_FUSED || 'aisoc.alerts.fused';
// T1.4 (v8.0): graph-update channel. The ingest-side graph writer publishes
// one envelope per node/edge upsert on `security.graph_updates`; we fan out
// to WebSocket clients subscribed to `/ws/graph`. Topic name matches the
// default in services/ingest/internal/config/config.go (env
// `AISOC_GRAPH_UPDATES_TOPIC`) so the two services agree without manual
// plumbing. Set to an empty string to disable the consumer entirely (useful
// in tests that don't spin up Kafka for graph traffic).
const KAFKA_TOPIC_GRAPH_UPDATES =
  process.env.AISOC_GRAPH_UPDATES_TOPIC ||
  process.env.KAFKA_TOPIC_GRAPH_UPDATES ||
  'security.graph_updates';
const VAPID_PUBLIC_KEY = process.env.VAPID_PUBLIC_KEY || '';
const VAPID_PRIVATE_KEY = process.env.VAPID_PRIVATE_KEY || '';
const VAPID_SUBJECT = process.env.VAPID_SUBJECT || 'mailto:[email protected]';
const PUSH_REDIS = new Redis(REDIS_URL);

// Mirror the shared Python helper in services/api/app/core/cors.py:
//   1. AISOC_CORS_ORIGINS (canonical, comma-separated)
//   2. CORS_ORIGINS (legacy alias kept for Helm charts / dev scripts)
//   3. Default allow-list (local dev only — deployed origins set the env var)
// SSE + WebSocket connections from the console carry the auth cookie, so
// allow_credentials is effectively in play here. If an operator sets the
// allow-list to "*" we refuse to start in production rather than silently
// turn /sse into a cross-origin CSRF target.
const DEFAULT_CORS_ORIGINS = [
  'http://localhost:3000',
  'http://localhost:3001',
  'http://127.0.0.1:3000',
  'http://127.0.0.1:3001',
];

function resolveCorsOrigins(): string[] {
  for (const env of ['AISOC_CORS_ORIGINS', 'CORS_ORIGINS']) {
    const raw = (process.env[env] || '').trim();
    if (!raw) continue;
    const parts = raw.split(',').map((s) => s.trim()).filter(Boolean);
    if (parts.length > 0) return parts;
  }
  return [...DEFAULT_CORS_ORIGINS];
}

function isProductionEnv(): boolean {
  const env = (process.env.AISOC_ENV || process.env.ENVIRONMENT || process.env.APP_ENV || '')
    .trim()
    .toLowerCase();
  return env === 'production' || env === 'prod';
}

const CORS_ORIGINS = resolveCorsOrigins();
const CORS_ALLOW_CREDENTIALS = !CORS_ORIGINS.includes('*');
if (CORS_ORIGINS.includes('*')) {
  if (isProductionEnv()) {
    // Fail loud at startup instead of silently exposing /sse + /internal/*.
    throw new Error(
      'realtime: refusing to start with wildcard CORS origin in production. ' +
        'Set AISOC_CORS_ORIGINS to an explicit allow-list.',
    );
  }
  log.warn(
    'CORS wildcard origin in dev — disabling credentials for the SSE + internal routes',
  );
}
log.info(
  { origins: CORS_ORIGINS, allowCredentials: CORS_ALLOW_CREDENTIALS },
  'CORS configured',
);

// --- Realtime ticket auth (Issue #239) -------------------------------------
// Resolve the shared HS256 secret used to verify the short-lived tickets the
// API mints at POST /api/v1/realtime/ticket. `null` means we are NOT configured
// to verify (production with AISOC_REALTIME_JWT_SECRET unset/insecure) — in that
// case every WS upgrade and SSE request is rejected (fail closed) rather than
// silently accepting unauthenticated subscribers (the bug Issue #239 closes).
const REALTIME_TICKET_SECRET = resolveTicketSecret();
if (REALTIME_TICKET_SECRET === null) {
  log.error(
    'AISOC_REALTIME_JWT_SECRET is unset or insecure in a production environment — ' +
      'realtime WS/SSE connections will be rejected until a real secret is wired.',
  );
}

/**
 * Verify the `?token=` ticket on a realtime request URL. Returns the verified
 * tenant_id, or `null` if the request must be rejected. Centralises the
 * fail-closed decision for both the WS upgrade and the SSE handler.
 */
function authenticateRealtime(url: URL): string | null {
  if (REALTIME_TICKET_SECRET === null) return null;
  const token = url.searchParams.get('token') || '';
  const claims = verifyRealtimeTicket(token, REALTIME_TICKET_SECRET);
  if (!claims) return null;
  return claims.tenant_id;
}

const pushManager = new PushManager({
  redis: PUSH_REDIS,
  logger: log,
  vapidPublicKey: VAPID_PUBLIC_KEY,
  vapidPrivateKey: VAPID_PRIVATE_KEY,
  vapidSubject: VAPID_SUBJECT,
});

// --- Express setup ---
const app = express();
// Use an explicit allow-list rather than the default reflective CORS. The
// console (apps/web) sends a session cookie on `/v1/push/*` and `/internal/*`
// calls go agent-to-realtime over the private network; we must never echo
// back a wildcard with credentials enabled.
app.use(
  cors({
    origin(origin, callback) {
      if (!origin) {
        // Same-origin / curl / health probes — let them through.
        callback(null, true);
        return;
      }
      if (CORS_ORIGINS.includes('*') || CORS_ORIGINS.includes(origin)) {
        callback(null, true);
        return;
      }
      callback(new Error(`Origin ${origin} not allowed by CORS`));
    },
    credentials: CORS_ALLOW_CREDENTIALS,
    methods: ['GET', 'POST', 'OPTIONS'],
    allowedHeaders: [
      'Accept',
      'Authorization',
      'Content-Type',
      'X-Tenant-ID',
      'X-Internal-Token',
    ],
    maxAge: 300,
  }),
);
app.use(express.json());

const server = http.createServer(app);

// --- WebSocket server ---
// Accept both `/ws` (legacy) and `/ws/:channel` (preferred). The channel lets
// callers say up-front what they care about (alerts, cases, agents, insights,
// graph, all) so we can avoid spamming a panel that only renders alerts with
// case/agent traffic.
//
// The `insights` channel (T3.1) is consumed by
// apps/web/src/app/(app)/dashboards/soc-insights/page.tsx to know when to
// re-fetch the aggregator endpoint. The payload itself stays small —
// it's a poke, not a data delivery.
//
// The `graph` channel (T1.4, v8.0) is consumed by the Investigation Rail and
// Attack Chain views to light up new entity nodes / edges in near-real-time.
// Unlike `insights`, the payload IS the delivery: each message carries the
// full GraphUpdate envelope from `security.graph_updates` (entity_id,
// change_type, label, rel_type, from, to, properties, schema_version), so
// the client can apply the mutation locally without re-querying Neo4j.
type Channel = 'alerts' | 'cases' | 'agents' | 'insights' | 'graph' | 'all';
const VALID_CHANNELS: Channel[] = ['alerts', 'cases', 'agents', 'insights', 'graph', 'all'];

const wss = new WebSocketServer({ noServer: true });

server.on('upgrade', (req, socket, head) => {
  const url = new URL(req.url || '/', `http://localhost`);
  const parts = url.pathname.split('/').filter(Boolean);
  // /ws            → channel "all"
  // /ws/<channel>  → that channel (must be in VALID_CHANNELS)
  if (parts[0] !== 'ws') {
    socket.destroy();
    return;
  }
  const requested = (parts[1] as Channel | undefined) || 'all';
  if (!VALID_CHANNELS.includes(requested)) {
    socket.destroy();
    return;
  }
  // Authenticate the connection from the short-lived ticket BEFORE completing
  // the upgrade. Reject (401 + close) when the ticket is missing/invalid or the
  // service is not configured to verify — never accept an anonymous subscriber.
  const verifiedTenant = authenticateRealtime(url);
  if (verifiedTenant === null) {
    socket.write('HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n');
    socket.destroy();
    return;
  }
  wss.handleUpgrade(req, socket, head, (ws) => {
    (ws as any)._aisocChannel = requested;
    // Tenant is derived from the verified ticket claim, NOT the URL query, so a
    // client can never subscribe to another tenant's fan-out.
    (ws as any)._aisocTenant = verifiedTenant;
    wss.emit('connection', ws, req);
  });
});

// Client registry keyed by tenantId. We additionally tag each socket with the
// channel it subscribed to so broadcasts can filter cheaply.
const clients = new Map<string, Set<any>>();

// Simple IP-based connection rate limiter: max 20 WS connections per IP per minute.
const wsConnectCounts = new Map<string, { count: number; resetAt: number }>();
const WS_RATE_LIMIT = 20;
const WS_RATE_WINDOW_MS = 60_000;

function wsRateLimitExceeded(ip: string): boolean {
  const now = Date.now();
  let entry = wsConnectCounts.get(ip);
  if (!entry || now > entry.resetAt) {
    entry = { count: 0, resetAt: now + WS_RATE_WINDOW_MS };
    wsConnectCounts.set(ip, entry);
  }
  entry.count += 1;
  return entry.count > WS_RATE_LIMIT;
}

wss.on('connection', (ws, req) => {
  const ip = (req.headers['x-forwarded-for'] as string | undefined)?.split(',')[0]?.trim()
    || req.socket.remoteAddress
    || 'unknown';
  if (wsRateLimitExceeded(ip)) {
    ws.close(1008, 'Rate limit exceeded');
    return;
  }
  // Tenant comes from the verified ticket claim stashed during the upgrade
  // (see server.on('upgrade')), never from a client-supplied query parameter.
  const tenantId = (ws as any)._aisocTenant || 'default';
  const channel: Channel = (ws as any)._aisocChannel ?? 'all';

  if (!clients.has(tenantId)) {
    clients.set(tenantId, new Set());
  }
  clients.get(tenantId)!.add(ws);
  log.info(
    { tenantId, channel, totalClients: clients.get(tenantId)!.size },
    'WebSocket client connected',
  );

  ws.on('close', () => {
    clients.get(tenantId)?.delete(ws);
    log.info({ tenantId, channel }, 'WebSocket client disconnected');
  });

  ws.send(JSON.stringify({ type: 'connected', tenantId, channel }));
});

/**
 * Map message type → which channel(s) should receive it. "all" always wins.
 * Add new mappings here when you wire additional Kafka topics through.
 */
const CHANNEL_FOR_TYPE: Record<string, Channel[]> = {
  'alert.fused': ['alerts', 'all'],
  'case.updated': ['cases', 'all', 'insights'],
  'agent.event': ['agents', 'all'],
  // T3.1: SOC Insights dashboard tick. Both the dedicated `insights`
  // subscribers and any `all`-subscribed admin consoles get the poke.
  'insights_updated': ['insights', 'all'],
  // T1.4 (v8.0): graph-mutation fan-out from the ingest-side writer.
  // `graph.update` carries the full GraphUpdate envelope; `graph` and
  // `all` subscribers both receive it. We deliberately do NOT include
  // `insights` here — the SOC Insights dashboard is a stats poke, not a
  // graph-render surface.
  'graph.update': ['graph', 'all'],
};

function broadcastToTenant(tenantId: string, message: { type: string } & Record<string, unknown>) {
  const tenantClients = clients.get(tenantId);
  if (!tenantClients) return;

  const allowed = CHANNEL_FOR_TYPE[message.type] ?? ['all'];
  const payload = JSON.stringify(message);
  for (const client of tenantClients) {
    if (client.readyState !== 1 /* OPEN */) continue;
    const subscribed: Channel = client._aisocChannel ?? 'all';
    if (subscribed === 'all' || allowed.includes(subscribed)) {
      client.send(payload);
    }
  }
}

// --- Insights tick (T3.1) ---------------------------------------------------
// The SOC Insights dashboard (apps/web/src/app/(app)/dashboards/soc-insights)
// renders 7 rolling-window tiles that need to stay fresh while an analyst
// keeps the page open. Rather than have the client poll on a setInterval
// (which fights React's strict-mode double-renders and burns auth headers),
// we drive the cadence from the server: every 30s, broadcast a tiny
// `insights_updated` poke to every tenant with an active socket. The web
// client listens on the `insights` channel and uses the poke to revalidate
// its SWR cache against /v1/insights/soc.
//
// The poke is intentionally payload-free so we don't have to keep the
// realtime service in sync with the aggregator's schema; the dashboard
// pulls fresh numbers from the API.
const INSIGHTS_TICK_MS = Number.parseInt(
  process.env.INSIGHTS_TICK_MS || '30000',
  10,
);

function broadcastInsightsTick(): void {
  for (const tenantId of clients.keys()) {
    broadcastToTenant(tenantId, {
      type: 'insights_updated',
      reason: 'tick',
      timestamp: new Date().toISOString(),
    });
  }
}

// In tests we set INSIGHTS_TICK_MS=0 to suppress the timer; production keeps
// the 30s cadence the dashboard spec calls for.
if (INSIGHTS_TICK_MS > 0) {
  setInterval(broadcastInsightsTick, INSIGHTS_TICK_MS).unref?.();
}

// Rate limiters using express-rate-limit (recognised by CodeQL js/missing-rate-limiting).
const sseRateLimit = rateLimit({
  windowMs: 60_000,
  max: 20,
  standardHeaders: true,
  legacyHeaders: false,
  message: { error: 'Rate limit exceeded' },
});

const internalEventRateLimit = rateLimit({
  windowMs: 60_000,
  max: 200,
  standardHeaders: true,
  legacyHeaders: false,
  message: { error: 'Rate limit exceeded' },
});

const internalPushRateLimit = rateLimit({
  windowMs: 60_000,
  max: 200,
  standardHeaders: true,
  legacyHeaders: false,
  message: { error: 'Rate limit exceeded' },
});

// --- SSE endpoint ---
app.get('/sse', sseRateLimit, (req, res) => {
  // Authenticate from the short-lived ticket and derive the tenant from the
  // verified claim. Reject (401) when the ticket is missing/invalid or the
  // service is not configured to verify — SSE is no longer an open stream.
  const reqUrl = new URL(req.originalUrl, 'http://localhost');
  const tenantId = authenticateRealtime(reqUrl);
  if (tenantId === null) {
    res.status(401).json({ error: 'unauthorized' });
    return;
  }

  res.setHeader('Content-Type', 'text/event-stream');
  res.setHeader('Cache-Control', 'no-cache');
  res.setHeader('Connection', 'keep-alive');
  // CORS headers for SSE are emitted by the `cors()` middleware on lines
  // ~118–143 above, which the `cors` npm package implements safely (CodeQL
  // recognises `cors({ origin: <function> })` as a sanitiser). SSE auth is
  // done via the short-lived `?token=` ticket verified above (Issue #239),
  // not via a session cookie, so we deliberately do not enable
  // `Access-Control-Allow-Credentials` on this endpoint — that eliminates
  // the entire "CORS misconfiguration for credentials transfer" attack
  // surface that CodeQL warns about.
  res.flushHeaders();

  const heartbeat = setInterval(() => {
    res.write('event: heartbeat\ndata: {}\n\n');
  }, 30000);

  // Register as SSE client via Redis pub/sub.
  //
  // `subscribe()` returns a promise, and its rejection was unhandled: Redis
  // being briefly unreachable when a client opened an SSE stream raised an
  // unhandled rejection, which Node terminates the process on. One
  // subscriber's bad luck took down the fan-out for every connected tenant.
  // Log it and leave the stream open on heartbeats instead.
  const sub = new Redis(REDIS_URL);
  sub.subscribe(`aisoc:events:${tenantId}`).catch((err: unknown) => {
    log.error({ err, tenantId }, 'SSE: Redis subscribe failed; stream will carry heartbeats only');
  });
  sub.on('message', (_channel: string, message: string) => {
    res.write(`data: ${message}\n\n`);
  });

  req.on('close', () => {
    clearInterval(heartbeat);
    sub.disconnect();
  });
});

// Single shared Kafka client so we don't spin up a second TCP fan-out for
// each topic; kafkajs multiplexes consumers internally.
const kafka = new Kafka({
  clientId: 'aisoc-realtime',
  brokers: KAFKA_BROKERS,
  retry: { retries: 5 },
  // kafkajs leaves `ssl` and `sasl` undefined, which is plaintext, and this
  // client had neither — so the service fanning normalized alerts out to
  // every connected browser read them off the broker in the clear. The
  // resolver throws in a protected environment rather than returning a
  // default, so a misconfigured deployment fails at startup.
  ...resolveKafkaTransport(),
});

// --- Subscription state, reported on /health ---
//
// `/health` returned a hardcoded `status: 'healthy'` with a client count and
// nothing about Kafka, so a realtime service whose consumers had never
// connected was indistinguishable from one fanning out every alert. The two
// consumers below register here and `reportHealth` answers 503 while either
// is detached — the same contract the Python services answer on /readyz.
type SubscriptionState = {
  attached: boolean;
  lastError?: string;
  attempts: number;
};
const subscriptions = new Map<string, SubscriptionState>();

// Bounded backoff, capped. A detached consumer is retried because the usual
// cause is a broker or topic that is not there *yet*, which a retry fixes;
// the cap stops a permanently-broken cluster becoming a busy loop, and the
// state above stops a permanent failure being silent while it retries.
const RETRY_BASE_MS = 2_000;
const RETRY_MAX_MS = 60_000;

/**
 * Keep `start` running, and record whether it is.
 *
 * The previous code called each starter inside a try/catch that logged
 * "Kafka consumer failed to start (will retry)" and then returned. Nothing
 * retried. The message described behaviour the code did not have, which is
 * worse than no message: it tells a reader the gap is already handled.
 */
function superviseConsumer(
  name: string,
  start: (onCrash: (err: unknown) => void) => Promise<void>,
): void {
  subscriptions.set(name, { attached: false, attempts: 0 });

  const run = async (): Promise<void> => {
    const state = subscriptions.get(name)!;
    for (;;) {
      state.attempts += 1;
      try {
        let crashed = false;
        await start((err: unknown) => {
          // kafkajs surfaces a dead consumer loop through its CRASH event,
          // not by rejecting the promise `run()` returned — so without this
          // hook a consumer that died after a successful start stayed
          // recorded as attached for the life of the process.
          if (crashed) return;
          crashed = true;
          state.attached = false;
          state.lastError = err instanceof Error ? err.message : String(err);
          log.error({ err, name }, 'Kafka consumer crashed; reconnecting');
        });
        state.attached = true;
        state.lastError = undefined;
        log.info({ name, attempts: state.attempts }, 'Kafka consumer attached');
        // Wait for a crash rather than returning: this function owns the
        // retry, so it has to still be here when the consumer stops.
        while (!crashed) {
          await new Promise((resolve) => setTimeout(resolve, 5_000));
        }
      } catch (err) {
        state.attached = false;
        state.lastError = err instanceof Error ? err.message : String(err);
        const delay = Math.min(
          RETRY_BASE_MS * 2 ** Math.min(state.attempts - 1, 5),
          RETRY_MAX_MS,
        );
        log.warn(
          { err, name, attempts: state.attempts, retryInMs: delay },
          'Kafka consumer not attached; retrying',
        );
        await new Promise((resolve) => setTimeout(resolve, delay));
      }
    }
  };

  void run();
}

// --- Kafka consumer: bridge fused alerts to WebSocket clients ---
async function startKafkaConsumer(onCrash: (err: unknown) => void) {
  const consumer = kafka.consumer({ groupId: 'aisoc-realtime-ws' });
  consumer.on(consumer.events.CRASH, (event) => onCrash(event.payload.error));

  await consumer.connect();
  await consumer.subscribe({ topic: KAFKA_TOPIC_FUSED, fromBeginning: false });

  log.info({ topic: KAFKA_TOPIC_FUSED }, 'Kafka consumer connected');

  await consumer.run({
    eachMessage: async ({ message }) => {
      if (!message.value) return;
      try {
        const event = JSON.parse(message.value.toString());
        const tenantId = event?.alert?.tenant_id || event?.tenant_id || 'default';

        broadcastToTenant(tenantId, {
          type: 'alert.fused',
          payload: event,
          timestamp: new Date().toISOString(),
        });

        // Fan out P0/critical alerts to mobile responders. Lower-severity
        // alerts stay WebSocket-only so we don't pager-storm on-call.
        const severity = String(
          event?.alert?.severity ?? event?.severity ?? '',
        ).toLowerCase();
        if (
          pushManager.enabled &&
          (severity === 'critical' || severity === 'high')
        ) {
          const alertId =
            event?.alert?.id ?? event?.alert?.alert_id ?? event?.id ?? 'unknown';
          const title = `${severity === 'critical' ? 'P0' : 'P1'} alert · ${
            event?.alert?.title ?? event?.title ?? 'New alert'
          }`;
          pushManager
            .sendToTarget(
              { tenant_id: tenantId, topic: 'p0_alert' },
              {
                title,
                body:
                  event?.alert?.summary ??
                  event?.summary ??
                  'Tap to triage in the responder console',
                url: `/responder/triage/${alertId}`,
                tag: `alert-${alertId}`,
                topic: 'p0_alert',
                severity,
                alert_id: String(alertId),
              },
            )
            .catch((err: unknown) =>
              log.warn({ err, tenantId }, 'push fan-out for fused alert failed'),
            );
        }
      } catch (err) {
        log.warn({ err }, 'Failed to parse Kafka message');
      }
    },
  });
}

// --- Kafka consumer: bridge graph mutations to WebSocket clients (T1.4) ---
// The ingest-side graph writer (services/ingest/internal/graph/writer.go)
// publishes one envelope per node/edge upsert on `security.graph_updates`.
// We fan that stream out to clients subscribed to `/ws/graph` (or `/ws/all`),
// scoped by tenant. We deliberately run this on its own consumer group so a
// graph-side stall can't backpressure the fused-alert fan-out — and vice
// versa. A failure to start (Kafka unreachable, topic missing, etc.) is
// logged and surfaced via the outer retry wrapper; it does NOT crash the
// process, because the alert fan-out path is the higher-priority surface.
async function startGraphUpdateConsumer(onCrash: (err: unknown) => void) {
  if (!KAFKA_TOPIC_GRAPH_UPDATES) {
    log.info('Graph update consumer disabled (empty KAFKA_TOPIC_GRAPH_UPDATES)');
    return;
  }

  const consumer = kafka.consumer({ groupId: 'aisoc-realtime-graph' });
  consumer.on(consumer.events.CRASH, (event) => onCrash(event.payload.error));

  await consumer.connect();
  await consumer.subscribe({
    topic: KAFKA_TOPIC_GRAPH_UPDATES,
    // We never want to replay the full graph history on a realtime restart —
    // a panel that connects today should see deltas from now, not from the
    // start of the topic.
    fromBeginning: false,
  });

  log.info(
    { topic: KAFKA_TOPIC_GRAPH_UPDATES, groupId: 'aisoc-realtime-graph' },
    'Graph update Kafka consumer connected',
  );

  await consumer.run({
    eachMessage: async ({ message }) => {
      if (!message.value) return;
      try {
        // The Go writer emits the camel-case-on-the-wire form documented in
        // services/ingest/internal/graph/writer.go: `entity_id`,
        // `change_type`, `ts`, `label`, `rel_type`, `from`, `to`,
        // `properties`, `schema_version`, `tenant_id`. Anything else is
        // treated as a malformed envelope and dropped.
        const update = JSON.parse(message.value.toString()) as {
          entity_id?: string;
          change_type?: string;
          ts?: string;
          label?: string;
          rel_type?: string;
          from?: string;
          to?: string;
          properties?: Record<string, unknown>;
          schema_version?: string;
          tenant_id?: string;
        };

        if (!update.entity_id || !update.change_type) {
          // Headerless or partial envelopes are almost always a producer
          // bug — log once at warn so we notice in CI but don't tear the
          // consumer down. (kafkajs auto-commits the offset either way.)
          log.warn(
            { entity_id: update.entity_id, change_type: update.change_type },
            'Dropping malformed graph update (missing entity_id or change_type)',
          );
          return;
        }

        // Per-tenant fan-out. Default to "default" so single-tenant
        // self-hosted deploys keep working without explicit tenant tagging
        // (matches the policy on the fused-alerts consumer above).
        const tenantId = update.tenant_id || 'default';

        broadcastToTenant(tenantId, {
          type: 'graph.update',
          payload: update,
          timestamp: new Date().toISOString(),
        });
      } catch (err) {
        log.warn({ err }, 'Failed to parse graph update Kafka message');
      }
    },
  });
}

// --- Web Push routes (Phase 4B mobile responder PWA) ---
// Public key is needed by the browser to subscribe; the rest are
// authenticated routes the API gateway forwards on behalf of a user.
// Rate-limit push mutation routes: 20 req / min per IP to prevent abuse.
const pushRateLimit = rateLimit({
  windowMs: 60_000,
  max: 20,
  standardHeaders: true,
  legacyHeaders: false,
  message: { error: 'Rate limit exceeded' },
});

// Express 4 does not await a handler, so a rejected promise from an `async`
// one never reaches the error middleware — it surfaces as an unhandled
// rejection, which Node terminates the process on. Five routes were
// registered that way, including the two internal fan-out endpoints the API
// and agents services call. A single failed `webpush` send or Redis write
// would have taken the whole realtime service down and dropped every open
// WebSocket with it.
//
// The wrapper hands the rejection to `next()`, which is the contract Express
// error handling is built on.
function asyncRoute(
  handler: (req: express.Request, res: express.Response) => Promise<void>,
): express.RequestHandler {
  return (req, res, next) => {
    handler(req, res).catch(next);
  };
}

// `public-key` is the VAPID public key and is public by definition.
app.get('/v1/push/public-key', pushManager.publicKeyHandler);

// The other three carried only a rate limiter, and took the tenant from a
// request header, then a query parameter, then the literal string 'default'.
// So any caller who could reach the port could enrol a push endpoint against
// any tenant, unsubscribe another tenant's devices, or make this service send
// a notification. They are reached through the API's `/api/v1/push/*` proxy,
// which authenticates the analyst and stamps the tenant, so the same internal
// token `/internal/*` requires is the right contract: it makes the proxy the
// only way in, which is what the module already assumed.
app.post('/v1/push/subscribe', pushRateLimit, asyncRoute(async (req, res) => {
  if (!requireInternal(req, res)) return;
  await pushManager.subscribeHandler(req, res);
}));
app.post('/v1/push/unsubscribe', pushRateLimit, asyncRoute(async (req, res) => {
  if (!requireInternal(req, res)) return;
  await pushManager.unsubscribeHandler(req, res);
}));
app.post('/v1/push/test', pushRateLimit, asyncRoute(async (req, res) => {
  if (!requireInternal(req, res)) return;
  await pushManager.testNotifyHandler(req, res);
}));

// --- Internal broadcast endpoint (called by other services) ---
// POST /internal/agent-event
// Body: { tenant_id?: string, run_id: string, kind: string, agent: string, summary: string, data?: unknown }
// The realtime service re-broadcasts to all WebSocket clients on the `agents` channel.
// `REALTIME_INTERNAL_TOKEN` is the name the API sends this under and the name
// `make up` generates; `INTERNAL_TOKEN` is read second so an existing
// deployment that set the older name keeps working.
const INTERNAL_TOKEN = (process.env.REALTIME_INTERNAL_TOKEN || process.env.INTERNAL_TOKEN || '').trim();

function requireInternal(req: express.Request, res: express.Response): boolean {
  // Fails closed. This returned `true` — authorized — when the token was
  // unset, and no shipped manifest set it, so the guard never ran: any caller
  // who could reach the port could inject events into any tenant's live stream
  // by naming the tenant in the body, and could push a notification with
  // attacker-chosen title, body and URL to that tenant's devices
  // (GHSA-mqjp-pcpr-7c37).
  if (!INTERNAL_TOKEN) {
    res.status(503).json({
      error: 'realtime internal auth is not configured',
      detail: 'set REALTIME_INTERNAL_TOKEN (run `make env`, which generates it, and restart)',
    });
    return false;
  }
  // Both spellings. `x-internal-token` is what the agents service and the
  // API's approval route send; `x-aisoc-internal-token` is what the API's
  // push proxy and the chatops notifier send, and what slack-bot reads. One
  // receiver accepting both is the only fix that does not require every
  // sender to change in the same release.
  const auth = req.headers['x-internal-token'] ?? req.headers['x-aisoc-internal-token'];
  if (typeof auth !== 'string' || auth.length !== INTERNAL_TOKEN.length) {
    res.status(401).json({ error: 'unauthorized' });
    return false;
  }
  // Constant-time: a length-independent compare on a bearer is a timing oracle.
  if (!crypto.timingSafeEqual(Buffer.from(auth), Buffer.from(INTERNAL_TOKEN))) {
    res.status(401).json({ error: 'unauthorized' });
    return false;
  }
  return true;
}

// Internal push fan-out used by the agents/api services to send a
// notification to a tenant, user list, or topic. Same auth contract as
// `internal/agent-event`.
app.post(
  '/internal/push',
  internalPushRateLimit,
  asyncRoute(async (req, res) => {
    if (!requireInternal(req, res)) return;

    await pushManager.internalNotifyHandler(req, res);
  }),
);

app.post('/internal/agent-event', internalEventRateLimit, (req, res) => {
  if (!requireInternal(req, res)) return;

  const { tenant_id, run_id, kind, agent, summary, data } = req.body as {
    tenant_id?: string;
    run_id: string;
    kind: string;
    agent: string;
    summary: string;
    data?: Record<string, unknown> | null;
  };

  if (!run_id || !kind || !agent) {
    res.status(400).json({ error: 'run_id, kind, and agent are required' });
    return;
  }

  const tenantId = tenant_id || 'default';
  broadcastToTenant(tenantId, {
    type: 'agent.event',
    run_id,
    kind,
    agent,
    summary,
    data: data ?? null,
    timestamp: new Date().toISOString(),
  });

  // Also publish to Redis for SSE subscribers
  const redisPub = new Redis(REDIS_URL);
  const payload = JSON.stringify({
    type: 'agent.event',
    run_id,
    kind,
    agent,
    summary,
    timestamp: new Date().toISOString(),
  });
  redisPub.publish(`aisoc:events:${tenantId}`, payload)
    .then(() => redisPub.disconnect())
    .catch((err: unknown) => log.warn({ err }, 'Redis publish failed'));

  // Approval requests page on-call directly. Anything else stays
  // websocket-only so the device doesn't buzz on every reasoning step.
  if (
    pushManager.enabled &&
    (kind === 'APPROVAL_REQUEST' || kind === 'approval_request')
  ) {
    const approvalId =
      (data && (data.approval_id as string | undefined)) ?? run_id;
    const caseId = (data?.case_id as string | undefined) ?? undefined;
    const userIds =
      data &&
      (data.notify_user_ids as string[] | undefined) instanceof Array
        ? (data.notify_user_ids as string[])
        : undefined;
    pushManager
      .sendToTarget(
        userIds && userIds.length > 0
          ? { tenant_id: tenantId, user_ids: userIds }
          : { tenant_id: tenantId, topic: 'agent_approval' },
        {
          title: 'Agent needs your approval',
          body: summary || `${agent} is waiting for a decision.`,
          url: caseId
            ? `/responder/case/${caseId}?approval=${approvalId}`
            : `/responder/approvals?focus=${approvalId}`,
          tag: `approval-${approvalId}`,
          topic: 'agent_approval',
          severity: 'high',
          approval_id: String(approvalId),
          case_id: caseId,
        },
      )
      .catch((err: unknown) =>
        log.warn({ err, tenantId }, 'push fan-out for approval failed'),
      );
  }

  res.status(202).json({ broadcast: true, tenantId });
});

// --- Health endpoint ---
// Expose both `/health` (canonical) and `/healthz` (k8s + frontend default) so
// callers don't have to guess.
const reportHealth = (_req: express.Request, res: express.Response) => {
  const detached = [...subscriptions.entries()]
    .filter(([, state]) => !state.attached)
    .map(([name]) => name);

  res.status(detached.length > 0 ? 503 : 200).json({
    status: detached.length > 0 ? 'degraded' : 'healthy',
    service: 'aisoc-realtime',
    clients: wss.clients.size,
    kafka: detached.length === 0,
    // Reported whether or not anything is wrong, so a healthy answer says
    // which subscriptions were checked rather than only that none failed.
    subscriptions: [...subscriptions.entries()].map(([name, state]) => ({
      topic: name,
      attached: state.attached,
      attempts: state.attempts,
      ...(state.lastError ? { last_error: state.lastError } : {}),
    })),
  });
};
app.get('/health', reportHealth);
app.get('/healthz', reportHealth);
app.get('/readyz', reportHealth);

// --- Start ---
// Bind to "::" so we are reachable over Fly.io's 6PN private network (IPv6).
// Node defaults to "::" when IPv6 is available, but be explicit to match the
// other services and avoid surprises if a future Node release changes the
// default or the container is launched with IPv6 disabled.
// Tracing. `ingest` and `realtime` were the two uninstrumented ends of the
// Kafka spine, so a trace started at the API stopped at the pipeline
// boundary — exactly where the interesting latency is. No-ops unless
// OTEL_EXPORTER_OTLP_ENDPOINT is set.
let shutdownTelemetry: Shutdown = async () => {};

server.listen(PORT, '::', () => {
  log.info({ port: PORT, host: '::' }, 'AiSOC Real-time service started');

  // The listen callback is declared to return void, so an `async` one hands
  // Node a promise nobody awaits: a rejection here — an unreachable OTLP
  // collector, say — ended the process rather than the tracing. Tracing is
  // optional; serving is not.
  void (async () => {
    try {
      shutdownTelemetry = await setupTelemetry(log);
    } catch (err) {
      log.warn({ err }, 'telemetry setup failed; continuing without tracing');
    }
  })();

  // Start the two Kafka consumers under their own supervisors, so a failure
  // on the graph topic (e.g. it doesn't exist yet on a brand-new cluster)
  // does NOT block the higher-priority fused-alerts fan-out, and neither one
  // stays silently detached. Fired rather than awaited so the HTTP/WS
  // listener is fully up before either Kafka round-trip completes.
  superviseConsumer(KAFKA_TOPIC_FUSED, startKafkaConsumer);

  // Only supervised when it is configured. An empty KAFKA_TOPIC_GRAPH_UPDATES
  // turns this fan-out off deliberately, and reporting a switched-off consumer
  // as detached would make /health red for a supported configuration.
  if (KAFKA_TOPIC_GRAPH_UPDATES) {
    superviseConsumer(KAFKA_TOPIC_GRAPH_UPDATES, startGraphUpdateConsumer);
  }
});

// Flush spans on the way out. Without this the exporter drops the spans
// from the final seconds before a restart, which are disproportionately
// the ones someone is looking for.
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
  process.once(signal, () => {
    log.info({ signal }, 'shutting down');
    void shutdownTelemetry().finally(() => {
      server.close(() => process.exit(0));
      // Do not wait forever on lingering WebSocket connections: a
      // fan-out service always has open sockets, so close() alone may
      // never resolve and the container would be SIGKILLed with spans
      // still buffered.
      setTimeout(() => process.exit(0), 5000).unref();
    });
  });
}