Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

14.4 KB
Raw Blame
// SSE stream handler for task-run event streams.
//
// Async generator yielding SSE byte chunks. Wire protocol is byte-identical to
// the Python implementation: during cutover Django and this Node service read
// the SAME Redis stream and serve the SAME clients. Any drift corrupts live runs.
//
// SSE framing (format_sse_event — byte-identical to Python):
//   [event: <name>\n]   — only when eventName is truthy
//   [id: <eventId>\n]   — only when eventId is truthy
//   data: <JSON>\n
//   \n
//
// Event types:
//   Normal stream event  — no eventName, id = Redis stream ID
//   Keepalive            — eventName='keepalive', data={"type":"keepalive"}
//   Terminal             — eventName='stream-end',  data={"status":"complete"}
//   Rotation             — eventName='end',          data={"type":"rotated"}
//   Resync               — eventName='end',          data={"type":"resync","reason":"trimmed"}
//   Stream error         — eventName='error',        data={"error":"<msg>"}
//   Stream unavailable   — eventName='error',        data={"error":"Stream not available"}

import type { Redis } from 'ioredis'

import {
    CONNECTION_MAX_MS,
    KEEPALIVE_INTERVAL_MS,
    SSE_EVENT_END,
    SSE_EVENT_ERROR,
    SSE_EVENT_KEEPALIVE,
    SSE_EVENT_STREAM_END,
    SSE_PAYLOAD_KEEPALIVE,
    SSE_PAYLOAD_RESYNC,
    SSE_PAYLOAD_ROTATED,
    SSE_PAYLOAD_STREAM_END,
    STREAM_CURSOR_RECHECK_STALL_MS,
    WAIT_DELAY_INCREMENT_MS,
    WAIT_INITIAL_DELAY_MS,
    WAIT_MAX_DELAY_MS,
    WAIT_TIMEOUT_MS,
    WATCHED_REFRESH_INTERVAL_MS,
    makeSseErrorPayload,
} from '../lib/constants.js'
import { logger } from '../lib/logging.js'
import { TaskRunRedisStream } from '../lib/redis-stream.js'
import type { StreamConnectionOutcome } from '../lib/types.js'
import { TaskRunStreamCursorTrimmedError, TaskRunStreamError } from '../lib/types.js'
import {
    observeStreamConnectionClosed,
    observeStreamConnectionOpened,
    observeStreamLengthOnConnect,
    observeStreamResumeGap,
} from './metrics.js'

// ---------------------------------------------------------------------------
// SSE framing
// ---------------------------------------------------------------------------

/**
 * Format one SSE event frame, byte-identical to the Python format_sse_event.
 *
 * Order: [event: <name>\n] [id: <id>\n] data: <json>\n\n
 *
 * The result is encoded as UTF-8 and returned as a Buffer so callers can pipe
 * it directly into an HTTP response body without additional conversion.
 */
export function formatSseEvent(data: Record<string, unknown>, opts?: { eventId?: string; eventName?: string }): Buffer {
    const parts: string[] = []
    if (opts?.eventName) {
        parts.push(`event: ${opts.eventName}`)
    }
    if (opts?.eventId) {
        parts.push(`id: ${opts.eventId}`)
    }
    parts.push(`data: ${JSON.stringify(data)}`)
    return Buffer.from(parts.join('\n') + '\n\n', 'utf8')
}

// ---------------------------------------------------------------------------
// SSE stream generator
// ---------------------------------------------------------------------------

/**
 * Async generator that yields SSE-formatted byte chunks for one task-run
 * stream connection.
 *
 * Mirrors Python stream_task_run_events exactly:
 *   1. Records opened metric; sets default outcome = 'client_disconnect'.
 *   2. Wait-for-stream loop: polls exists() with linear backoff until the
 *      stream appears or the 120s timeout fires (yields unavailable error event
 *      and returns on timeout; yields keepalive every 20s while waiting).
 *   3. Resolves startId from lastEventId / startLatest / '0'.
 *   4. On reconnect (lastEventId set): observes stream length (best-effort) and
 *      checks for a resume gap. A resync-capable client gets the resync frame
 *      when its cursor was trimmed, and a stream error when the check fails.
 *   5. Iterates readStreamEntries:
 *      - null  → yield keepalive
 *      - entry → yield formatSseEvent(event, {eventId})
 *      A resync-capable client also gets the resync frame when its cursor is
 *      trimmed while the connection is stalled mid-stream.
 *   6. After generator returns (completion sentinel consumed): yield stream-end
 *      terminal event; set outcome = 'completed'.
 *   7. On TaskRunStreamError: yield error event; set outcome = 'stream_error'.
 *   8. Finally: record closed metric with outcome and duration.
 *
 * @yields {Buffer} SSE-formatted byte chunk (UTF-8 encoded).
 */
export async function* streamTaskRunEvents(
    streamKey: string,
    redis: Redis,
    opts: {
        originProduct?: string
        lastEventId?: string | null
        startLatest?: boolean
        presenceGated?: boolean
        isTerminal?: boolean
        resyncCapable?: boolean
    }
): AsyncGenerator<Buffer, void, unknown> {
    const originProduct = opts.originProduct ?? 'unknown'
    const lastEventId = opts.lastEventId ?? null
    const startLatest = opts.startLatest ?? false
    const presenceGated = opts.presenceGated ?? false
    const isTerminal = opts.isTerminal ?? false
    const resyncCapable = opts.resyncCapable ?? false

    const redisStream = new TaskRunRedisStream(streamKey, redis)
    const connectionStartedAt = Date.now()

    let outcome: StreamConnectionOutcome = 'client_disconnect'
    let opened = false
    // Dedicated blocking-read connection created just before the main read loop.
    // Declared here so the finally block can close it regardless of how the
    // generator exits (completion, error, or client disconnect).
    let dedupRedis: Redis | undefined

    let lastWatchedRefreshAt: number | null = null
    const refreshWatched = async (now: number): Promise<void> => {
        if (lastWatchedRefreshAt !== null && now - lastWatchedRefreshAt < WATCHED_REFRESH_INTERVAL_MS) {
            return
        }
        try {
            await redisStream.markWatched()
            lastWatchedRefreshAt = now
        } catch (err) {
            logger.warn('stream:mark_watched_failed', {
                streamKey,
                error: err instanceof Error ? err.message : String(err),
            })
        }
    }

    try {
        observeStreamConnectionOpened(originProduct)
        opened = true

        await refreshWatched(Date.now())

        // -- Wait-for-stream loop --
        let delay = WAIT_INITIAL_DELAY_MS
        const waitStartedAt = Date.now()
        let lastKeepaliveAt = waitStartedAt
        let waitedForStream = false
        while (!(await redisStream.exists())) {
            if (isTerminal) {
                if (lastEventId && resyncCapable) {
                    observeStreamResumeGap(originProduct)
                    logger.warn('stream:resume_gap', { streamKey, lastEventId, resyncCapable, reason: 'expired' })
                    outcome = 'resync'
                    yield formatSseEvent(SSE_PAYLOAD_RESYNC, { eventName: SSE_EVENT_END })
                    return
                }
                outcome = 'drained'
                yield formatSseEvent(SSE_PAYLOAD_STREAM_END, { eventName: SSE_EVENT_STREAM_END })
                return
            }
            if (!waitedForStream) {
                waitedForStream = true
                logger.debug('stream:waiting', { streamKey })
            }

            if (presenceGated) {
                break
            }

            const now = Date.now()
            await refreshWatched(now)

            if (now - waitStartedAt >= WAIT_TIMEOUT_MS) {
                outcome = 'unavailable'
                logger.warn('stream:unavailable', { streamKey, waitedMs: now - waitStartedAt })
                yield formatSseEvent(makeSseErrorPayload('Stream not available'), {
                    eventName: SSE_EVENT_ERROR,
                })
                return
            }

            if (now - lastKeepaliveAt >= KEEPALIVE_INTERVAL_MS) {
                lastKeepaliveAt = now
                yield formatSseEvent(SSE_PAYLOAD_KEEPALIVE, { eventName: SSE_EVENT_KEEPALIVE })
            }

            await sleep(delay)
            delay = Math.min(delay + WAIT_DELAY_INCREMENT_MS, WAIT_MAX_DELAY_MS)
        }

        if (waitedForStream) {
            logger.debug('stream:available', { streamKey, waitedMs: Date.now() - waitStartedAt })
        }

        // -- Resolve start ID --
        let startId: string
        if (lastEventId) {
            startId = lastEventId
        } else if (startLatest && !waitedForStream) {
            startId = (await redisStream.getLatestStreamId()) ?? '0'
        } else {
            startId = '0'
        }

        // -- Resume gap detection (only on reconnects) --
        let resumePointTrimmed = false
        if (lastEventId) {
            try {
                observeStreamLengthOnConnect(await redisStream.getLength())
            } catch (err) {
                logger.warn('stream:attach_observe_failed', {
                    streamKey,
                    error: err instanceof Error ? err.message : String(err),
                })
            }
            try {
                resumePointTrimmed = await redisStream.resumePointTrimmed(lastEventId)
            } catch (err) {
                logger.warn('stream:resume_check_failed', {
                    streamKey,
                    lastEventId,
                    resyncCapable,
                    error: err instanceof Error ? err.message : String(err),
                })
                if (resyncCapable) {
                    outcome = 'stream_error'
                    yield formatSseEvent(makeSseErrorPayload('Connection lost to task run stream'), {
                        eventName: SSE_EVENT_ERROR,
                    })
                    return
                }
            }
            if (resumePointTrimmed) {
                observeStreamResumeGap(originProduct)
                logger.warn('stream:resume_gap', { streamKey, lastEventId, resyncCapable, reason: 'trimmed' })
            }
        }
        if (resumePointTrimmed && resyncCapable) {
            outcome = 'resync'
            yield formatSseEvent(SSE_PAYLOAD_RESYNC, { eventName: SSE_EVENT_END })
            return
        }

        // Dedicated connection for the blocking XREAD loop so it cannot delay the
        // shared client's ingest writes (XADD, WATCH/MULTI) or non-blocking reads.
        // Connection count is bounded upstream by StreamCapacity (app.ts), which caps
        // concurrent streams per pod and per run before this duplicate is created.
        // FOLLOWUP: if pod-level connection count becomes a concern within those caps,
        // replace the per-stream duplicate with a small bounded pool of blocking-read
        // connections shared across streams. Not worth the complexity until profiling
        // shows a connection-count problem.
        // enableOfflineQueue:false + lazyConnect means we must call connect() explicitly
        // before the first command and register an error listener.
        dedupRedis = redis.duplicate()
        dedupRedis.on('error', (err: Error) => logger.warn('stream:redis_dup_error', { streamKey, error: err.message }))
        try {
            await dedupRedis.connect()
        } catch (err) {
            // The dedicated connection could not be established (e.g. Redis at its
            // connection limit). Degrade like a mid-stream connection loss: surface
            // an SSE error event rather than throwing a raw error out of the handler.
            outcome = 'stream_error'
            logger.error('stream:redis_dup_connect_failed', {
                streamKey,
                error: err instanceof Error ? err.message : String(err),
            })
            yield formatSseEvent(makeSseErrorPayload('Connection lost to task run stream'), {
                eventName: SSE_EVENT_ERROR,
            })
            return
        }

        // -- Main read loop --
        try {
            for await (const streamItem of redisStream.readStreamEntries({
                startId,
                keepaliveIntervalMs: KEEPALIVE_INTERVAL_MS,
                blockingRedis: dedupRedis,
                ...(resyncCapable ? { cursorRecheckAfterStallMs: STREAM_CURSOR_RECHECK_STALL_MS } : {}),
            })) {
                await refreshWatched(Date.now())

                if (streamItem === null) {
                    // Idle keepalive signal from readStreamEntries
                    yield formatSseEvent(SSE_PAYLOAD_KEEPALIVE, { eventName: SSE_EVENT_KEEPALIVE })
                } else {
                    const [eventId, event] = streamItem
                    yield formatSseEvent(event, { eventId })
                }

                if (Date.now() - connectionStartedAt >= CONNECTION_MAX_MS) {
                    outcome = 'rotated'
                    yield formatSseEvent(SSE_PAYLOAD_ROTATED, { eventName: SSE_EVENT_END })
                    return
                }
            }

            // Generator returned normally — completion sentinel was consumed.
            outcome = 'completed'
            yield formatSseEvent(SSE_PAYLOAD_STREAM_END, { eventName: SSE_EVENT_STREAM_END })
        } catch (err) {
            if (err instanceof TaskRunStreamCursorTrimmedError) {
                observeStreamResumeGap(originProduct)
                logger.warn('stream:resume_gap', { streamKey, cursor: err.cursor, resyncCapable, reason: 'stalled' })
                outcome = 'resync'
                yield formatSseEvent(SSE_PAYLOAD_RESYNC, { eventName: SSE_EVENT_END })
            } else if (err instanceof TaskRunStreamError) {
                outcome = 'stream_error'
                logger.error('stream:error', { streamKey, error: err.message })
                yield formatSseEvent(makeSseErrorPayload(err.message), { eventName: SSE_EVENT_ERROR })
            } else {
                throw err
            }
        }
    } finally {
        if (opened) {
            const durationSeconds = (Date.now() - connectionStartedAt) / 1000
            observeStreamConnectionClosed(originProduct, outcome, durationSeconds)
        }
        // Close the dedicated blocking-read connection. ioredis v4 disconnect() is
        // synchronous (returns void) and also aborts any in-flight XREAD BLOCK on
        // client disconnect, which is what we want.
        try {
            dedupRedis?.disconnect()
        } catch {
            // best-effort cleanup — never let teardown throw out of the finally
        }
    }
}

// ---------------------------------------------------------------------------
// Internal helpers
// ---------------------------------------------------------------------------

function sleep(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms))
}