Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

34.1 KB
Raw Blame
import { create } from '@bufbuild/protobuf'
import { Code, ConnectError, ConnectRouter, HandlerContext } from '@connectrpc/connect'
import { connectNodeAdapter } from '@connectrpc/connect-node'
import * as http2 from 'node:http2'
import { Counter, Gauge } from 'prom-client'

import {
    IngestStreamRequest,
    IngestStreamResponse,
    IngestStreamResponseSchema,
    KafkaMessage,
    MessageRefSchema,
    StreamHello,
    StreamReadySchema,
    SubBatchAckSchema,
    SubBatchStatus,
    WorkerIngest,
} from '~/common/generated/ingestion-worker/ingestion/worker/v1/worker_pb'
import { logger } from '~/common/utils/logger'
import { budgetDeadline } from '~/ingestion/framework/batch-budget'
import { FeedResult } from '~/ingestion/framework/batching-pipeline'

import { AcceptedBatch, batchAccepted, batchFailed, batchProcessed, batchReleased } from './batch-metrics'
import { FeedOrderSentinel } from './feed-order-sentinel'
import { SerializedKafkaMessage } from './types'

const grpcStreams = new Gauge({
    name: 'ingestion_api_grpc_streams',
    help: 'Open WorkerIngest streams (one per connected consumer process)',
})

const grpcSubBatches = new Counter({
    name: 'ingestion_api_grpc_sub_batches_total',
    help: 'Sub-batches resolved on the gRPC ingest path',
    labelNames: ['status'],
})

const grpcMessagesProcessed = new Counter({
    name: 'ingestion_api_grpc_messages_processed_total',
    help: 'Messages acked on the gRPC ingest path',
})

const grpcProtocolErrors = new Counter({
    name: 'ingestion_api_grpc_protocol_errors_total',
    help: 'Streams closed for protocol violations (missing hello, seq gap or regression)',
    labelNames: ['kind'],
})

const grpcFeedSlotWaiters = new Gauge({
    name: 'ingestion_api_grpc_feed_slot_waiters',
    help: 'Streams waiting for a pipeline batch slot (FIFO admission queue depth)',
})

const grpcCapacityRetries = new Counter({
    name: 'ingestion_api_grpc_capacity_retries_total',
    help: 'Feeds rejected at_capacity despite holding an admission slot (capacity accounting drift)',
})

const grpcAdmissionExpired = new Counter({
    name: 'ingestion_api_grpc_admission_expired_total',
    help: 'Sub-batches whose soft deadline passed while they waited for an admission slot',
})

const grpcConnectionsRejected = new Counter({
    name: 'ingestion_api_grpc_connections_rejected_total',
    help: 'Sessions or streams refused because a concurrency cap was hit',
    labelNames: ['reason'],
})

export type MessageCoordinates = Pick<SerializedKafkaMessage, 'partition' | 'offset'>

/** A batch the pipeline finished processing. */
export interface CompletedSubBatch {
    streamId: number
    seq: number
    accepted: number
    unfinished: MessageCoordinates[]
    /**
     * Resolves when the batch's side effects are durably done — the ack
     * barrier. Kept as a promise (not awaited inside `next()`) so the pump
     * can settle many completed batches concurrently: one batch's slow side
     * effects must not head-of-line block every other stream's acks.
     */
    settled: Promise<void>
}

export interface SubBatchBudget {
    /** Epoch milliseconds at frame read, so the wait for an admission slot counts. */
    armedAt: number
    /** `0` means no deadline. */
    softBudgetMs: number
}

/**
 * Pipeline mechanics behind the stream protocol. The server owns ordering,
 * acks, capacity admission, and stream lifecycle; the driver owns how a
 * sub-batch enters the shared pipeline and when its processing is durably
 * done (`settled`).
 */
export interface StreamIngestDriver {
    feed(streamId: number, seq: number, messages: SerializedKafkaMessage[], budget: SubBatchBudget): Promise<FeedResult>
    next(): Promise<CompletedSubBatch | null>
}

export interface WorkerIngestServerOptions {
    port: number
    /**
     * Pipeline batch capacity, mirrored by the server's FIFO admission queue:
     * feeds are granted in arrival order across all streams, so a busy stream
     * cannot starve another (a retry race here wedged whole consumers).
     * Must match the gRPC pipeline's `concurrentBatches`.
     */
    maxConcurrentBatches: number
    /** Fallback delay if the pipeline still reports at_capacity despite an admission slot. */
    capacityRetryMs?: number
    /** Delay before re-pumping when the pipeline reports drained but sub-batches are still in flight. */
    pumpIdleMs?: number
    /**
     * Total concurrent streams the server will hold open across all sessions.
     * The topology is roughly one stream per connected consumer, so this is a
     * generous ceiling that only trips on misbehaviour — a stream past it is
     * refused with ResourceExhausted rather than accumulating unbounded.
     */
    maxStreams?: number
    /** Total concurrent HTTP/2 sessions; a session past this is destroyed on open. */
    maxSessions?: number
    /** Per-session stream limit, advertised via HTTP/2 SETTINGS and enforced by Node. */
    maxStreamsPerSession?: number
    /** Per-session memory budget in MB (HTTP/2 `maxSessionMemory`). */
    sessionMemoryMb?: number
    /**
     * Largest gRPC message the server will deserialize; bigger frames get
     * ResourceExhausted. The consumer targets 10 MiB sub-batches, but an
     * over-limit frame is replayed verbatim, so keep generous headroom.
     */
    readMaxBytes?: number
    /** Idle time before a session with no activity is closed, reaping dead peers. */
    sessionIdleTimeoutMs?: number
    /**
     * How long stop() waits for fed sub-batches to settle and their acks to
     * flush before failing what remains. Without the drain, every scale-down
     * makes consumers replay work the pipeline already processed.
     */
    drainTimeoutMs?: number
}

/** Thrown from `FifoSlots.acquire` when the waiting stream is cancelled. */
class SlotAcquisitionAborted extends Error {}

/**
 * FIFO slot queue: capacity grants go to waiters in arrival order. A freed
 * slot passes directly to the oldest waiter instead of returning to a pool a
 * newer arrival could win.
 *
 * A waiter can be cancelled via its `AbortSignal`: a stream that disconnects
 * while parked here is removed from the queue, so a freed slot skips it and
 * goes to a live stream instead of feeding work that can never be acked.
 */
class FifoSlots {
    private available: number
    private waiters: (() => void)[] = []

    constructor(capacity: number) {
        this.available = capacity
    }

    acquire(signal?: AbortSignal): Promise<void> {
        if (signal?.aborted) {
            return Promise.reject(new SlotAcquisitionAborted())
        }
        if (this.available > 0) {
            this.available--
            return Promise.resolve()
        }
        grpcFeedSlotWaiters.inc()
        return new Promise<void>((resolve, reject) => {
            const grant = (): void => {
                signal?.removeEventListener('abort', onAbort)
                grpcFeedSlotWaiters.dec()
                resolve()
            }
            const onAbort = (): void => {
                const index = this.waiters.indexOf(grant)
                if (index === -1) {
                    // Already granted a slot; the caller's post-acquire check
                    // hands it back. Nothing queued to remove here.
                    return
                }
                this.waiters.splice(index, 1)
                grpcFeedSlotWaiters.dec()
                reject(new SlotAcquisitionAborted())
            }
            this.waiters.push(grant)
            signal?.addEventListener('abort', onAbort, { once: true })
        })
    }

    release(): void {
        const next = this.waiters.shift()
        if (next) {
            next()
            return
        }
        this.available++
    }
}

interface WorkerIngestServerDeps {
    driver: StreamIngestDriver
    /** Shared with the HTTP path so both transports check the same invariant. */
    feedOrderSentinel?: FeedOrderSentinel
    /**
     * A pipeline error poisons the shared pipeline for every stream, so the
     * server fails all streams and hands the error up — the owner mirrors the
     * HTTP path's crash-and-rebuild contract.
     */
    onFatal: (error: Error) => void
}

/** Unbounded FIFO handed from the pump to each stream's response generator. */
class AckQueue {
    private items: IngestStreamResponse[] = []
    private waiter: {
        resolve: (value: IteratorResult<IngestStreamResponse>) => void
        reject: (err: Error) => void
    } | null = null
    private done = false
    private error: Error | null = null

    push(item: IngestStreamResponse): void {
        if (this.done) {
            return
        }
        if (this.waiter) {
            const { resolve } = this.waiter
            this.waiter = null
            resolve({ value: item, done: false })
            return
        }
        this.items.push(item)
    }

    /** No more acks will arrive; the generator completes after draining. */
    end(): void {
        this.done = true
        if (this.waiter && this.items.length === 0) {
            const { resolve } = this.waiter
            this.waiter = null
            resolve({ value: undefined, done: true })
        }
    }

    /** Fail the consuming generator (stream error towards the consumer). */
    fail(error: Error): void {
        this.error = error
        this.done = true
        if (this.waiter) {
            const { reject } = this.waiter
            this.waiter = null
            reject(error)
        }
    }

    [Symbol.asyncIterator](): AsyncIterator<IngestStreamResponse> {
        return {
            next: (): Promise<IteratorResult<IngestStreamResponse>> => {
                if (this.items.length > 0) {
                    return Promise.resolve({ value: this.items.shift()!, done: false })
                }
                if (this.error) {
                    return Promise.reject(this.error)
                }
                if (this.done) {
                    return Promise.resolve({ value: undefined, done: true })
                }
                return new Promise((resolve, reject) => {
                    this.waiter = { resolve, reject }
                })
            },
        }
    }
}

interface StreamState {
    id: number
    hello: StreamHello | null
    nextSeq: number
    inFlight: Set<number>
    readerDone: boolean
    acks: AckQueue
}

export function grpcMessageToSerialized(message: KafkaMessage): SerializedKafkaMessage {
    return {
        topic: message.topic,
        partition: message.partition,
        offset: Number(message.offset),
        timestamp: Number(message.timestamp),
        key: message.key ?? null,
        value: message.value ?? null,
        headers: message.headers,
    }
}

function readyFrame(): IngestStreamResponse {
    return create(IngestStreamResponseSchema, {
        msg: { case: 'ready', value: create(StreamReadySchema, {}) },
    })
}

function okAck(seq: number, accepted: number): IngestStreamResponse {
    return create(IngestStreamResponseSchema, {
        msg: {
            case: 'ack',
            value: create(SubBatchAckSchema, { seq: BigInt(seq), status: SubBatchStatus.OK, accepted }),
        },
    })
}

function partialAck(seq: number, accepted: number, unfinished: MessageCoordinates[]): IngestStreamResponse {
    return create(IngestStreamResponseSchema, {
        msg: {
            case: 'ack',
            value: create(SubBatchAckSchema, {
                seq: BigInt(seq),
                status: SubBatchStatus.PARTIAL,
                accepted,
                unfinished: unfinished.map(({ partition, offset }) =>
                    create(MessageRefSchema, { partition, offset: BigInt(offset) })
                ),
            }),
        },
    })
}

function failedAck(seq: number, error: string): IngestStreamResponse {
    return create(IngestStreamResponseSchema, {
        msg: {
            case: 'ack',
            value: create(SubBatchAckSchema, { seq: BigInt(seq), status: SubBatchStatus.FAILED, error }),
        },
    })
}

/**
 * Serves `ingestion.worker.v1.WorkerIngest`: the ordered streaming transport
 * from the Rust ingestion consumer.
 *
 * Ordering: each stream has one reader loop that awaits the pipeline's
 * acceptance of a sub-batch before reading the next frame, so feed order
 * equals stream order — and HTTP/2 flow control backpressures the consumer
 * while the pipeline is at capacity, replacing the HTTP path's 503s.
 * Processing stays concurrent behind the feed; acks are pushed as batches
 * complete, out of order, correlated by `seq`.
 *
 * Failure: any pipeline failure poisons the shared pipeline, so every open
 * stream is failed and `onFatal` fires — the consumer treats a broken stream
 * as a dead lane and replays its un-acked tail.
 */
export class WorkerIngestServer {
    private server: http2.Http2Server | null = null
    private streams = new Map<number, StreamState>()
    private nextStreamId = 1
    private stopped = false
    private draining = false
    private readonly drainAbort = new AbortController()
    private readonly stopAbort = new AbortController()
    private sessions = new Set<http2.ServerHttp2Session>()
    private fatalError: Error | null = null
    private pumpTask: Promise<void> | null = null
    private wakePump: (() => void) | null = null
    private readonly capacityRetryMs: number
    private readonly pumpIdleMs: number
    private readonly maxStreams: number
    private readonly maxSessions: number
    private readonly maxStreamsPerSession: number
    private readonly sessionMemoryMb: number
    private readonly sessionIdleTimeoutMs: number
    private readonly readMaxBytes: number
    private readonly drainTimeoutMs: number
    private sessionCount = 0
    private readonly slots: FifoSlots
    /**
     * Fed sub-batches by `streamId:seq`, for the shared in-flight accounting the
     * processor autoscaler reads. Kept off StreamState so a batch whose stream
     * died mid-processing is still released when it settles.
     */
    private readonly acceptedBatches = new Map<string, AcceptedBatch>()

    constructor(
        private options: WorkerIngestServerOptions,
        private deps: WorkerIngestServerDeps
    ) {
        this.capacityRetryMs = options.capacityRetryMs ?? 20
        this.pumpIdleMs = options.pumpIdleMs ?? 20
        this.maxStreams = options.maxStreams ?? 256
        this.maxSessions = options.maxSessions ?? 256
        this.maxStreamsPerSession = options.maxStreamsPerSession ?? 8
        this.sessionMemoryMb = options.sessionMemoryMb ?? 64
        this.sessionIdleTimeoutMs = options.sessionIdleTimeoutMs ?? 300_000
        this.readMaxBytes = options.readMaxBytes ?? 32 * 1024 * 1024
        this.drainTimeoutMs = options.drainTimeoutMs ?? 15_000
        this.slots = new FifoSlots(options.maxConcurrentBatches)
    }

    async start(): Promise<void> {
        this.pumpTask = this.runPump()
        this.server = http2.createServer(
            {
                // Cap per-session streams (advertised via SETTINGS, enforced by
                // Node) and per-session buffered memory, so one peer cannot fan
                // out or buffer without bound. Total sessions and streams are
                // bounded separately below and in ingestStream.
                settings: { maxConcurrentStreams: this.maxStreamsPerSession },
                maxSessionMemory: this.sessionMemoryMb,
            },
            connectNodeAdapter({
                readMaxBytes: this.readMaxBytes,
                routes: (router: ConnectRouter) => {
                    router.service(WorkerIngest, {
                        ingestStream: (requests, context: HandlerContext) =>
                            this.ingestStream(requests, context.signal),
                    })
                },
            })
        )
        this.server.on('session', (session) => this.onSession(session))
        await new Promise<void>((resolve, reject) => {
            this.server!.once('error', reject)
            this.server!.listen(this.options.port, () => {
                this.server!.removeListener('error', reject)
                logger.info('🛜', `WorkerIngest gRPC server listening on port ${this.options.port}`)
                resolve()
            })
        })
    }

    /**
     * Bounds concurrent sessions and reaps idle ones. A session past the cap is
     * destroyed on open; an idle session is closed so a dead or slow peer cannot
     * hold resources indefinitely. The `error` listener keeps a session-level
     * error from crashing the process (an EventEmitter error with no listener
     * throws).
     */
    private onSession(session: http2.ServerHttp2Session): void {
        if (this.sessionCount >= this.maxSessions) {
            grpcConnectionsRejected.inc({ reason: 'max_sessions' })
            session.destroy()
            return
        }
        this.sessionCount++
        this.sessions.add(session)
        session.setTimeout(this.sessionIdleTimeoutMs, () => session.close())
        session.on('error', (error) => logger.warn('🛜', 'WorkerIngest session error', { error }))
        session.once('close', () => {
            this.sessionCount--
            this.sessions.delete(session)
        })
    }

    /**
     * Drains before tearing down: fed sub-batches settle and their acks flush,
     * so a scale-down does not make consumers replay already-processed work.
     */
    async stop(): Promise<void> {
        this.draining = true
        this.drainAbort.abort()
        // A poisoned pipeline never settles, so skip straight to teardown.
        const deadline = this.fatalError ? 0 : Date.now() + this.drainTimeoutMs
        while (this.totalInFlight() > 0 && Date.now() < deadline) {
            await sleep(10)
        }
        this.stopped = true
        this.stopAbort.abort()
        for (const stream of this.streams.values()) {
            if (stream.inFlight.size === 0) {
                stream.acks.end()
            } else {
                stream.acks.fail(new ConnectError('server shutting down', Code.Unavailable))
            }
        }
        this.wake()
        // http2's close() waits for every session to end and consumers hold
        // theirs open, so send a graceful GOAWAY instead of waiting.
        for (const session of this.sessions) {
            session.close()
        }
        if (this.server) {
            await new Promise<void>((resolve) => this.server!.close(() => resolve()))
            this.server = null
        }
        // A poisoned pipeline can leave the pump blocked in a next() that will
        // never resolve — bound the wait instead of hanging shutdown on it.
        await Promise.race([this.pumpTask, sleep(5_000)])
    }

    /** Open stream count, exposed for tests. */
    get streamCount(): number {
        return this.streams.size
    }

    private wake(): void {
        if (this.wakePump) {
            const wake = this.wakePump
            this.wakePump = null
            wake()
        }
    }

    private totalInFlight(): number {
        let total = 0
        for (const stream of this.streams.values()) {
            total += stream.inFlight.size
        }
        return total
    }

    /**
     * The single pump: drains completed batches from the driver and routes
     * acks to their stream. One loop per server — completed batches carry
     * their stream id, so streams never contend over the pipeline's results.
     */
    private async runPump(): Promise<void> {
        while (!this.stopped) {
            if (this.totalInFlight() === 0) {
                await new Promise<void>((resolve) => {
                    this.wakePump = resolve
                })
                continue
            }
            try {
                const completed = await this.raceStopped(this.deps.driver.next())
                if (completed === undefined) {
                    return
                }
                if (completed === null) {
                    // The pipeline drained between feeds while acked work is
                    // still being registered — yield briefly and re-pump.
                    await sleep(this.pumpIdleMs)
                    continue
                }
                // Settle concurrently: the pump goes straight back to the
                // pipeline while this batch's side effects finish, so one slow
                // batch cannot head-of-line block other streams' acks.
                this.settleAndAck(completed)
            } catch (error) {
                this.fatal(error instanceof Error ? error : new Error(String(error)))
                return
            }
        }
    }

    /**
     * Like Promise.race(work, stop) resolving `undefined` on stop, but without
     * race's per-call listener leak: the stop listener is removed once work
     * settles.
     */
    private raceStopped<T>(work: Promise<T>): Promise<T | undefined> {
        if (this.stopAbort.signal.aborted) {
            return Promise.resolve(undefined)
        }
        return new Promise((resolve, reject) => {
            const onStop = (): void => resolve(undefined)
            this.stopAbort.signal.addEventListener('abort', onStop, { once: true })
            work.then(
                (value) => {
                    this.stopAbort.signal.removeEventListener('abort', onStop)
                    resolve(value)
                },
                (error) => {
                    this.stopAbort.signal.removeEventListener('abort', onStop)
                    reject(error)
                }
            )
        })
    }

    private settleAndAck(completed: CompletedSubBatch): void {
        void (async () => {
            try {
                await completed.settled
            } catch (error) {
                this.releaseAccepted(completed, 'failed')
                this.slots.release()
                this.fatal(error instanceof Error ? error : new Error(String(error)))
                return
            }
            // The batch's admission slot frees only once its side effects are
            // durably done, matching the HTTP path's response barrier.
            this.releaseAccepted(completed, 'ok')
            this.slots.release()
            this.ackCompleted(completed)
        })()
    }

    private acceptedKey(streamId: number, seq: number): string {
        return `${streamId}:${seq}`
    }

    private releaseAccepted(completed: CompletedSubBatch, outcome: 'ok' | 'failed'): void {
        const key = this.acceptedKey(completed.streamId, completed.seq)
        const accepted = this.acceptedBatches.get(key)
        if (!accepted) {
            return
        }
        this.acceptedBatches.delete(key)
        if (outcome === 'ok') {
            batchProcessed(accepted.acceptedAt, completed.accepted)
        } else {
            batchFailed()
        }
        batchReleased(accepted)
    }

    private ackCompleted(completed: CompletedSubBatch): void {
        const stream = this.streams.get(completed.streamId)
        if (!stream) {
            // The stream died while its sub-batch was processing; the consumer
            // replays the un-acked tail on its next stream.
            return
        }
        stream.inFlight.delete(completed.seq)
        if (completed.unfinished.length > 0) {
            stream.acks.push(partialAck(completed.seq, completed.accepted, completed.unfinished))
            grpcSubBatches.inc({ status: 'partial' })
        } else {
            stream.acks.push(okAck(completed.seq, completed.accepted))
            grpcSubBatches.inc({ status: 'ok' })
        }
        grpcMessagesProcessed.inc(completed.accepted)
        this.maybeFinish(stream)
    }

    private expireParked(stream: StreamState, seq: number, messages: SerializedKafkaMessage[]): void {
        grpcAdmissionExpired.inc()
        stream.inFlight.delete(seq)
        stream.acks.push(partialAck(seq, 0, messages))
        grpcSubBatches.inc({ status: 'partial' })
        this.maybeFinish(stream)
    }

    private maybeFinish(stream: StreamState): void {
        if (stream.readerDone && stream.inFlight.size === 0) {
            stream.acks.end()
        }
    }

    /** Stop reading without failing the stream; pending acks still flush. */
    private finishReader(stream: StreamState, unfedSeq?: number): void {
        if (unfedSeq !== undefined) {
            stream.inFlight.delete(unfedSeq)
        }
        stream.readerDone = true
        this.maybeFinish(stream)
    }

    /**
     * A pipeline failure poisons the shared pipeline (same contract as the
     * HTTP handler's fatal path): fail every stream so consumers tear down
     * their lanes and replay, then hand the error to the owner to shut down.
     */
    private fatal(error: Error): void {
        if (this.fatalError) {
            return
        }
        this.fatalError = error
        logger.error('💥', 'WorkerIngest pipeline failed — failing all streams', { error: error.message })
        for (const stream of this.streams.values()) {
            stream.acks.fail(new ConnectError(`ingest pipeline failed: ${error.message}`, Code.Internal))
        }
        this.deps.onFatal(error)
    }

    async *ingestStream(
        requests: AsyncIterable<IngestStreamRequest>,
        signal?: AbortSignal
    ): AsyncIterable<IngestStreamResponse> {
        if (this.fatalError) {
            throw new ConnectError('ingest pipeline is poisoned', Code.Unavailable)
        }
        if (this.draining) {
            throw new ConnectError('server shutting down', Code.Unavailable)
        }
        // Total stream ceiling across every session: the per-session SETTINGS cap
        // bounds one peer, this bounds the whole server if a peer ignores it.
        if (this.streams.size >= this.maxStreams) {
            grpcConnectionsRejected.inc({ reason: 'max_streams' })
            throw new ConnectError('too many concurrent streams', Code.ResourceExhausted)
        }
        const stream: StreamState = {
            id: this.nextStreamId++,
            hello: null,
            nextSeq: 1,
            inFlight: new Set(),
            readerDone: false,
            acks: new AckQueue(),
        }
        this.streams.set(stream.id, stream)
        grpcStreams.inc()

        // connect-node defers response headers until the first response
        // message and the consumer's stream-open awaits them, so greet first.
        stream.acks.push(readyFrame())

        const reader = this.runReader(stream, requests, signal)
        // Reader failures surface through the ack queue.
        swallowRejection(reader)

        try {
            for await (const ack of stream.acks) {
                yield ack
            }
            // Normal completion: propagate a reader outcome the queue may not
            // have carried (it can only fail after a waiter is registered).
            await reader
        } finally {
            this.streams.delete(stream.id)
            grpcStreams.dec()
            this.wake()
        }
    }

    private async runReader(
        stream: StreamState,
        requests: AsyncIterable<IngestStreamRequest>,
        signal?: AbortSignal
    ): Promise<void> {
        // Admission waits abort on disconnect and on drain: either way the
        // sub-batch was never fed, so the consumer replays it for free.
        const admissionSignal = signal ? AbortSignal.any([signal, this.drainAbort.signal]) : this.drainAbort.signal
        try {
            for await (const request of requests) {
                if (this.draining) {
                    this.finishReader(stream)
                    return
                }
                if (request.msg.case === 'hello') {
                    if (stream.hello) {
                        throw this.protocolError(stream, 'duplicate_hello', 'received a second hello frame')
                    }
                    stream.hello = request.msg.value
                    continue
                }
                if (request.msg.case !== 'subBatch') {
                    throw this.protocolError(stream, 'unknown_frame', 'frame is neither hello nor sub_batch')
                }
                if (!stream.hello) {
                    throw this.protocolError(stream, 'missing_hello', 'received a sub-batch before the hello frame')
                }

                const subBatch = request.msg.value
                const budget: SubBatchBudget = {
                    armedAt: Date.now(),
                    softBudgetMs: Number(subBatch.softBudgetMs),
                }
                const seq = Number(subBatch.seq)
                if (seq !== stream.nextSeq) {
                    throw this.protocolError(
                        stream,
                        'seq_mismatch',
                        `expected seq ${stream.nextSeq}, got ${seq} — frames were lost or reordered`
                    )
                }
                stream.nextSeq++

                if (subBatch.messages.length === 0) {
                    // An empty feed never completes a batch, so ack it directly.
                    stream.acks.push(okAck(seq, 0))
                    continue
                }

                const serialized = subBatch.messages.map(grpcMessageToSerialized)
                // The sentinel's sender scope includes both epochs, so
                // reconnects and rebalances rebaseline instead of counting
                // legitimate replays as violations.
                this.deps.feedOrderSentinel?.check(
                    serialized,
                    `${stream.hello.consumerId}#${stream.hello.streamEpoch}#${subBatch.assignmentEpoch}`,
                    subBatch.replay
                )

                // Await admission before reading the next frame: stream order
                // becomes feed order, and the stalled read backpressures the
                // consumer via HTTP/2 flow control. Slots are granted in FIFO
                // order because a retry race here let busy streams starve
                // quiet ones, wedging whole consumers.
                stream.inFlight.add(seq)
                // A sub-batch that spends its whole allowance waiting here is given up, not started.
                const softAt = budgetDeadline(budget.armedAt, budget.softBudgetMs)
                const parkedTooLong = new AbortController()
                const expiryTimer =
                    softAt === null ? null : setTimeout(() => parkedTooLong.abort(), Math.max(0, softAt - Date.now()))
                expiryTimer?.unref?.()
                try {
                    await this.slots.acquire(AbortSignal.any([admissionSignal, parkedTooLong.signal]))
                } catch (error) {
                    if (error instanceof SlotAcquisitionAborted) {
                        if (parkedTooLong.signal.aborted && !admissionSignal.aborted) {
                            this.expireParked(stream, seq, serialized)
                            continue
                        }
                        this.finishReader(stream, seq)
                        return
                    }
                    throw error
                } finally {
                    if (expiryTimer) {
                        clearTimeout(expiryTimer)
                    }
                }
                if (signal?.aborted) {
                    // Cancelled between the slot grant and here: hand the slot
                    // straight back so a live stream gets it, and stop reading.
                    this.slots.release()
                    return
                }
                let feedAccepted = false
                try {
                    while (true) {
                        if (this.draining) {
                            this.finishReader(stream, seq)
                            return
                        }
                        const result = await this.feedGuarded(stream, seq, serialized, budget)
                        if (result.ok) {
                            feedAccepted = true
                            this.acceptedBatches.set(this.acceptedKey(stream.id, seq), batchAccepted(serialized.length))
                            break
                        }
                        if (result.kind === 'at_capacity') {
                            // The admission queue mirrors pipeline capacity, so
                            // this means the two drifted — count it and retry.
                            grpcCapacityRetries.inc()
                            await sleep(this.capacityRetryMs)
                            continue
                        }
                        stream.inFlight.delete(seq)
                        stream.acks.push(failedAck(seq, result.reason))
                        grpcSubBatches.inc({ status: 'failed' })
                        throw new ConnectError(`sub-batch ${seq} rejected: ${result.reason}`, Code.Internal)
                    }
                } finally {
                    // An accepted feed hands its slot to the pump, which
                    // releases it after the batch settles; every other exit
                    // must give the slot back here.
                    if (!feedAccepted) {
                        this.slots.release()
                    }
                }
                this.wake()
            }
            this.finishReader(stream)
        } catch (error) {
            stream.readerDone = true
            const connectError = error instanceof ConnectError ? error : new ConnectError(String(error), Code.Internal)
            stream.acks.fail(connectError)
            throw connectError
        }
    }

    /** Feed through the driver, converting a thrown pipeline error into the fatal path. */
    private async feedGuarded(
        stream: StreamState,
        seq: number,
        messages: SerializedKafkaMessage[],
        budget: SubBatchBudget
    ): Promise<FeedResult> {
        try {
            return await this.deps.driver.feed(stream.id, seq, messages, budget)
        } catch (error) {
            this.fatal(error instanceof Error ? error : new Error(String(error)))
            throw new ConnectError('ingest pipeline failed', Code.Internal)
        }
    }

    private protocolError(stream: StreamState, kind: string, message: string): ConnectError {
        grpcProtocolErrors.inc({ kind })
        logger.warn('⚠️', 'WorkerIngest protocol violation', { streamId: stream.id, kind, message })
        return new ConnectError(message, Code.FailedPrecondition)
    }
}

function sleep(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms))
}

/** For promises whose failure is reported elsewhere; an unobserved rejection would crash the process. */
function swallowRejection(promise: Promise<unknown>): void {
    promise.catch(() => {})
}