Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

77.5 KB
Raw Blame
// Tests for ingest-handler.ts
//
// Coverage goals (mirrors products/tasks/backend/stream/tests/):
//   - NDJSON parsing across chunk boundaries
//   - Byte/line/count limits → 413 with last_accepted_seq
//   - Sequence gap + already-completed → 409
//   - Accepted/duplicate counting + last_accepted_seq tracking
//   - Completion line handling (happy path, mismatch, ordering errors)
//   - Side-effect triggering: turn-complete vs session/update vs throttled heartbeat
//   - Best-effort side effects: callback failure does not fail ingest
//
// Wire-protocol invariants tested:
//   - Stream key: "task-run-stream:{run_id}"
//   - Sequence numbers start at 1 (0 is the initial sentinel)
//   - Completion sentinel: {"type":"STREAM_STATUS","status":"complete"}
//   - Redis field name is "data"

import type { Redis } from 'ioredis'
import { describe, it, expect, vi, beforeEach } from 'vitest'

import { observeStreamWriteSkipped } from '@/hono/metrics.js'
import type { Config } from '@/lib/config.js'
import {
    MAX_EVENT_LINE_BYTES,
    MAX_REQUEST_BYTES,
    MAX_EVENTS_PER_REQUEST,
    HEARTBEAT_THROTTLE_SECONDS,
} from '@/lib/constants.js'
import { logger } from '@/lib/logging.js'
import { TaskRunRedisStream, getCompletedKey, getStreamKey, getWatchedKey } from '@/lib/redis-stream.js'
import { heartbeatWorkflowIfNeeded, notifyPermissionRequestIfNeeded } from '@/lib/side-effects.js'
import type { SandboxEventIngestTokenPayload } from '@/lib/types.js'

// ---------------------------------------------------------------------------
// Minimal in-memory Redis fake
//
// Implements: get, set, exists, del, expire, xadd, watch, unwatch, multi
// Enough surface for TaskRunRedisStream.writeEventWithSequence,
// getLastSequence, setAgentActive, getAgentActive, claimAgentActiveHeartbeat,
// markCompleteAfterSequence, and deleteStream.
// ---------------------------------------------------------------------------

interface StreamEntry {
    id: string
    data: Record<string, string>
}

class FakeRedis {
    private strings = new Map<string, { value: string; expireAt: number | null }>()
    private streams = new Map<string, StreamEntry[]>()
    private watchedKeys = new Set<string>()
    private watchConflict = false
    private seq = 0

    // Allow tests to simulate a WATCH conflict once.
    simulateWatchConflict(): void {
        this.watchConflict = true
    }

    private nextStreamId(): string {
        this.seq++
        return `${Date.now()}-${this.seq}`
    }

    async get(key: string): Promise<string | null> {
        const entry = this.strings.get(key)
        if (!entry) {
            return null
        }
        if (entry.expireAt !== null && Date.now() > entry.expireAt) {
            this.strings.delete(key)
            return null
        }
        return entry.value
    }

    async set(key: string, value: string, ...args: (string | number)[]): Promise<string | null> {
        const previous = await this.get(key)
        let ttl: number | null = null
        let nx = false
        for (let i = 0; i < args.length; i++) {
            const arg = args[i]
            if (typeof arg === 'string' && arg.toUpperCase() === 'EX') {
                const raw = args[i + 1]
                ttl = typeof raw === 'number' ? raw : parseInt(String(raw), 10)
                i++
            }
            if (typeof arg === 'string' && arg.toUpperCase() === 'NX') {
                nx = true
            }
        }
        if (nx && this.strings.has(key)) {
            const existing = this.strings.get(key)!
            if (existing.expireAt === null || Date.now() <= existing.expireAt) {
                return 'OK' as const // NX: don't overwrite — return 'OK' but value unchanged
                // NOTE: real ioredis returns null on NX fail. We handle this below.
            }
        }
        this.strings.set(key, {
            value,
            expireAt: ttl !== null ? Date.now() + ttl * 1000 : null,
        })
        return args.includes('GET') ? previous : 'OK'
    }

    // Real ioredis returns null when SET NX fails. Override for that contract.
    async setNX(key: string, value: string, ttl: number): Promise<string | null> {
        const existing = this.strings.get(key)
        if (existing && (existing.expireAt === null || Date.now() <= existing.expireAt)) {
            return null
        }
        this.strings.set(key, { value, expireAt: Date.now() + ttl * 1000 })
        return 'OK'
    }

    async exists(...keys: string[]): Promise<number> {
        let count = 0
        for (const key of keys) {
            const entry = this.strings.get(key)
            if (entry && (entry.expireAt === null || Date.now() <= entry.expireAt)) {
                count++
            }
        }
        return count
    }

    async expire(key: string, seconds: number): Promise<number> {
        const entry = this.strings.get(key)
        if (entry) {
            entry.expireAt = Date.now() + seconds * 1000
        }
        return entry ? 1 : 0
    }

    async del(...keys: string[]): Promise<number> {
        let deleted = 0
        for (const key of keys) {
            if (this.strings.delete(key)) {
                deleted++
            }
            if (this.streams.delete(key)) {
                deleted++
            }
        }
        return deleted
    }

    async xadd(
        key: string,
        _maxlenToken: 'MAXLEN',
        _approxToken: '~',
        _maxLen: number,
        _idSpec: '*',
        ...fieldValues: string[]
    ): Promise<string> {
        const id = this.nextStreamId()
        const data: Record<string, string> = {}
        for (let i = 0; i + 1 < fieldValues.length; i += 2) {
            data[fieldValues[i]!] = fieldValues[i + 1]!
        }
        const entries = this.streams.get(key) ?? []
        entries.push({ id, data })
        this.streams.set(key, entries)
        return id
    }

    async xrange(key: string, _start = '-', _end = '+'): Promise<[string, Record<string, string>][]> {
        const entries = this.streams.get(key) ?? []
        return entries.map((e) => [e.id, e.data])
    }

    async watch(...keys: string[]): Promise<'OK'> {
        for (const k of keys) {
            this.watchedKeys.add(k)
        }
        return 'OK'
    }

    async unwatch(): Promise<'OK'> {
        this.watchedKeys.clear()
        return 'OK'
    }

    // Returns a fake pipeline (MULTI).
    multi(): FakePipeline {
        const conflict = this.watchConflict
        this.watchConflict = false
        return new FakePipeline(this, conflict)
    }
}

// Rewrite `set` so NX behaviour is correct (returns null not 'OK' on failure).
// We patch FakeRedis.set to correctly handle NX per ioredis contract.
const _origSet = FakeRedis.prototype.set
FakeRedis.prototype.set = async function (
    this: FakeRedis,
    key: string,
    value: string,
    ...args: (string | number)[]
): Promise<string | null> {
    let nx = false
    for (const arg of args) {
        if (typeof arg === 'string' && arg.toUpperCase() === 'NX') {
            nx = true
        }
    }
    if (nx) {
        const existing = await this.get(key)
        if (existing !== null) {
            // NX fail — don't overwrite, return null (ioredis contract)
            return null as unknown as 'OK'
        }
    }
    return _origSet.call(this, key, value, ...args)
} as typeof FakeRedis.prototype.set

class FakePipeline {
    private ops: Array<() => Promise<[null, unknown]>> = []
    private conflictOnExec: boolean

    constructor(
        private redis: FakeRedis,
        conflict: boolean
    ) {
        this.conflictOnExec = conflict
    }

    xadd(
        key: string,
        maxlenToken: 'MAXLEN',
        approxToken: '~',
        maxLen: number,
        idSpec: '*',
        ...fieldValues: string[]
    ): this {
        this.ops.push(async () => {
            const id = await this.redis.xadd(key, maxlenToken, approxToken, maxLen, idSpec, ...fieldValues)
            return [null, id]
        })
        return this
    }

    set(key: string, value: string, ...args: (string | number)[]): this {
        this.ops.push(async () => {
            const result = await this.redis.set(key, value, ...args)
            return [null, result]
        })
        return this
    }

    expire(key: string, seconds: number): this {
        this.ops.push(async () => {
            const result = await this.redis.expire(key, seconds)
            return [null, result]
        })
        return this
    }

    async exec(): Promise<Array<[null, unknown]> | null> {
        if (this.conflictOnExec) {
            return null
        }
        const results: Array<[null, unknown]> = []
        for (const op of this.ops) {
            results.push(await op())
        }
        return results
    }
}

// ---------------------------------------------------------------------------
// Test helpers
// ---------------------------------------------------------------------------

function makeFakeRedis(): FakeRedis {
    return new FakeRedis()
}

function makeRedisStream(fakeRedis: FakeRedis, runId: string): TaskRunRedisStream {
    return new TaskRunRedisStream(getStreamKey(runId), fakeRedis as unknown as Redis, { timeout: 60, maxLength: 20000 })
}

// Build a minimal Config with no Django callback URL so side-effect HTTP calls are skipped.
function makeConfig(overrides?: Partial<Config>): Config {
    return {
        redisUrl: 'redis://localhost:6379',
        sandboxJwtPublicKeysPem: [],
        corsOrigins: new Set(),
        djangoCallbackBaseUrl: '',
        agentProxyCallbackSecret: '',
        maxConcurrentStreams: 1000,
        maxStreamsPerRun: 25,
        metricsToken: '',
        port: 8003,
        host: '0.0.0.0',
        shutdownGraceMs: 300_000,
        shutdownPrestopDelayMs: 0,
        ...overrides,
    }
}

// Build claims for the given runId/taskId/teamId.
function makeClaims(overrides?: Partial<SandboxEventIngestTokenPayload>): SandboxEventIngestTokenPayload {
    return {
        runId: 'run-123',
        taskId: 'task-abc',
        teamId: 42,
        presenceGated: false,
        thinTail: false,
        originProduct: 'unknown',
        ...overrides,
    }
}

// Build a ReadableStream body from an array of Uint8Array chunks.
function makeChunkedBody(chunks: Uint8Array[]): ReadableStream<Uint8Array> {
    let idx = 0
    return new ReadableStream({
        pull(controller) {
            const chunk = chunks[idx++]
            if (chunk === undefined) {
                controller.close()
                return
            }
            controller.enqueue(chunk)
        },
    })
}

// Build a ReadableStream body from a plain string.
function makeStringBody(text: string): ReadableStream<Uint8Array> {
    const encoded = new TextEncoder().encode(text)
    return makeChunkedBody([encoded])
}

// Import the private helpers by re-exporting a thin shim that calls them.
// Because ingest-handler.ts doesn't export the inner loop helpers, we drive
// the behavior through the exported handleIngest() function with a mocked Hono
// Context, a mocked JWT validator, and a mocked run-exists check.
//
// We avoid unit-testing the private functions directly; instead we observe
// their behavior via the public surface (handleIngest or the sub-module
// heartbeatWorkflowIfNeeded exported from side-effects.ts).

// Inline the re-exported NDJSON + Redis write helpers under test via the
// ingestEventLines path driven through the public handler.
// We wire handleIngest with a fake Hono context and a pre-validated claims object
// (by vi.mocking the JWT module).

vi.mock('@/lib/jwt.js', () => ({
    validateSandboxEventIngestToken: vi.fn(),
    validateStreamReadToken: vi.fn(),
    loadPublicKey: vi.fn(),
}))

import { handleIngest } from '@/hono/ingest-handler.js'
import { validateSandboxEventIngestToken } from '@/lib/jwt.js'

const mockValidate = vi.mocked(validateSandboxEventIngestToken)

// Build a minimal Hono Context-like object sufficient for handleIngest.
function makeContext(opts: {
    runId?: string
    method?: string
    token?: string
    body?: ReadableStream<Uint8Array> | null
}): Parameters<typeof handleIngest>[0] {
    const { runId = 'run-123', method = 'POST', token = 'test-token', body = null } = opts

    return {
        req: {
            method,
            param: () => ({ run: runId }),
            header: (name: string) => {
                const lc = name.toLowerCase()
                if (lc === 'authorization') {
                    return token ? `Bearer ${token}` : undefined
                }
                return undefined
            },
            raw: new Request('http://localhost/', {
                method: 'POST',
                body,
                // Required for streaming body in Node
                // @ts-expect-error duplex is Node-only
                duplex: 'half',
            }),
        },
        json: (data: unknown, status: number) => {
            return new Response(JSON.stringify(data), {
                status,
                headers: { 'Content-Type': 'application/json' },
            })
        },
    } as unknown as Parameters<typeof handleIngest>[0]
}

// Decode a JSON response body.
async function decodeJson(res: Response): Promise<unknown> {
    return res.json()
}

// ---------------------------------------------------------------------------
// Describe block
// ---------------------------------------------------------------------------

describe('ingest-handler', () => {
    let fakeRedis: FakeRedis
    let redisStream: TaskRunRedisStream
    const RUN_ID = 'run-123'
    const TASK_ID = 'task-abc'
    const TEAM_ID = 42

    beforeEach(() => {
        fakeRedis = makeFakeRedis()
        redisStream = makeRedisStream(fakeRedis, RUN_ID)
        vi.clearAllMocks()
        // Default: token is valid, claims match the route params
        mockValidate.mockResolvedValue(makeClaims())
    })

    it.each([
        { line: { seq: 1, event: { type: 'notification', notification: { method: 'late' } } }, legacy: false },
        { line: { type: '_posthog/stream_complete', final_seq: 0 }, legacy: false },
        { line: { seq: 1, event: { type: 'notification', notification: { method: 'late' } } }, legacy: true },
        { line: { type: '_posthog/stream_complete', final_seq: 0 }, legacy: true },
    ])('rejects an old sender after the stream resumes: %j', async ({ line, legacy }) => {
        mockValidate.mockResolvedValue({ ...makeClaims(), ...(legacy ? {} : { streamGeneration: 'initial' }) })
        await fakeRedis.set(`${getStreamKey(RUN_ID)}:generation`, 'initial')
        const initial = await handleIngest(
            makeContext({ body: makeStringBody(JSON.stringify(line) + '\n') }),
            fakeRedis as unknown as Redis,
            makeConfig(),
            []
        )
        expect(initial.status).toBe(200)
        await redisStream.deleteStream()
        await fakeRedis.set(`${getStreamKey(RUN_ID)}:generation`, 'replacement')
        const response = await handleIngest(
            makeContext({ body: makeStringBody(JSON.stringify(line) + '\n') }),
            fakeRedis as unknown as Redis,
            makeConfig(),
            []
        )
        expect(response.status).toBe(403)
        expect(await response.json()).not.toHaveProperty('last_accepted_seq')
        expect(await redisStream.getLastSequence()).toBe(0)
        expect(await redisStream.isComplete()).toBe(false)
    })

    // -----------------------------------------------------------------------
    // Auth / routing guards
    // -----------------------------------------------------------------------

    it('returns 405 on non-POST', async () => {
        const config = makeConfig()
        const ctx = makeContext({ method: 'GET' })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(405)
    })

    it('returns 401 when Authorization header is missing', async () => {
        const config = makeConfig()
        const ctx = makeContext({ token: '' })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(401)
    })

    it('returns 401 when JWT validation fails', async () => {
        mockValidate.mockRejectedValue(new Error('invalid signature'))
        const config = makeConfig()
        const ctx = makeContext({ body: makeStringBody('') })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(401)
    })

    it('returns 403 when run id in claims does not match route', async () => {
        mockValidate.mockResolvedValue(makeClaims({ runId: 'different-run' }))
        const config = makeConfig()
        const ctx = makeContext({ body: makeStringBody('') })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(403)
    })

    // -----------------------------------------------------------------------
    // Happy-path: single event accepted
    // -----------------------------------------------------------------------

    it('returns 200 with accepted=1 duplicate=0 last_accepted_seq=1 for a single valid event', async () => {
        const config = makeConfig()
        const ndjson = JSON.stringify({ seq: 1, event: { type: 'message' } }) + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)
        const body = await decodeJson(res)
        expect(body).toMatchObject({ accepted: 1, duplicate: 0, last_accepted_seq: 1 })
    })

    // -----------------------------------------------------------------------
    // Presence gating
    // -----------------------------------------------------------------------

    it.each([
        { name: 'skips the mirror when no reader is attached', watched: false, expectedEntries: 0 },
        { name: 'mirrors when a reader is attached', watched: true, expectedEntries: 1 },
    ])('presence-gated ingest $name', async ({ watched, expectedEntries }) => {
        mockValidate.mockResolvedValue(makeClaims({ presenceGated: true, originProduct: 'signals_scout' }))
        if (watched) {
            await fakeRedis.set(getWatchedKey(getStreamKey(RUN_ID)), '1', 'EX', 300)
        }
        const config = makeConfig()
        const ndjson = JSON.stringify({ seq: 1, event: { type: 'message' } }) + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })

        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])

        expect(res.status).toBe(200)
        expect(await decodeJson(res)).toMatchObject({ accepted: 1, duplicate: 0, last_accepted_seq: 1 })
        expect(await redisStream.getLastSequence()).toBe(1)
        expect(await fakeRedis.xrange(getStreamKey(RUN_ID))).toHaveLength(expectedEntries)
        if (watched) {
            expect(observeStreamWriteSkipped).not.toHaveBeenCalled()
        } else {
            expect(observeStreamWriteSkipped).toHaveBeenCalledWith('ingest', 'signals_scout')
        }
    })

    it('still writes the completion sentinel for a presence-gated run with no reader', async () => {
        mockValidate.mockResolvedValue(makeClaims({ presenceGated: true }))
        const config = makeConfig()
        const ndjson =
            JSON.stringify({ seq: 1, event: { type: 'message' } }) +
            '\n' +
            JSON.stringify({ type: '_posthog/stream_complete', final_seq: 1 }) +
            '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })

        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])

        expect(res.status).toBe(200)
        const entries = await fakeRedis.xrange(getStreamKey(RUN_ID))
        expect(entries).toHaveLength(1)
        expect(JSON.parse(entries[0]![1]['data']!)).toEqual({ type: 'STREAM_STATUS', status: 'complete' })
    })

    it('writes complete NDJSON lines before the request body closes', async () => {
        const config = makeConfig()
        const encoder = new TextEncoder()
        let controller!: ReadableStreamDefaultController<Uint8Array>
        const body = new ReadableStream<Uint8Array>({
            start(c) {
                controller = c
            },
        })

        const resPromise = handleIngest(makeContext({ body }), fakeRedis as unknown as Redis, config, [] as CryptoKey[])

        controller.enqueue(encoder.encode(`${JSON.stringify({ seq: 1, event: { type: 'first-live' } })}\n`))

        await vi.waitFor(async () => {
            const entries = await fakeRedis.xrange(getStreamKey(RUN_ID))
            expect(entries).toHaveLength(1)
            expect(entries[0]?.[1].data).toContain('"first-live"')
        })

        controller.enqueue(encoder.encode(`${JSON.stringify({ seq: 2, event: { type: 'second-live' } })}\n`))
        controller.close()

        const res = await resPromise
        expect(res.status).toBe(200)
        const responseBody = await decodeJson(res)
        expect(responseBody).toMatchObject({ accepted: 2, duplicate: 0, last_accepted_seq: 2 })
    })

    it('accepts multiple sequential events', async () => {
        const config = makeConfig()
        const lines =
            [
                JSON.stringify({ seq: 1, event: { type: 'a' } }),
                JSON.stringify({ seq: 2, event: { type: 'b' } }),
                JSON.stringify({ seq: 3, event: { type: 'c' } }),
            ].join('\n') + '\n'
        const ctx = makeContext({ body: makeStringBody(lines) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)
        const body = (await decodeJson(res)) as { accepted: number; duplicate: number; last_accepted_seq: number }
        expect(body.accepted).toBe(3)
        expect(body.duplicate).toBe(0)
        expect(body.last_accepted_seq).toBe(3)
    })

    // -----------------------------------------------------------------------
    // Duplicate handling
    // -----------------------------------------------------------------------

    it('counts duplicate when same seq sent twice', async () => {
        const config = makeConfig()
        // First request: accept seq 1
        const ndjson1 = JSON.stringify({ seq: 1, event: { type: 'first' } }) + '\n'
        const ctx1 = makeContext({ body: makeStringBody(ndjson1) })
        await handleIngest(ctx1, fakeRedis as unknown as Redis, config, [] as CryptoKey[])

        // Second request: resend seq 1 → duplicate
        const ndjson2 = JSON.stringify({ seq: 1, event: { type: 'first-again' } }) + '\n'
        const ctx2 = makeContext({ body: makeStringBody(ndjson2) })
        const res2 = await handleIngest(ctx2, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res2.status).toBe(200)
        const body = (await decodeJson(res2)) as { accepted: number; duplicate: number; last_accepted_seq: number }
        expect(body.accepted).toBe(0)
        expect(body.duplicate).toBe(1)
        expect(body.last_accepted_seq).toBe(1)
    })

    it('sends budget steers through the authenticated callback on accepted and replayed events', async () => {
        const callback = vi
            .spyOn(globalThis, 'fetch')
            .mockResolvedValue(new Response(JSON.stringify({ dispatched: true }), { status: 200 }))
        try {
            const config = makeConfig({
                djangoCallbackBaseUrl: 'http://django.example.com',
                agentProxyCallbackSecret: 'proxy-secret',
            })
            const event = {
                type: 'notification',
                timestamp: '2026-01-01T00:00:06.000Z',
                notification: {
                    method: '_posthog/budget_steer',
                    params: {
                        stage: 'critical',
                        mode: 'publish',
                        delivered: true,
                        spent_usd: 7,
                        cap_usd: 10,
                        threshold_spent_usd: 6.5,
                        threshold_at: '2026-01-01T00:00:00.000Z',
                        delivered_at: '2026-01-01T00:00:05.000Z',
                        team_id: 999,
                        secret: 'not-for-analytics',
                    },
                },
            }
            const body = makeStringBody(JSON.stringify({ seq: 1, event }) + '\n')
            const first = await handleIngest(
                makeContext({ body }),
                fakeRedis as unknown as Redis,
                config,
                [] as CryptoKey[]
            )
            expect(first.status).toBe(200)
            const retry = await handleIngest(
                makeContext({ body: makeStringBody(JSON.stringify({ seq: 1, event }) + '\n') }),
                fakeRedis as unknown as Redis,
                config,
                [] as CryptoKey[]
            )
            expect(await decodeJson(retry)).toMatchObject({ accepted: 0, duplicate: 1 })
            await vi.waitFor(() => expect(callback).toHaveBeenCalledTimes(2))
            const [url, request] = callback.mock.calls[0]!
            expect(url).toBe('http://django.example.com/internal/tasks/runs/run-123/agent-proxy-callback/')
            expect(request?.headers).toMatchObject({
                Authorization: 'Bearer test-token',
                'X-Agent-Proxy-Secret': 'proxy-secret',
            })
            expect(JSON.parse(String(request?.body))).toMatchObject({
                kind: 'budget_steer',
                task_id: TASK_ID,
                team_id: TEAM_ID,
                sequence: 1,
                timestamp: '2026-01-01T00:00:06.000Z',
                stage: 'critical',
                mode: 'publish',
                delivered: true,
                threshold_spent_usd: 6.5,
                threshold_at: '2026-01-01T00:00:00.000Z',
                delivered_at: '2026-01-01T00:00:05.000Z',
            })
            expect(String(request?.body)).not.toContain('not-for-analytics')
            expect(callback.mock.calls[1]![1]?.body).toBe(request?.body)
        } finally {
            callback.mockRestore()
        }
    })

    it('forwards a process kill once, without the command line', async () => {
        const callback = vi
            .spyOn(globalThis, 'fetch')
            .mockResolvedValue(new Response(JSON.stringify({ dispatched: true }), { status: 200 }))
        try {
            const config = makeConfig({ djangoCallbackBaseUrl: 'http://django.example.com' })
            const event = {
                type: 'notification',
                notification: {
                    method: '_posthog/process_killed',
                    params: {
                        pid: 443,
                        comm: 'bash',
                        signal: 'SIGTERM',
                        treeRssBytes: 12_884_901_888,
                        memoryCurrentBytes: 14_698_577_920,
                        memoryLimitBytes: 17_179_869_184,
                        cmdline: ['bash', '-c', 'deploy --token not-for-analytics'],
                    },
                },
            }
            const line = JSON.stringify({ seq: 1, event }) + '\n'
            for (let attempt = 0; attempt < 2; attempt++) {
                const response = await handleIngest(
                    makeContext({ body: makeStringBody(line) }),
                    fakeRedis as unknown as Redis,
                    config,
                    [] as CryptoKey[]
                )
                expect(response.status).toBe(200)
            }
            await vi.waitFor(() => expect(callback).toHaveBeenCalledTimes(1))
            const request = callback.mock.calls[0]![1]
            expect(JSON.parse(String(request?.body))).toMatchObject({
                kind: 'process_killed',
                task_id: TASK_ID,
                team_id: TEAM_ID,
                sequence: 1,
                process_killed: {
                    comm: 'bash',
                    signal: 'SIGTERM',
                    tree_rss_bytes: 12_884_901_888,
                    memory_current_bytes: 14_698_577_920,
                    memory_limit_bytes: 17_179_869_184,
                },
            })
            expect(String(request?.body)).not.toContain('not-for-analytics')
        } finally {
            callback.mockRestore()
        }
    })

    it('reports an accepted permission request once, and skips replays and malformed requests', async () => {
        const callback = vi
            .spyOn(globalThis, 'fetch')
            .mockResolvedValue(new Response(JSON.stringify({ dispatched: true }), { status: 200 }))
        try {
            const config = makeConfig({ djangoCallbackBaseUrl: 'http://django.example.com' })
            const params = { requestId: 'perm-1', toolCall: { title: 'Run tests' }, options: [{ optionId: 'allow' }] }
            const request = { type: 'notification', notification: { method: '_posthog/permission_request', params } }
            const malformed = {
                type: 'notification',
                notification: { method: '_posthog/permission_request', params: { ...params, options: [] } },
            }
            const lines = [
                { seq: 1, event: request },
                { seq: 1, event: request },
                { seq: 2, event: malformed },
            ]
            for (const line of lines) {
                const response = await handleIngest(
                    makeContext({ body: makeStringBody(JSON.stringify(line) + '\n') }),
                    fakeRedis as unknown as Redis,
                    config,
                    [] as CryptoKey[]
                )
                expect(response.status).toBe(200)
            }
            expect(callback).toHaveBeenCalledTimes(1)
            expect(JSON.parse(String(callback.mock.calls[0]![1]?.body))).toMatchObject({
                kind: 'permission_requested',
                agent_active: false,
                task_id: TASK_ID,
                team_id: TEAM_ID,
                sequence: 1,
                request_id: 'perm-1',
            })
        } finally {
            callback.mockRestore()
        }
    })

    it('retries a budget steer callback after a temporary capture failure', async () => {
        const callback = vi
            .spyOn(globalThis, 'fetch')
            .mockResolvedValueOnce(new Response('', { status: 503 }))
            .mockResolvedValueOnce(new Response(JSON.stringify({ dispatched: true }), { status: 200 }))
        try {
            const event = {
                type: 'notification',
                notification: {
                    method: '_posthog/budget_steer',
                    params: { stage: 'warn', mode: 'publish', delivered: true, spent_usd: 5, cap_usd: 10 },
                },
            }
            const response = await handleIngest(
                makeContext({ body: makeStringBody(JSON.stringify({ seq: 1, event }) + '\n') }),
                fakeRedis as unknown as Redis,
                makeConfig({ djangoCallbackBaseUrl: 'http://django.example.com' }),
                [] as CryptoKey[]
            )
            expect(response.status).toBe(200)
            await vi.waitFor(() => expect(callback).toHaveBeenCalledTimes(2), { timeout: 3000 })
        } finally {
            callback.mockRestore()
        }
    })

    it('mixes accepted and duplicate when some events are re-sent', async () => {
        const config = makeConfig()
        // Seed seq 1 via a prior request
        const seed = JSON.stringify({ seq: 1, event: { type: 'seed' } }) + '\n'
        await handleIngest(
            makeContext({ body: makeStringBody(seed) }),
            fakeRedis as unknown as Redis,
            config,
            [] as CryptoKey[]
        )

        // Resend seq 1 + new seq 2
        const mixed =
            [
                JSON.stringify({ seq: 1, event: { type: 'duplicate' } }),
                JSON.stringify({ seq: 2, event: { type: 'new' } }),
            ].join('\n') + '\n'
        const res = await handleIngest(
            makeContext({ body: makeStringBody(mixed) }),
            fakeRedis as unknown as Redis,
            config,
            [] as CryptoKey[]
        )
        expect(res.status).toBe(200)
        const body = (await decodeJson(res)) as { accepted: number; duplicate: number; last_accepted_seq: number }
        expect(body.accepted).toBe(1)
        expect(body.duplicate).toBe(1)
        expect(body.last_accepted_seq).toBe(2)
    })

    // -----------------------------------------------------------------------
    // Sequence gap → 409
    // -----------------------------------------------------------------------

    it('returns 409 for a sequence gap with last_accepted_seq', async () => {
        const config = makeConfig()
        // seq 2 without first accepting seq 1
        const ndjson = JSON.stringify({ seq: 2, event: { type: 'jump' } }) + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(409)
        const body = (await decodeJson(res)) as { last_accepted_seq: number }
        expect(body.last_accepted_seq).toBe(0)
    })

    it('reports correct last_accepted_seq in sequence gap response after prior accepts', async () => {
        const config = makeConfig()
        // Accept 1 and 2 first
        const setup =
            [JSON.stringify({ seq: 1, event: { type: 'a' } }), JSON.stringify({ seq: 2, event: { type: 'b' } })].join(
                '\n'
            ) + '\n'
        await handleIngest(
            makeContext({ body: makeStringBody(setup) }),
            fakeRedis as unknown as Redis,
            config,
            [] as CryptoKey[]
        )

        // Now send seq 4 (gap at 3)
        const gap = JSON.stringify({ seq: 4, event: { type: 'skip' } }) + '\n'
        const res = await handleIngest(
            makeContext({ body: makeStringBody(gap) }),
            fakeRedis as unknown as Redis,
            config,
            [] as CryptoKey[]
        )
        expect(res.status).toBe(409)
        const body = (await decodeJson(res)) as { last_accepted_seq: number }
        expect(body.last_accepted_seq).toBe(2)
    })

    // -----------------------------------------------------------------------
    // Already-completed → 409
    // -----------------------------------------------------------------------

    it('returns 409 when writing to an already-completed stream', async () => {
        const config = makeConfig()
        // Accept 1 and mark complete
        const ndjson1 =
            JSON.stringify({ seq: 1, event: { type: 'msg' } }) +
            '\n' +
            JSON.stringify({ type: '_posthog/stream_complete', final_seq: 1 }) +
            '\n'
        await handleIngest(
            makeContext({ body: makeStringBody(ndjson1) }),
            fakeRedis as unknown as Redis,
            config,
            [] as CryptoKey[]
        )

        // Second request: try to write seq 2
        const ndjson2 = JSON.stringify({ seq: 2, event: { type: 'late' } }) + '\n'
        const res = await handleIngest(
            makeContext({ body: makeStringBody(ndjson2) }),
            fakeRedis as unknown as Redis,
            config,
            [] as CryptoKey[]
        )
        expect(res.status).toBe(409)
        const body = (await decodeJson(res)) as { last_accepted_seq: number }
        expect(body.last_accepted_seq).toBe(1)
    })

    // -----------------------------------------------------------------------
    // Completion line: happy path
    // -----------------------------------------------------------------------

    it('accepts completion line with final_seq matching last accepted', async () => {
        const config = makeConfig()
        const ndjson =
            [
                JSON.stringify({ seq: 1, event: { type: 'msg' } }),
                JSON.stringify({ type: '_posthog/stream_complete', final_seq: 1 }),
            ].join('\n') + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)
        const body = (await decodeJson(res)) as { accepted: number; last_accepted_seq: number }
        expect(body.accepted).toBe(1)
        expect(body.last_accepted_seq).toBe(1)

        // Verify the completion sentinel was written to the Redis stream.
        const entries = await fakeRedis.xrange(getStreamKey(RUN_ID))
        const sentinelEntry = entries.find(([, fields]) => {
            const raw = fields['data']
            if (!raw) {
                return false
            }
            const parsed = JSON.parse(raw)
            return parsed.type === 'STREAM_STATUS' && parsed.status === 'complete'
        })
        expect(sentinelEntry).toBeTruthy()
    })

    it('returns 409 when completion final_seq does not match last accepted', async () => {
        const config = makeConfig()
        const ndjson =
            [
                JSON.stringify({ seq: 1, event: { type: 'msg' } }),
                // final_seq=2 but only seq 1 was accepted
                JSON.stringify({ type: '_posthog/stream_complete', final_seq: 2 }),
            ].join('\n') + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(409)
        const body = (await decodeJson(res)) as { last_accepted_seq: number }
        expect(body.last_accepted_seq).toBe(1)
    })

    it('accepts completion line with final_seq=0 when no events were sent', async () => {
        const config = makeConfig()
        const ndjson = JSON.stringify({ type: '_posthog/stream_complete', final_seq: 0 }) + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)
    })

    it('returns 400 when an event line follows the completion line', async () => {
        const config = makeConfig()
        const ndjson =
            [
                JSON.stringify({ type: '_posthog/stream_complete', final_seq: 0 }),
                JSON.stringify({ seq: 1, event: { type: 'after' } }),
            ].join('\n') + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(400)
    })

    it('returns 400 when completion line appears more than once', async () => {
        const config = makeConfig()
        const ndjson =
            [
                JSON.stringify({ type: '_posthog/stream_complete', final_seq: 0 }),
                JSON.stringify({ type: '_posthog/stream_complete', final_seq: 0 }),
            ].join('\n') + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(400)
    })

    // -----------------------------------------------------------------------
    // NDJSON parsing: chunk-boundary splitting
    // -----------------------------------------------------------------------

    it('parses NDJSON correctly when JSON object is split across two chunks', async () => {
        const config = makeConfig()
        const line = JSON.stringify({ seq: 1, event: { type: 'chunked' } })
        const full = line + '\n'
        const mid = Math.floor(full.length / 2)
        const enc = new TextEncoder()
        const chunk1 = enc.encode(full.slice(0, mid))
        const chunk2 = enc.encode(full.slice(mid))
        const body = makeChunkedBody([chunk1, chunk2])
        const ctx = makeContext({ body })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)
        const rb = (await decodeJson(res)) as { accepted: number }
        expect(rb.accepted).toBe(1)
    })

    it('parses multiple events when newlines land at chunk boundaries', async () => {
        const config = makeConfig()
        const enc = new TextEncoder()
        // Build two lines, split so the newline after line 1 is in its own chunk
        const line1 = JSON.stringify({ seq: 1, event: { type: 'a' } })
        const line2 = JSON.stringify({ seq: 2, event: { type: 'b' } })
        const chunks = [enc.encode(line1), enc.encode('\n'), enc.encode(line2), enc.encode('\n')]
        const body = makeChunkedBody(chunks)
        const ctx = makeContext({ body })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)
        const rb = (await decodeJson(res)) as { accepted: number }
        expect(rb.accepted).toBe(2)
    })

    it('handles a trailing partial line with no trailing newline', async () => {
        const config = makeConfig()
        const line = JSON.stringify({ seq: 1, event: { type: 'no-newline' } })
        // No trailing newline — the flush-on-done path handles it
        const ctx = makeContext({ body: makeStringBody(line) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)
        const rb = (await decodeJson(res)) as { accepted: number }
        expect(rb.accepted).toBe(1)
    })

    it('skips blank lines between events', async () => {
        const config = makeConfig()
        const ndjson =
            '\n' +
            JSON.stringify({ seq: 1, event: { type: 'a' } }) +
            '\n\n' +
            JSON.stringify({ seq: 2, event: { type: 'b' } }) +
            '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)
        const rb = (await decodeJson(res)) as { accepted: number }
        expect(rb.accepted).toBe(2)
    })

    // -----------------------------------------------------------------------
    // Byte / line / count limits → 413
    // -----------------------------------------------------------------------

    it('returns 413 when request body exceeds MAX_REQUEST_BYTES', async () => {
        const config = makeConfig()
        const enc = new TextEncoder()

        // Chunk 1: a valid event line that fits within MAX_REQUEST_BYTES.
        // This chunk is processed first; seq 1 is yielded and accepted.
        const line1 = JSON.stringify({ seq: 1, event: { type: 'ok' } }) + '\n'

        // Chunk 2: another event line whose addition pushes the cumulative total
        // over MAX_REQUEST_BYTES, triggering the 413 before seq 2 is accepted.
        // The check is requestSize > MAX_REQUEST_BYTES after adding the chunk,
        // so we need chunk2.length > MAX_REQUEST_BYTES - chunk1.length.
        const padding = 'x'.repeat(MAX_REQUEST_BYTES)
        const line2 = JSON.stringify({ seq: 2, event: { type: 'big', data: padding } }) + '\n'

        const body = makeChunkedBody([enc.encode(line1), enc.encode(line2)])
        const ctx = makeContext({ body })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(413)
        const rb = (await decodeJson(res)) as { last_accepted_seq: number }
        // seq 1 was accepted (chunk 1 was processed) before chunk 2 exceeded the limit
        expect(rb.last_accepted_seq).toBe(1)
    })

    it('returns 413 when a single line exceeds MAX_EVENT_LINE_BYTES', async () => {
        const config = makeConfig()
        // A single event line larger than 1_000_000 bytes.
        const hugePayload = 'y'.repeat(MAX_EVENT_LINE_BYTES + 1)
        const ndjson = JSON.stringify({ seq: 1, event: { data: hugePayload } }) + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(413)
    })

    it('returns 413 when event count exceeds MAX_EVENTS_PER_REQUEST', async () => {
        const config = makeConfig()
        const lines: string[] = []
        for (let i = 1; i <= MAX_EVENTS_PER_REQUEST + 1; i++) {
            lines.push(JSON.stringify({ seq: i, event: { type: 'e' } }))
        }
        const ndjson = lines.join('\n') + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(413)
        const body = (await decodeJson(res)) as { last_accepted_seq: number }
        // last_accepted_seq should be MAX_EVENTS_PER_REQUEST (the last accepted before the limit)
        expect(body.last_accepted_seq).toBe(MAX_EVENTS_PER_REQUEST)
    })

    // -----------------------------------------------------------------------
    // NDJSON parse errors → 400
    // -----------------------------------------------------------------------

    it('returns 400 for invalid JSON', async () => {
        const config = makeConfig()
        const ctx = makeContext({ body: makeStringBody('not json\n') })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(400)
    })

    it('returns 400 for a JSON array (not an object)', async () => {
        const config = makeConfig()
        const ctx = makeContext({ body: makeStringBody('[1,2]\n') })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(400)
    })

    it('returns 400 for seq=0 (must be >= 1)', async () => {
        const config = makeConfig()
        const ctx = makeContext({ body: makeStringBody(JSON.stringify({ seq: 0, event: {} }) + '\n') })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(400)
    })

    it('returns 400 when seq is not an integer', async () => {
        const config = makeConfig()
        const ctx = makeContext({ body: makeStringBody(JSON.stringify({ seq: 1.5, event: {} }) + '\n') })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(400)
    })

    it('returns 400 when event field is not an object', async () => {
        const config = makeConfig()
        const ctx = makeContext({ body: makeStringBody(JSON.stringify({ seq: 1, event: 'string' }) + '\n') })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(400)
    })

    it('returns 400 when completion final_seq is negative', async () => {
        const config = makeConfig()
        const ctx = makeContext({
            body: makeStringBody(JSON.stringify({ type: '_posthog/stream_complete', final_seq: -1 }) + '\n'),
        })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(400)
    })

    it('returns 400 when completion final_seq is not an integer', async () => {
        const config = makeConfig()
        const ctx = makeContext({
            body: makeStringBody(JSON.stringify({ type: '_posthog/stream_complete', final_seq: 1.5 }) + '\n'),
        })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(400)
    })

    it.each([
        {
            variant: 'ECONNRESET code',
            makeError: (): Error => Object.assign(new Error('read failed'), { code: 'ECONNRESET' }),
        },
        { variant: 'aborted message', makeError: (): Error => new Error('aborted') },
        {
            variant: 'AbortError name',
            makeError: (): Error => Object.assign(new Error('This operation was aborted'), { name: 'AbortError' }),
        },
        {
            variant: 'premature close code',
            makeError: (): Error => Object.assign(new Error('Premature close'), { code: 'ERR_STREAM_PREMATURE_CLOSE' }),
        },
    ])('treats a mid-body $variant as a client disconnect, not a server error', async ({ makeError }) => {
        const infoSpy = vi.spyOn(logger, 'info')
        const errorSpy = vi.spyOn(logger, 'error')
        const config = makeConfig()
        const enc = new TextEncoder()
        const line = enc.encode(JSON.stringify({ seq: 1, event: { type: 'before-drop' } }) + '\n')
        let sentFirstChunk = false
        const body = new ReadableStream<Uint8Array>({
            pull(controller) {
                if (!sentFirstChunk) {
                    sentFirstChunk = true
                    controller.enqueue(line)
                    return
                }
                controller.error(makeError())
            },
        })

        const res = await handleIngest(makeContext({ body }), fakeRedis as unknown as Redis, config, [] as CryptoKey[])

        expect(res.status).toBe(200)
        const responseBody = (await decodeJson(res)) as { accepted: number; last_accepted_seq: number }
        expect(responseBody.accepted).toBe(1)
        expect(responseBody.last_accepted_seq).toBe(1)

        const entries = await fakeRedis.xrange(getStreamKey(RUN_ID))
        expect(entries).toHaveLength(1)

        const log = infoSpy.mock.calls.find((c) => c[0] === 'ingest:client_disconnect')?.[1] as Record<string, unknown>
        expect(log).toMatchObject({ run: RUN_ID, accepted: 1, lastSeq: 1, classification: 'idle' })
        expect(errorSpy).not.toHaveBeenCalledWith('http.unhandled_error', expect.anything())
    })

    it.each([
        {
            state: 'agent actively streaming',
            classification: 'mid_turn',
            setup: async (): Promise<void> => {
                await redisStream.setAgentActive(true)
            },
            sendEventBeforeDrop: true,
            expectedStatus: 408,
        },
        {
            state: 'agent idle after turn-complete',
            classification: 'idle',
            setup: async (): Promise<void> => {
                await redisStream.setAgentActive(false)
            },
            sendEventBeforeDrop: true,
            expectedStatus: 200,
        },
        {
            state: 'stream already completed',
            classification: 'run_over',
            setup: async (): Promise<void> => {
                await fakeRedis.set(getCompletedKey(getStreamKey(RUN_ID)), '1')
            },
            sendEventBeforeDrop: false,
            expectedStatus: 200,
        },
    ])(
        'classifies a disconnect with $state as $classification -> $expectedStatus',
        async ({ classification, setup, sendEventBeforeDrop, expectedStatus }) => {
            const infoSpy = vi.spyOn(logger, 'info')
            const config = makeConfig()
            const enc = new TextEncoder()
            const line = enc.encode(JSON.stringify({ seq: 1, event: { type: 'before-drop' } }) + '\n')
            await setup()

            let sentFirstChunk = false
            const body = new ReadableStream<Uint8Array>({
                pull(controller) {
                    if (sendEventBeforeDrop && !sentFirstChunk) {
                        sentFirstChunk = true
                        controller.enqueue(line)
                        return
                    }
                    controller.error(Object.assign(new Error('read failed'), { code: 'ECONNRESET' }))
                },
            })

            const res = await handleIngest(
                makeContext({ body }),
                fakeRedis as unknown as Redis,
                config,
                [] as CryptoKey[]
            )

            expect(res.status).toBe(expectedStatus)
            const log = infoSpy.mock.calls.find((c) => c[0] === 'ingest:client_disconnect')?.[1] as Record<
                string,
                unknown
            >
            expect(log).toMatchObject({ classification })
        }
    )

    // -----------------------------------------------------------------------
    // Empty body
    // -----------------------------------------------------------------------

    it('returns 200 with accepted=0 for an empty body', async () => {
        const config = makeConfig()
        const ctx = makeContext({ body: makeStringBody('') })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)
        const body = (await decodeJson(res)) as { accepted: number; duplicate: number; last_accepted_seq: number }
        expect(body.accepted).toBe(0)
        expect(body.duplicate).toBe(0)
        expect(body.last_accepted_seq).toBe(0)
    })

    // -----------------------------------------------------------------------
    // Side effects: turn-complete
    // -----------------------------------------------------------------------

    it.each([
        ['omitted', {}, true],
        ['completed', { stopReason: 'end_turn' }, true],
        ['idle_resume', { stopReason: 'idle_resume' }, false],
    ])('sets agent inactive and reports completion for %s turn-complete', async (_name, params, turnCompleted) => {
        const fetchCalls: { url: string; body: unknown }[] = []
        const originalFetch = global.fetch
        global.fetch = vi.fn(async (url, init) => {
            fetchCalls.push({ url: String(url), body: JSON.parse(String((init as RequestInit).body)) })
            return new Response('', { status: 200 })
        }) as typeof fetch

        const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })
        mockValidate.mockResolvedValue(makeClaims())

        const turnCompleteEvent = {
            type: 'notification',
            notification: { method: '_posthog/turn_complete', params },
        }
        await redisStream.setAgentActive(true)
        const ndjson = JSON.stringify({ seq: 1, event: turnCompleteEvent }) + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)

        // Agent must have been set inactive.
        const agentActive = await redisStream.getAgentActive()
        expect(agentActive).toBe(false)

        // Flush any queued microtasks so the detached fire-and-forget promise runs.
        await new Promise((r) => setTimeout(r, 0))

        const callbackCall = fetchCalls.find((c) => c.url.includes('agent-proxy-callback'))
        expect(callbackCall).toBeTruthy()
        expect(callbackCall?.body).toMatchObject({
            kind: 'awaiting_input',
            agent_active: false,
            turn_completed: turnCompleted,
        })

        global.fetch = originalFetch
    })

    it('dispatches the first command callback once', async () => {
        const fetchCalls: { body: unknown }[] = []
        const originalFetch = global.fetch
        global.fetch = vi.fn(async (_url, init) => {
            fetchCalls.push({ body: JSON.parse(String((init as RequestInit).body)) })
            return Response.json({ dispatched: true }, { status: 200 })
        }) as typeof fetch

        const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })
        const commandEvent = {
            type: 'notification',
            notification: { method: '_posthog/agent_command_dispatched', params: {} },
        }
        const ndjson =
            [JSON.stringify({ seq: 1, event: commandEvent }), JSON.stringify({ seq: 2, event: commandEvent })].join(
                '\n'
            ) + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })

        const response = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        await new Promise((resolve) => setTimeout(resolve, 0))

        expect(response.status).toBe(200)
        expect(fetchCalls.map(({ body }) => (body as { kind: string }).kind)).toEqual(['command_dispatched'])
        global.fetch = originalFetch
    })

    // -----------------------------------------------------------------------
    // Side effects: session/update → agent active
    // -----------------------------------------------------------------------

    it.each([
        {
            name: 'prompt echo',
            event: {
                type: 'notification',
                notification: {
                    method: 'session/update',
                    params: { update: { sessionUpdate: 'user_message_chunk' } },
                },
            },
            expectedKinds: [],
            expectedActive: false,
        },
        {
            name: 'agent output',
            event: {
                type: 'notification',
                notification: {
                    method: 'session/update',
                    params: { update: { sessionUpdate: 'agent_message_chunk' } },
                },
            },
            expectedKinds: ['agent_activity', 'heartbeat'],
            expectedActive: true,
        },
        {
            name: 'restored plan',
            event: {
                type: 'notification',
                notification: {
                    method: 'session/update',
                    params: { update: { sessionUpdate: 'plan' } },
                },
            },
            expectedKinds: [],
            expectedActive: false,
        },
        {
            name: 'resumed session commands',
            event: {
                type: 'notification',
                notification: {
                    method: 'session/update',
                    params: { update: { sessionUpdate: 'available_commands_update' } },
                },
            },
            expectedKinds: [],
            expectedActive: false,
        },
        {
            name: 'idle resume turn complete',
            event: {
                type: 'notification',
                notification: { method: '_posthog/turn_complete', params: { stopReason: 'idle_resume' } },
            },
            expectedKinds: ['awaiting_input'],
            expectedActive: false,
        },
        {
            name: 'command dispatch',
            event: {
                type: 'notification',
                notification: { method: '_posthog/agent_command_dispatched', params: {} },
            },
            expectedKinds: ['command_dispatched'],
            expectedActive: false,
        },
    ])('dispatches side effects for $name', async ({ event, expectedKinds, expectedActive }) => {
        const fetchCalls: { url: string; body: unknown }[] = []
        const originalFetch = global.fetch
        global.fetch = vi.fn(async (url, init) => {
            fetchCalls.push({ url: String(url), body: JSON.parse(String((init as RequestInit).body)) })
            return new Response('', { status: 200 })
        }) as typeof fetch

        const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })
        mockValidate.mockResolvedValue(makeClaims())

        const ndjson = JSON.stringify({ seq: 1, event }) + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)

        const agentActive = await redisStream.getAgentActive()
        expect(agentActive).toBe(expectedActive)

        await new Promise((r) => setTimeout(r, 0))

        const callbackKinds = fetchCalls
            .filter((c) => c.url.includes('agent-proxy-callback'))
            .map((c) => (c.body as { kind: string }).kind)
        expect(callbackKinds).toEqual(expectedKinds)

        global.fetch = originalFetch
    })

    // -----------------------------------------------------------------------
    // Side effects: heartbeat throttle
    // -----------------------------------------------------------------------

    it('does not fire second heartbeat callback when heartbeat key is already claimed', async () => {
        const fetchCalls: { url: string }[] = []
        const originalFetch = global.fetch
        global.fetch = vi.fn(async (url) => {
            fetchCalls.push({ url: String(url) })
            return new Response('', { status: 200 })
        }) as typeof fetch

        const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })

        // Pre-seed: set agent active so heartbeat path runs
        await redisStream.setAgentActive(true)
        // Claim the heartbeat key so the throttle fires (first heartbeat claimed)
        await redisStream.claimAgentActiveHeartbeat(HEARTBEAT_THROTTLE_SECONDS)

        const sessionUpdateEvent = {
            type: 'notification',
            notification: { method: 'session/update' },
        }
        const ndjson =
            [
                JSON.stringify({ seq: 1, event: sessionUpdateEvent }),
                JSON.stringify({ seq: 2, event: { type: 'other' } }),
            ].join('\n') + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)

        await new Promise((r) => setTimeout(r, 0))

        // The heartbeat key was already claimed; no callback should fire.
        const callbackCalls = fetchCalls.filter((c) => c.url.includes('agent-proxy-callback'))
        // session/update on seq 1 tries to claim → already claimed → no callback
        // seq 2 is a non-session-update event; agentActive is true; tries to claim → already claimed → no callback
        expect(callbackCalls.length).toBe(0)

        global.fetch = originalFetch
    })

    // -----------------------------------------------------------------------
    // Side effects: best-effort (callback failure does not fail ingest)
    // -----------------------------------------------------------------------

    it.each([
        [
            'throws',
            async () => {
                throw new Error('network failure')
            },
        ],
        ['answers 200 with a body that is not JSON', async () => new Response('', { status: 200 })],
    ])('releases a command claim when the Django callback %s', async (_label, respond) => {
        const originalFetch = global.fetch
        global.fetch = vi.fn(respond) as typeof fetch

        const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })

        const commandEvent = {
            type: 'notification',
            notification: { method: '_posthog/agent_command_dispatched', params: {} },
        }
        const ndjson = JSON.stringify({ seq: 1, event: commandEvent }) + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        // Callback failure must NOT affect ingest response
        expect(res.status).toBe(200)

        // Let the detached promise settle without crashing the test
        await new Promise((r) => setTimeout(r, 10))

        expect(await redisStream.claimFirstAgentCommand()).toBe(true)
        global.fetch = originalFetch
    })

    it('keeps the command claim when the Django callback answers 400', async () => {
        const originalFetch = global.fetch
        global.fetch = vi.fn(async () => new Response('', { status: 400 })) as typeof fetch

        const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })

        const commandEvent = {
            type: 'notification',
            notification: { method: '_posthog/agent_command_dispatched', params: {} },
        }
        const ndjson = JSON.stringify({ seq: 1, event: commandEvent }) + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)

        await new Promise((r) => setTimeout(r, 10))

        expect(await redisStream.claimFirstAgentCommand()).toBe(false)
        global.fetch = originalFetch
    })

    it('still returns 200 when the Django callback returns a non-2xx status', async () => {
        const originalFetch = global.fetch
        global.fetch = vi.fn(async () => new Response('', { status: 500 })) as typeof fetch

        const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })

        const turnCompleteEvent = {
            type: 'notification',
            notification: { method: '_posthog/turn_complete' },
        }
        const ndjson = JSON.stringify({ seq: 1, event: turnCompleteEvent }) + '\n'
        const ctx = makeContext({ body: makeStringBody(ndjson) })
        const res = await handleIngest(ctx, fakeRedis as unknown as Redis, config, [] as CryptoKey[])
        expect(res.status).toBe(200)

        await new Promise((r) => setTimeout(r, 10))

        global.fetch = originalFetch
    })

    // -----------------------------------------------------------------------
    // heartbeatWorkflowIfNeeded (side-effects.ts) unit tests
    // -----------------------------------------------------------------------

    describe('heartbeatWorkflowIfNeeded', () => {
        it.each([
            [
                'an end_turn turn-complete event',
                {
                    type: 'notification',
                    notification: { method: '_posthog/turn_complete', params: { stopReason: 'end_turn' } },
                },
                true,
            ],
            [
                'a turn-complete event without end_turn',
                { type: 'notification', notification: { method: '_posthog/turn_complete' } },
                false,
            ],
        ])('sets agent inactive and fires awaiting_input for %s', async (_label, event, expectedSucceeded) => {
            const fired: { kind: string; turn_succeeded?: boolean }[] = []
            const originalFetch = global.fetch
            global.fetch = vi.fn(async (_, init) => {
                fired.push(JSON.parse(String((init as RequestInit).body)))
                return new Response('', { status: 200 })
            }) as typeof fetch

            const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })
            await heartbeatWorkflowIfNeeded(redisStream, RUN_ID, event, TASK_ID, TEAM_ID, 'tok', config)

            expect(await redisStream.getAgentActive()).toBe(false)
            await new Promise((r) => setTimeout(r, 0))
            expect(fired.find((f) => f.kind === 'awaiting_input')?.turn_succeeded).toBe(expectedSucceeded)

            global.fetch = originalFetch
        })

        it.each([
            ['a normal pi turn_completed', { type: 'pi_event', event: { type: 'turn_completed' } }, 'awaiting_input'],
            [
                'a pi turn_completed with stopReason "error"',
                { type: 'pi_event', event: { type: 'turn_completed', stopReason: 'error' } },
                'turn_failed',
            ],
        ])('fires %s as %s', async (_label, event, expectedKind) => {
            const fired: { kind: string }[] = []
            const originalFetch = global.fetch
            global.fetch = vi.fn(async (_, init) => {
                fired.push(JSON.parse(String((init as RequestInit).body)))
                return new Response('', { status: 200 })
            }) as typeof fetch

            const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })
            await heartbeatWorkflowIfNeeded(redisStream, RUN_ID, event, TASK_ID, TEAM_ID, 'tok', config)

            expect(await redisStream.getAgentActive()).toBe(false)
            await new Promise((r) => setTimeout(r, 0))
            expect(fired.some((f) => f.kind === expectedKind)).toBe(true)

            global.fetch = originalFetch
        })

        const turnComplete = { type: 'notification', notification: { method: '_posthog/turn_complete' } }
        const piTurnError = { type: 'pi_event', event: { type: 'turn_completed', stopReason: 'error' } }
        const sessionUpdate = {
            type: 'notification',
            notification: { method: 'session/update', params: { update: { sessionUpdate: 'agent_message_chunk' } } },
        }
        const permissionRequest = {
            type: 'notification',
            notification: {
                method: '_posthog/permission_request',
                params: { requestId: 'perm-1', toolCall: {}, options: [{ optionId: 'allow' }] },
            },
        }
        const networkFailure = Object.assign(new TypeError('fetch failed'), { cause: { code: 'ECONNRESET' } })

        it.each([
            ['awaiting_input', turnComplete, 'network failure', networkFailure, 2],
            ['awaiting_input', turnComplete, '503', new Response('', { status: 503 }), 2],
            ['awaiting_input', turnComplete, '400', new Response('', { status: 400 }), 1],
            ['turn_failed', piTurnError, 'network failure', networkFailure, 2],
            ['turn_failed', piTurnError, '503', new Response('', { status: 503 }), 2],
            ['permission_requested', permissionRequest, '503', new Response('', { status: 503 }), 2],
            ['heartbeat', sessionUpdate, 'network failure', networkFailure, 1],
            ['heartbeat', sessionUpdate, '503', new Response('', { status: 503 }), 1],
        ])(
            'a %s callback whose first attempt ends in a %s is sent %i time(s) in total',
            async (_kind, event, _label, firstOutcome, expectedCalls) => {
                vi.useFakeTimers()
                const originalFetch = global.fetch
                const fetchMock = vi
                    .fn()
                    .mockImplementationOnce(() =>
                        firstOutcome instanceof Error ? Promise.reject(firstOutcome) : Promise.resolve(firstOutcome)
                    )
                    .mockResolvedValue(new Response('{"dispatched": true}', { status: 200 }))
                global.fetch = fetchMock as typeof fetch

                await redisStream.claimFirstAgentActivity()
                const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })
                await heartbeatWorkflowIfNeeded(redisStream, RUN_ID, event, TASK_ID, TEAM_ID, 'tok', config)
                notifyPermissionRequestIfNeeded(RUN_ID, 1, event, TASK_ID, TEAM_ID, 'tok', config)
                await vi.advanceTimersByTimeAsync(2000)

                expect(fetchMock).toHaveBeenCalledTimes(expectedCalls)
                expect((fetchMock.mock.calls[0]?.[1] as RequestInit).signal).toBeInstanceOf(AbortSignal)

                global.fetch = originalFetch
                vi.useRealTimers()
            }
        )

        it.each([
            {
                type: 'notification',
                notification: {
                    method: 'session/update',
                    params: { update: { sessionUpdate: 'agent_message_chunk' } },
                },
            },
            ...['assistant_message_chunk', 'assistant_thought_chunk', 'tool_call_started', 'tool_call_updated'].map(
                (type) => ({ type: 'pi_event', event: { type } })
            ),
        ])('reports new activity across consecutive turns for %j', async (event) => {
            const fired: { kind: string; activity_started?: boolean }[] = []
            const originalFetch = global.fetch
            global.fetch = vi.fn(async (_, init) => {
                fired.push(JSON.parse(String((init as RequestInit).body)))
                return new Response('', { status: 200 })
            }) as typeof fetch

            const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })
            await heartbeatWorkflowIfNeeded(redisStream, RUN_ID, event, TASK_ID, TEAM_ID, 'tok', config)
            await heartbeatWorkflowIfNeeded(redisStream, RUN_ID, event, TASK_ID, TEAM_ID, 'tok', config)
            expect(await redisStream.getAgentActive()).toBe(true)
            await heartbeatWorkflowIfNeeded(
                redisStream,
                RUN_ID,
                { type: 'pi_event', event: { type: 'turn_completed' } },
                TASK_ID,
                TEAM_ID,
                'tok',
                config
            )
            expect(await redisStream.getAgentActive()).toBe(false)
            await heartbeatWorkflowIfNeeded(redisStream, RUN_ID, event, TASK_ID, TEAM_ID, 'tok', config)
            expect(await redisStream.getAgentActive()).toBe(true)
            await new Promise((r) => setTimeout(r, 0))
            expect(fired.filter((f) => f.kind === 'heartbeat').map((f) => f.activity_started)).toEqual([true, true])

            global.fetch = originalFetch
        })

        it('does not fire heartbeat for a plain event when agent is not active', async () => {
            const fired: unknown[] = []
            const originalFetch = global.fetch
            global.fetch = vi.fn(async (_, init) => {
                fired.push(JSON.parse(String((init as RequestInit).body)))
                return new Response('', { status: 200 })
            }) as typeof fetch

            const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })
            const event = { type: 'notification', notification: { method: 'other' } }
            await heartbeatWorkflowIfNeeded(redisStream, RUN_ID, event, TASK_ID, TEAM_ID, 'tok', config)

            await new Promise((r) => setTimeout(r, 0))
            expect(fired).toHaveLength(0)

            global.fetch = originalFetch
        })

        it('fires heartbeat for a plain event when agent is already active', async () => {
            const fired: { kind: string }[] = []
            const originalFetch = global.fetch
            global.fetch = vi.fn(async (_, init) => {
                fired.push(JSON.parse(String((init as RequestInit).body)))
                return new Response('', { status: 200 })
            }) as typeof fetch

            await redisStream.setAgentActive(true)
            const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })
            const event = { type: 'notification', notification: { method: 'other' } }
            await heartbeatWorkflowIfNeeded(redisStream, RUN_ID, event, TASK_ID, TEAM_ID, 'tok', config)

            await new Promise((r) => setTimeout(r, 0))
            expect(fired.some((f) => f.kind === 'heartbeat')).toBe(true)

            global.fetch = originalFetch
        })

        it('skips the heartbeat callback when the throttle key is already claimed', async () => {
            const fired: unknown[] = []
            const originalFetch = global.fetch
            global.fetch = vi.fn(async (_, init) => {
                fired.push(JSON.parse(String((init as RequestInit).body)))
                return new Response('', { status: 200 })
            }) as typeof fetch

            await redisStream.setAgentActive(true)
            // Pre-claim the heartbeat slot
            await redisStream.claimAgentActiveHeartbeat(HEARTBEAT_THROTTLE_SECONDS)

            const config = makeConfig({ djangoCallbackBaseUrl: 'http://django' })
            const event = { type: 'notification', notification: { method: 'other' } }
            await heartbeatWorkflowIfNeeded(redisStream, RUN_ID, event, TASK_ID, TEAM_ID, 'tok', config)

            await new Promise((r) => setTimeout(r, 0))
            expect(fired).toHaveLength(0)

            global.fetch = originalFetch
        })

        it('does not fire any callback when djangoCallbackBaseUrl is empty', async () => {
            const fetchSpy = vi.spyOn(global, 'fetch')
            const config = makeConfig({ djangoCallbackBaseUrl: '' })
            const event = { type: 'notification', notification: { method: '_posthog/turn_complete' } }
            await heartbeatWorkflowIfNeeded(redisStream, RUN_ID, event, TASK_ID, TEAM_ID, 'tok', config)
            await new Promise((r) => setTimeout(r, 0))
            expect(fetchSpy).not.toHaveBeenCalled()
            fetchSpy.mockRestore()
        })
    })

    // -----------------------------------------------------------------------
    // Body-arrival timing diagnostic
    //
    // The ingest log carries chunks/bodyBytes/firstChunkMs/lastChunkMs/chunkSpanMs
    // so operators can tell a live upload (chunks spread over the request) from a
    // body buffered upstream and delivered in one burst at request close. If those
    // numbers are wrong the diagnostic misleads that investigation, so lock in that
    // they reflect the actual body read.
    // -----------------------------------------------------------------------

    describe('body-arrival timing', () => {
        it('reports chunk count, byte total, and a consistent span on the ingest log', async () => {
            const infoSpy = vi.spyOn(logger, 'info')
            const enc = new TextEncoder()
            const chunk1 = enc.encode(JSON.stringify({ seq: 1, event: { type: 'a' } }) + '\n')
            const chunk2 = enc.encode(JSON.stringify({ seq: 2, event: { type: 'b' } }) + '\n')

            const ctx = makeContext({ body: makeChunkedBody([chunk1, chunk2]) })
            const res = await handleIngest(ctx, fakeRedis as unknown as Redis, makeConfig(), [] as CryptoKey[])
            expect(res.status).toBe(200)

            const log = infoSpy.mock.calls.find((c) => c[0] === 'ingest')?.[1] as Record<string, number>
            expect(log).toBeTruthy()
            expect(log.chunks).toBe(2)
            expect(log.bodyBytes).toBe(chunk1.length + chunk2.length)
            const firstChunkMs = log.firstChunkMs as number
            const lastChunkMs = log.lastChunkMs as number
            expect(typeof firstChunkMs).toBe('number')
            expect(lastChunkMs).toBeGreaterThanOrEqual(firstChunkMs)
            expect(log.chunkSpanMs as number).toBe(lastChunkMs - firstChunkMs)
        })

        it('reports zero chunks and null timing for an empty body', async () => {
            const infoSpy = vi.spyOn(logger, 'info')

            const ctx = makeContext({ body: makeStringBody('') })
            const res = await handleIngest(ctx, fakeRedis as unknown as Redis, makeConfig(), [] as CryptoKey[])
            expect(res.status).toBe(200)

            const log = infoSpy.mock.calls.find((c) => c[0] === 'ingest')?.[1] as Record<string, unknown>
            expect(log).toBeTruthy()
            expect(log.chunks).toBe(0)
            expect(log.bodyBytes).toBe(0)
            expect(log.firstChunkMs).toBeNull()
            expect(log.lastChunkMs).toBeNull()
            expect(log.chunkSpanMs).toBeNull()
        })
    })
})