:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
// NDJSON event ingest handler for POST /v1/runs/:run/ingest
//
// Wire protocol stays byte-identical to event_ingest.py — both the Python and
// Node services read/write the same Redis stream during the cutover window.
//
// HTTP status codes:
// 200 success, or post-turn client disconnect (sandbox teardown): {accepted, duplicate, last_accepted_seq}
// 400 bad NDJSON: invalid JSON, non-object, wrong field types, ordering
// 401 missing / invalid JWT
// 403 JWT claims don't match URL params
// 405 wrong HTTP method
// 408 client disconnected mid-turn: {error, last_accepted_seq}
// 409 SequenceGap / AlreadyCompleted / CompletionSequenceMismatch: {error, last_accepted_seq}
// 413 payload too large: {error, last_accepted_seq}
import type { CryptoKey } from 'crypto'
import type { Context } from 'hono'
import type { Redis } from 'ioredis'
import type { Config } from '../lib/config.js'
import {
MAX_EVENT_LINE_BYTES,
MAX_REQUEST_BYTES,
MAX_EVENTS_PER_REQUEST,
STREAM_COMPLETE_CONTROL_TYPE,
STREAM_INITIAL_GENERATION,
} from '../lib/constants.js'
import { validateSandboxEventIngestToken } from '../lib/jwt.js'
import { logger } from '../lib/logging.js'
import { TaskRunRedisStream, getStreamKey } from '../lib/redis-stream.js'
import {
captureBudgetSteerIfNeeded,
captureProcessKilledIfNeeded,
heartbeatWorkflowIfNeeded,
notifyPermissionRequestIfNeeded,
} from '../lib/side-effects.js'
import {
ClientDisconnected,
EventIngestBadRequest,
EventIngestPayloadTooLarge,
TaskRunStreamSequenceGap,
TaskRunStreamAlreadyCompleted,
TaskRunStreamGenerationMismatch,
TaskRunStreamCompletionSequenceMismatch,
type DisconnectClassification,
type IngestEventLine,
type IngestCompleteLine,
type IngestLine,
type EventIngestResult,
type SandboxEventIngestTokenPayload,
} from '../lib/types.js'
import { observeIngestClientDisconnect, observeStreamIngestEvents, observeStreamWriteSkipped } from './metrics.js'
// Diagnostic (temporary): records when request-body chunks arrive at the Node
// process, to tell a live upload (chunks spread across the request lifetime)
// from a body buffered upstream and delivered in one burst at request close.
// The sandbox holds one long-lived chunked POST open for the whole turn; if
// chunks only land when that request closes, the stall is upstream of this
// process. Remove once the buffering hop is pinned down.
interface IngestBodyTiming {
startedAt: number
firstChunkAt: number | null
lastChunkAt: number | null
chunks: number
bytes: number
}
// ---------------------------------------------------------------------------
// Route handler (exported; wired in app.ts)
// ---------------------------------------------------------------------------
export async function handleIngest(
c: Context,
redis: Redis,
config: Config,
publicKeys: CryptoKey[]
): Promise<Response> {
// Method guard (Hono enforces the route method, but keep an explicit check
// so the handler is safe when registered on a catch-all route too).
if (c.req.method !== 'POST') {
return c.json({ error: 'Method not allowed' }, 405)
}
// Authorization: Bearer <token> only (no ?token= fallback on ingest).
const token = extractBearerToken(c)
if (token === null) {
return c.json({ error: 'Missing authorization bearer token' }, 401)
}
// JWT validation.
let claims: SandboxEventIngestTokenPayload
try {
claims = await validateSandboxEventIngestToken(token, publicKeys)
} catch (err: unknown) {
const code = err instanceof Error ? err.constructor.name : 'UnknownError'
return c.json({ error: 'Invalid event ingest token', code }, 401)
}
// The run-scoped JWT is the authority (Django minted it for this run/task/team);
// the run path segment only has to agree with the token's run claim. team/task
// come from the verified token, not the URL.
const { run } = c.req.param() as { run: string }
if (claims.runId !== run) {
return c.json({ error: 'Token does not match run' }, 403)
}
// No run-existence check. The run-scoped sandbox_event_ingest JWT is the
// authorization — Django minted it for this exact run/task/team. Keeping the
// ingest plane free of a per-request Django round-trip is the whole point of
// the standalone proxy; events for a run deleted mid-stream land in the TTL-
// and MAXLEN-bounded Redis stream and are never read.
// NDJSON body parsing + Redis writes.
const streamKey = getStreamKey(claims.runId)
const redisStream = new TaskRunRedisStream(streamKey, redis, {
presenceGated: claims.presenceGated,
thinTail: claims.thinTail,
expectedGeneration: claims.streamGeneration ?? STREAM_INITIAL_GENERATION,
})
const bodyTiming: IngestBodyTiming = {
startedAt: Date.now(),
firstChunkAt: null,
lastChunkAt: null,
chunks: 0,
bytes: 0,
}
let result: EventIngestResult
try {
result = await ingestEventLines(redisStream, claims, token, config, c.req.raw, bodyTiming)
} catch (err: unknown) {
if (err instanceof ClientDisconnected) {
const classification = await classifyDisconnect(redisStream)
observeIngestClientDisconnect(classification)
observeStreamIngestEvents({ accepted: err.accepted, duplicate: err.duplicate })
logger.info('ingest:client_disconnect', {
run: claims.runId,
classification,
accepted: err.accepted,
duplicate: err.duplicate,
lastSeq: err.lastAcceptedSeq,
chunks: bodyTiming.chunks,
bodyBytes: bodyTiming.bytes,
})
if (classification === 'mid_turn') {
return c.json({ error: 'Client disconnected', last_accepted_seq: err.lastAcceptedSeq }, 408)
}
return c.json(
{ accepted: err.accepted, duplicate: err.duplicate, last_accepted_seq: err.lastAcceptedSeq },
200
)
}
if (err instanceof EventIngestBadRequest) {
return c.json({ error: err.message }, 400)
}
if (err instanceof EventIngestPayloadTooLarge) {
return c.json({ error: err.message, last_accepted_seq: err.lastAcceptedSeq }, 413)
}
if (err instanceof TaskRunStreamGenerationMismatch) {
return c.json({ error: err.message, code: 'stream_generation_mismatch' }, 403)
}
if (err instanceof TaskRunStreamSequenceGap) {
return c.json({ error: err.message, last_accepted_seq: err.lastAcceptedSeq }, 409)
}
if (err instanceof TaskRunStreamAlreadyCompleted) {
return c.json({ error: err.message, last_accepted_seq: err.lastAcceptedSeq }, 409)
}
if (err instanceof TaskRunStreamCompletionSequenceMismatch) {
return c.json({ error: err.message, last_accepted_seq: err.lastAcceptedSeq }, 409)
}
throw err
}
observeStreamIngestEvents({ accepted: result.accepted, duplicate: result.duplicate })
logger.info('ingest', {
run: claims.runId,
accepted: result.accepted,
duplicate: result.duplicate,
lastSeq: result.last_accepted_seq,
// Body-arrival timing. Live upload: firstChunkMs small, chunkSpanMs
// large. Buffered upstream: firstChunkMs ~= lastChunkMs, chunkSpanMs ~0
// (every chunk lands together when the request closes).
chunks: bodyTiming.chunks,
bodyBytes: bodyTiming.bytes,
firstChunkMs: bodyTiming.firstChunkAt === null ? null : bodyTiming.firstChunkAt - bodyTiming.startedAt,
lastChunkMs: bodyTiming.lastChunkAt === null ? null : bodyTiming.lastChunkAt - bodyTiming.startedAt,
chunkSpanMs:
bodyTiming.firstChunkAt === null || bodyTiming.lastChunkAt === null
? null
: bodyTiming.lastChunkAt - bodyTiming.firstChunkAt,
})
return c.json(
{
accepted: result.accepted,
duplicate: result.duplicate,
last_accepted_seq: result.last_accepted_seq,
},
200
)
}
// Post-turn disconnects are expected sandbox teardown; fail toward 'mid_turn' so real cuts stay visible.
async function classifyDisconnect(redisStream: TaskRunRedisStream): Promise<DisconnectClassification> {
try {
if (await redisStream.isComplete()) {
return 'run_over'
}
return (await redisStream.getAgentActive()) ? 'mid_turn' : 'idle'
} catch {
return 'mid_turn'
}
}
// ---------------------------------------------------------------------------
// NDJSON body parsing + Redis write loop
// ---------------------------------------------------------------------------
async function ingestEventLines(
redisStream: TaskRunRedisStream,
claims: SandboxEventIngestTokenPayload,
originalToken: string,
config: Config,
rawRequest: Request,
bodyTiming: IngestBodyTiming
): Promise<EventIngestResult> {
const result: EventIngestResult = {
accepted: 0,
duplicate: 0,
last_accepted_seq: await redisStream.getLastSequence(),
}
let eventCount = 0
let completionLineFinalSeq: number | null = null
try {
for await (const line of iterRequestLines(rawRequest, claims.runId, bodyTiming)) {
const parsed = parseIngestLine(line)
if (parsed.kind === 'complete') {
if (completionLineFinalSeq !== null) {
throw new EventIngestBadRequest('Completion line must be the final event stream line')
}
completionLineFinalSeq = (parsed as IngestCompleteLine).finalSeq
continue
}
if (completionLineFinalSeq !== null) {
throw new EventIngestBadRequest('Completion line must be the final event stream line')
}
const eventLine = parsed as IngestEventLine
eventCount++
if (eventCount > MAX_EVENTS_PER_REQUEST) {
throw new EventIngestPayloadTooLarge('Too many events in request', result.last_accepted_seq)
}
const { seq, event } = eventLine
const write = await redisStream.writeEventWithSequence(event, seq)
captureBudgetSteerIfNeeded(claims.runId, seq, event, claims.taskId, claims.teamId, originalToken, config)
if (!write.accepted) {
// Duplicate: advance last_accepted_seq to whatever Redis has.
result.duplicate++
result.last_accepted_seq = Math.max(result.last_accepted_seq, await redisStream.getLastSequence())
continue
}
if (write.streamId === null) {
observeStreamWriteSkipped('ingest', claims.originProduct)
}
result.accepted++
result.last_accepted_seq = seq
captureProcessKilledIfNeeded(claims.runId, seq, event, claims.taskId, claims.teamId, originalToken, config)
notifyPermissionRequestIfNeeded(
claims.runId,
seq,
event,
claims.taskId,
claims.teamId,
originalToken,
config
)
await heartbeatWorkflowIfNeeded(
redisStream,
claims.runId,
event,
claims.taskId,
claims.teamId,
originalToken,
config
)
}
} catch (err: unknown) {
if (err instanceof ClientDisconnected) {
err.accepted = result.accepted
err.duplicate = result.duplicate
err.lastAcceptedSeq = result.last_accepted_seq
}
// Re-throw EventIngestPayloadTooLarge with the current last_accepted_seq
// if the error didn't carry one (mirrors Python's except re-raise).
if (err instanceof EventIngestPayloadTooLarge && result.last_accepted_seq > 0 && err.lastAcceptedSeq === 0) {
err.lastAcceptedSeq = result.last_accepted_seq
}
throw err
}
if (completionLineFinalSeq !== null) {
await redisStream.markCompleteAfterSequence(completionLineFinalSeq)
}
return result
}
// ---------------------------------------------------------------------------
// NDJSON line parser (mirrors _parse_ingest_line)
// ---------------------------------------------------------------------------
function parseIngestLine(line: string): IngestLine {
let payload: unknown
try {
payload = JSON.parse(line)
} catch {
throw new EventIngestBadRequest('Invalid JSON line')
}
if (typeof payload !== 'object' || payload === null || Array.isArray(payload)) {
throw new EventIngestBadRequest('Each event line must be a JSON object')
}
const obj = payload as Record<string, unknown>
// Completion line detection (must come before event line parsing).
if (obj['type'] === STREAM_COMPLETE_CONTROL_TYPE) {
const finalSeq = obj['final_seq']
// type(final_sequence) is not int or final_sequence < 0 (Python exact check).
// typeof guards booleans (typeof true === 'boolean', not 'number').
if (typeof finalSeq !== 'number' || !Number.isInteger(finalSeq) || finalSeq < 0) {
throw new EventIngestBadRequest('Completion final sequence must be a non-negative integer')
}
return { kind: 'complete', finalSeq }
}
// Event line.
const seq = obj['seq']
const event = obj['event']
// typeof guards booleans (typeof true === 'boolean', not 'number').
if (typeof seq !== 'number' || !Number.isInteger(seq) || seq < 1) {
throw new EventIngestBadRequest('Event sequence must be a positive integer')
}
if (typeof event !== 'object' || event === null || Array.isArray(event)) {
throw new EventIngestBadRequest('Event payload must be an object')
}
return { kind: 'event', seq, event: event as Record<string, unknown> }
}
// ---------------------------------------------------------------------------
// Streaming NDJSON body reader (mirrors _iter_request_lines)
// ---------------------------------------------------------------------------
async function* iterRequestLines(request: Request, run: string, timing: IngestBodyTiming): AsyncGenerator<string> {
const body = request.body
if (body === null) {
return
}
const reader = body.getReader()
let buffer = new Uint8Array(0)
let requestSize = 0
const newline = 0x0a // '\n'
try {
while (true) {
const { done, value } = await reader.read().catch((err: unknown) => {
throw isClientAbortError(err) ? new ClientDisconnected() : err
})
if (value !== undefined && value.length > 0) {
const chunkAt = Date.now()
if (timing.firstChunkAt === null) {
timing.firstChunkAt = chunkAt
}
timing.lastChunkAt = chunkAt
timing.chunks += 1
timing.bytes += value.length
logger.debug('ingest:body_chunk', {
run,
chunk: timing.chunks,
bytes: value.length,
sinceStartMs: chunkAt - timing.startedAt,
})
requestSize += value.length
if (requestSize > MAX_REQUEST_BYTES) {
throw new EventIngestPayloadTooLarge('Event ingest request is too large')
}
// Append chunk to buffer.
const combined = new Uint8Array(buffer.length + value.length)
combined.set(buffer)
combined.set(value, buffer.length)
buffer = combined
// Guard: if the buffer is larger than one max line with no newline,
// the line itself is already too large.
if (buffer.length > MAX_EVENT_LINE_BYTES && !buffer.includes(newline)) {
throw new EventIngestPayloadTooLarge('Event line is too large')
}
// Yield all complete lines (terminated by '\n').
let start = 0
for (let i = 0; i < buffer.length; i++) {
if (buffer[i] === newline) {
const lineBytes = buffer.subarray(start, i)
start = i + 1
if (lineBytes.length > MAX_EVENT_LINE_BYTES) {
throw new EventIngestPayloadTooLarge('Event line is too large')
}
// Trim whitespace (strip).
const trimmed = trimBytes(lineBytes)
if (trimmed.length === 0) {
continue
}
yield decodeUtf8(trimmed)
}
}
// Keep the partial trailing line in buffer.
buffer = buffer.subarray(start)
}
if (done) {
// Flush trailing partial line.
const trimmed = trimBytes(buffer)
if (trimmed.length > 0) {
if (trimmed.length > MAX_EVENT_LINE_BYTES) {
throw new EventIngestPayloadTooLarge('Event line is too large')
}
yield decodeUtf8(trimmed)
}
return
}
}
} finally {
reader.releaseLock()
}
}
function isClientAbortError(err: unknown): boolean {
if (!(err instanceof Error)) {
return false
}
if (err.name === 'AbortError') {
return true
}
const code = (err as NodeJS.ErrnoException).code
// 'aborted' is undici's body-read abort message; it carries no error code, unlike the others.
return code === 'ECONNRESET' || code === 'ERR_STREAM_PREMATURE_CLOSE' || err.message === 'aborted'
}
// Decode bytes as UTF-8, mapping decode errors to EventIngestBadRequest.
function decodeUtf8(bytes: Uint8Array): string {
try {
// TextDecoder with fatal:true throws on invalid byte sequences.
return new TextDecoder('utf-8', { fatal: true }).decode(bytes)
} catch {
throw new EventIngestBadRequest('Invalid UTF-8 in event stream')
}
}
// Trim leading and trailing ASCII whitespace bytes (space, tab, CR, LF).
function trimBytes(bytes: Uint8Array): Uint8Array {
const isWs = (b: number): boolean => b === 0x20 || b === 0x09 || b === 0x0d || b === 0x0a
let start = 0
let end = bytes.length
while (start < end && isWs(bytes[start] ?? 0)) {
start++
}
while (end > start && isWs(bytes[end - 1] ?? 0)) {
end--
}
return bytes.subarray(start, end)
}
// ---------------------------------------------------------------------------
// Token extraction
// ---------------------------------------------------------------------------
function extractBearerToken(c: Context): string | null {
const authorization = c.req.header('Authorization') ?? c.req.header('authorization')
if (!authorization) {
return null
}
const prefix = 'Bearer '
if (!authorization.startsWith(prefix)) {
return null
}
const token = authorization.slice(prefix.length).trim()
return token || null
}