:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
// Side-effect dispatch for the ingest handler.
//
// After each accepted ingest event the handler calls heartbeatWorkflowIfNeeded.
// Redis state mutations (setAgentActive, claimAgentActiveHeartbeat) are awaited
// because they gate the callback decision. The Django HTTP callback is
// fire-and-forget: the fetch result is consumed in a detached promise with
// error logging so failures never propagate into the ingest response.
//
// The Node service stays a pure streaming plane — Temporal signals and Celery
// push notifications go through a single internal Django endpoint. Django
// decides whether a run is interactive; Node always calls the callback and lets
// Django branch on mode.
import type { Config } from './config.js'
import { HEARTBEAT_THROTTLE_SECONDS } from './constants.js'
import { logger } from './logging.js'
import type { TaskRunRedisStream } from './redis-stream.js'
import type { SideEffectKind } from './types.js'
// ACP event field values (byte-identical to ee/hogai/sandbox/types.py)
const ACP_NOTIFICATION_TYPE = 'notification'
const TURN_COMPLETE_METHOD = '_posthog/turn_complete'
const STOP_REASON_END_TURN = 'end_turn'
const IDLE_RESUME_STOP_REASON = 'idle_resume'
const ACP_METHOD_SESSION_UPDATE = 'session/update'
const AGENT_COMMAND_DISPATCHED_METHOD = '_posthog/agent_command_dispatched'
const ACP_GENERATION_UPDATES = new Set([
'agent_message',
'agent_message_chunk',
'agent_thought_chunk',
'tool_call',
'tool_call_update',
])
const PI_GENERATION_EVENTS = new Set([
'assistant_message_chunk',
'assistant_thought_chunk',
'tool_call_started',
'tool_call_updated',
])
// pi agent event shapes (byte-identical to ee/hogai/sandbox/types.py)
const PI_EVENT_TYPE = 'pi_event'
const PI_TURN_COMPLETED_TYPE = 'turn_completed'
const PI_STOP_REASON_ERROR = 'error'
function asPiTurnCompletedEvent(event: Record<string, unknown>): Record<string, unknown> | null {
if (event['type'] !== PI_EVENT_TYPE) {
return null
}
const piEvent = event['event']
if (typeof piEvent !== 'object' || piEvent === null) {
return null
}
const inner = piEvent as Record<string, unknown>
return inner['type'] === PI_TURN_COMPLETED_TYPE ? inner : null
}
// isTurnComplete mirrors ee/hogai/sandbox/types.py:is_turn_complete exactly.
// Matches the raw ACP prompt response (result.stopReason == "end_turn"), the synthetic
// _posthog/turn_complete notification, and the pi-shaped turn_completed event.
//
// True for a pi turn that ended in a runtime error too — the turn is over either way. Check
// isPiTurnError to tell the two apart before treating this as a successful completion.
export function isTurnComplete(event: Record<string, unknown>): boolean {
if (event['type'] === PI_EVENT_TYPE) {
return asPiTurnCompletedEvent(event) !== null
}
if (event['type'] !== ACP_NOTIFICATION_TYPE) {
return false
}
const notification = event['notification']
if (typeof notification !== 'object' || notification === null) {
return false
}
const notif = notification as Record<string, unknown>
if (notif['method'] === TURN_COMPLETE_METHOD) {
return true
}
const result = notif['result']
return (
typeof result === 'object' &&
result !== null &&
(result as Record<string, unknown>)['stopReason'] === STOP_REASON_END_TURN
)
}
// isPiTurnError mirrors ee/hogai/sandbox/types.py:pi_turn_error exactly.
// True when a pi turn_completed event reports a terminal runtime failure, so the caller
// can route it to a failed run instead of a successful turn completion.
export function isPiTurnError(event: Record<string, unknown>): boolean {
const piTurnCompleted = asPiTurnCompletedEvent(event)
return piTurnCompleted !== null && piTurnCompleted['stopReason'] === PI_STOP_REASON_ERROR
}
export function turnCompletedSuccessfully(event: Record<string, unknown>): boolean {
if (event['type'] === PI_EVENT_TYPE) {
const piTurnCompleted = asPiTurnCompletedEvent(event)
return piTurnCompleted !== null && piTurnCompleted['stopReason'] === STOP_REASON_END_TURN
}
if (event['type'] !== ACP_NOTIFICATION_TYPE) {
return false
}
const notification = event['notification']
if (typeof notification !== 'object' || notification === null) {
return false
}
const notif = notification as Record<string, unknown>
const payload = notif['method'] === TURN_COMPLETE_METHOD ? notif['params'] : notif['result']
return (
typeof payload === 'object' &&
payload !== null &&
(payload as Record<string, unknown>)['stopReason'] === STOP_REASON_END_TURN
)
}
export function isIdleResumeTurnComplete(event: Record<string, unknown>): boolean {
if (event['type'] !== ACP_NOTIFICATION_TYPE) {
return false
}
const notification = event['notification']
if (typeof notification !== 'object' || notification === null) {
return false
}
const notif = notification as Record<string, unknown>
if (notif['method'] !== TURN_COMPLETE_METHOD) {
return false
}
const params = notif['params']
return (
typeof params === 'object' &&
params !== null &&
(params as Record<string, unknown>)['stopReason'] === IDLE_RESUME_STOP_REASON
)
}
// isSessionUpdate mirrors event_ingest.py:_is_session_update exactly.
export function isSessionUpdate(event: Record<string, unknown>): boolean {
if (event['type'] !== ACP_NOTIFICATION_TYPE) {
return false
}
const notification = event['notification']
if (typeof notification !== 'object' || notification === null) {
return false
}
return (notification as Record<string, unknown>)['method'] === ACP_METHOD_SESSION_UPDATE
}
export function isAgentCommandDispatched(event: Record<string, unknown>): boolean {
if (event['type'] !== ACP_NOTIFICATION_TYPE) {
return false
}
const notification = event['notification']
return (
typeof notification === 'object' &&
notification !== null &&
(notification as Record<string, unknown>)['method'] === AGENT_COMMAND_DISPATCHED_METHOD
)
}
export function isAgentGenerationEvent(event: Record<string, unknown>): boolean {
if (event['type'] === 'pi_event') {
const piEvent = event['event']
return (
typeof piEvent === 'object' &&
piEvent !== null &&
PI_GENERATION_EVENTS.has(String((piEvent as Record<string, unknown>)['type']))
)
}
if (!isSessionUpdate(event)) {
return false
}
const notification = event['notification'] as Record<string, unknown>
const params = notification['params']
if (typeof params !== 'object' || params === null) {
return false
}
const update = (params as Record<string, unknown>)['update']
return (
typeof update === 'object' &&
update !== null &&
ACP_GENERATION_UPDATES.has(String((update as Record<string, unknown>)['sessionUpdate']))
)
}
const CALLBACK_TIMEOUT_MS = 10_000
const RETRY_DELAY_MS = 1000
const RETRYABLE_KINDS: ReadonlySet<SideEffectKind> = new Set([
'awaiting_input',
'permission_requested',
'turn_failed',
'budget_steer',
'process_killed',
])
const PROCESS_KILLED_METHOD = '_posthog/process_killed'
const PERMISSION_REQUEST_TYPE = 'permission_request'
const PERMISSION_REQUEST_METHOD = '_posthog/permission_request'
interface ProcessKilledPayload {
comm: string
signal: string
tree_rss_bytes: number
memory_current_bytes: number
memory_limit_bytes: number
}
function fetchErrorCode(err: unknown): string | undefined {
if (!(err instanceof Error)) {
return undefined
}
const cause = (err as Error & { cause?: unknown }).cause
if (typeof cause === 'object' && cause !== null && 'code' in cause) {
return String((cause as { code: unknown }).code)
}
return err.name
}
async function postCallback(url: string, init: RequestInit, runId: string, kind: SideEffectKind): Promise<Response> {
const attempt = (): Promise<Response> => fetch(url, { ...init, signal: AbortSignal.timeout(CALLBACK_TIMEOUT_MS) })
try {
const response = await attempt()
if (!RETRYABLE_KINDS.has(kind) || response.status < 500) {
return response
}
logger.warn('side_effect:retry', { run: runId, kind, status: response.status })
} catch (err: unknown) {
if (!RETRYABLE_KINDS.has(kind)) {
throw err
}
logger.warn('side_effect:retry', { run: runId, kind, code: fetchErrorCode(err) })
}
await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS))
return attempt()
}
// fireCallback issues a best-effort POST to the Django agent-proxy callback.
// The call is fire-and-forget: the promise is not returned to the caller.
// Any failure is logged but never thrown into the ingest path.
//
// The original sandbox_event_ingest JWT is forwarded as the Authorization
// header so Django can re-validate it with validate_sandbox_event_ingest_token
// (same RS256 public key, same audience).
function fireCallback(
runId: string,
kind: SideEffectKind,
agentActive: boolean,
taskId: string,
teamId: number,
originalToken: string,
config: Config,
options: {
releaseClaim?: () => Promise<void>
turnCompleted?: boolean
turnSucceeded?: boolean
activityStarted?: boolean
budgetSteer?: { sequence: number; timestamp?: string; params: Record<string, unknown> }
processKilled?: { sequence: number; payload: ProcessKilledPayload }
permissionRequest?: { sequence: number; requestId: string }
} = {}
): void {
const {
releaseClaim,
turnCompleted,
turnSucceeded,
activityStarted,
budgetSteer,
processKilled,
permissionRequest,
} = options
if (!config.djangoCallbackBaseUrl) {
// Dev environment without AGENT_PROXY_DJANGO_CALLBACK_URL — skip silently.
void releaseMilestoneClaim(releaseClaim, runId, kind)
return
}
const url = `${config.djangoCallbackBaseUrl}/internal/tasks/runs/${runId}/agent-proxy-callback/`
const body = JSON.stringify({
...(budgetSteer
? {
sequence: budgetSteer.sequence,
timestamp: budgetSteer.timestamp,
stage: budgetSteer.params['stage'],
mode: budgetSteer.params['mode'],
delivered: budgetSteer.params['delivered'],
spent_usd: budgetSteer.params['spent_usd'],
cap_usd: budgetSteer.params['cap_usd'],
threshold_spent_usd: budgetSteer.params['threshold_spent_usd'],
threshold_at: budgetSteer.params['threshold_at'],
delivered_at: budgetSteer.params['delivered_at'],
}
: {}),
...(processKilled ? { sequence: processKilled.sequence, process_killed: processKilled.payload } : {}),
...(permissionRequest ? { sequence: permissionRequest.sequence, request_id: permissionRequest.requestId } : {}),
kind,
agent_active: agentActive,
task_id: taskId,
team_id: teamId,
turn_completed: turnCompleted,
turn_succeeded: turnSucceeded,
activity_started: activityStarted,
})
const headers: Record<string, string> = {
'Content-Type': 'application/json',
Authorization: `Bearer ${originalToken}`,
}
// Service-to-service secret proving this call came from the proxy, not directly from a sandbox
// (which also holds the ingest JWT). Django enforces it only when it has the same secret configured.
if (config.agentProxyCallbackSecret) {
headers['X-Agent-Proxy-Secret'] = config.agentProxyCallbackSecret
}
logger.debug('side_effect:fire', { run: runId, kind, agentActive })
// Detached promise — errors are swallowed and logged.
postCallback(url, { method: 'POST', headers, body }, runId, kind)
.then(async (response) => {
if (!response.ok) {
logger.warn('side_effect:non_2xx', { run: runId, kind, status: response.status })
if (response.status >= 500) {
await releaseMilestoneClaim(releaseClaim, runId, kind)
}
return
}
const payload: unknown = await response.json().catch(() => null)
const dispatched =
typeof payload === 'object' &&
payload !== null &&
'dispatched' in payload &&
payload.dispatched === true
if (!dispatched) {
if (kind === 'budget_steer' || kind === 'process_killed') {
logger.warn(`side_effect:${kind}_not_captured`, { run: runId })
}
await releaseMilestoneClaim(releaseClaim, runId, kind)
}
})
.catch(async (err: unknown) => {
const message = err instanceof Error ? err.message : String(err)
logger.error('side_effect:failed', { run: runId, kind, error: message, code: fetchErrorCode(err) })
await releaseMilestoneClaim(releaseClaim, runId, kind)
})
}
export function captureBudgetSteerIfNeeded(
runId: string,
sequence: number,
event: Record<string, unknown>,
taskId: string,
teamId: number,
originalToken: string,
config: Config
): void {
if (event['type'] !== ACP_NOTIFICATION_TYPE) {
return
}
const notification = event['notification']
if (typeof notification !== 'object' || notification === null || Array.isArray(notification)) {
return
}
const { method, params } = notification as Record<string, unknown>
if (method !== '_posthog/budget_steer' || typeof params !== 'object' || params === null || Array.isArray(params)) {
return
}
fireCallback(runId, 'budget_steer', false, taskId, teamId, originalToken, config, {
budgetSteer: {
sequence,
...(typeof event['timestamp'] === 'string' ? { timestamp: event['timestamp'] } : {}),
params: params as Record<string, unknown>,
},
})
}
function byteCount(value: unknown): number | null {
return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? Math.trunc(value) : null
}
function parseProcessKilled(event: Record<string, unknown>): ProcessKilledPayload | null {
if (event['type'] !== ACP_NOTIFICATION_TYPE) {
return null
}
const notification = event['notification']
if (typeof notification !== 'object' || notification === null || Array.isArray(notification)) {
return null
}
const { method, params } = notification as Record<string, unknown>
if (method !== PROCESS_KILLED_METHOD || typeof params !== 'object' || params === null || Array.isArray(params)) {
return null
}
const fields = params as Record<string, unknown>
const comm = fields['comm']
const signal = fields['signal']
const treeRssBytes = byteCount(fields['treeRssBytes'])
const memoryCurrentBytes = byteCount(fields['memoryCurrentBytes'])
const memoryLimitBytes = byteCount(fields['memoryLimitBytes'])
if (typeof comm !== 'string' || typeof signal !== 'string') {
return null
}
if (treeRssBytes === null || memoryCurrentBytes === null || memoryLimitBytes === null) {
return null
}
return {
comm,
signal,
tree_rss_bytes: treeRssBytes,
memory_current_bytes: memoryCurrentBytes,
memory_limit_bytes: memoryLimitBytes,
}
}
export function captureProcessKilledIfNeeded(
runId: string,
sequence: number,
event: Record<string, unknown>,
taskId: string,
teamId: number,
originalToken: string,
config: Config
): void {
const payload = parseProcessKilled(event)
if (payload === null) {
return
}
fireCallback(runId, 'process_killed', false, taskId, teamId, originalToken, config, {
processKilled: { sequence, payload },
})
}
function asObject(value: unknown): Record<string, unknown> | null {
return typeof value === 'object' && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
: null
}
function permissionRequestPayload(event: Record<string, unknown>): Record<string, unknown> | null {
if (event['type'] === PERMISSION_REQUEST_TYPE) {
return event
}
const notification = asObject(event['notification'])
if (notification === null || notification['method'] !== PERMISSION_REQUEST_METHOD) {
return null
}
return asObject(notification['params'])
}
// permissionRequestId mirrors permission_broker.py:parse_permission_request exactly, so the proxy
// pushes for the same events as the Django ingest. Unlike isTurnComplete and the other ACP
// predicates, the notification shape does not need type 'notification'.
export function permissionRequestId(event: Record<string, unknown>): string | null {
const payload = permissionRequestPayload(event)
if (payload === null) {
return null
}
const requestId = payload['requestId']
const options = payload['options']
const hasOption =
Array.isArray(options) &&
options.some((option) => {
const optionId = asObject(option)?.['optionId']
return typeof optionId === 'string' && optionId !== ''
})
if (typeof requestId !== 'string' || requestId === '' || asObject(payload['toolCall']) === null || !hasOption) {
return null
}
return requestId
}
export function notifyPermissionRequestIfNeeded(
runId: string,
sequence: number,
event: Record<string, unknown>,
taskId: string,
teamId: number,
originalToken: string,
config: Config
): void {
const requestId = permissionRequestId(event)
if (requestId === null) {
return
}
fireCallback(runId, 'permission_requested', false, taskId, teamId, originalToken, config, {
permissionRequest: { sequence, requestId },
})
}
async function releaseMilestoneClaim(
release: (() => Promise<void>) | undefined,
runId: string,
kind: SideEffectKind
): Promise<void> {
if (!release) {
return
}
try {
await release()
} catch (err: unknown) {
const message = err instanceof Error ? err.message : String(err)
logger.error('side_effect:claim_release_failed', { run: runId, kind, error: message })
}
}
export async function heartbeatWorkflowIfNeeded(
redisStream: TaskRunRedisStream,
runId: string,
event: Record<string, unknown>,
taskId: string,
teamId: number,
originalToken: string,
config: Config
): Promise<void> {
if (isAgentCommandDispatched(event) && (await redisStream.claimFirstAgentCommand())) {
fireCallback(runId, 'command_dispatched', false, taskId, teamId, originalToken, config, {
releaseClaim: () => redisStream.releaseFirstAgentCommand(),
})
} else if (isAgentGenerationEvent(event) && (await redisStream.claimFirstAgentActivity())) {
fireCallback(runId, 'agent_activity', true, taskId, teamId, originalToken, config, {
releaseClaim: () => redisStream.releaseFirstAgentActivity(),
})
}
if (isTurnComplete(event)) {
await redisStream.setAgentActive(false)
if (isPiTurnError(event)) {
// A pi runtime error ends the turn but is not a successful completion —
// fail the run outright rather than reporting the turn as answered.
fireCallback(runId, 'turn_failed', false, taskId, teamId, originalToken, config)
} else {
// Let Django decide whether the run is interactive; it will only
// dispatch the push notification for interactive mode runs.
fireCallback(runId, 'awaiting_input', false, taskId, teamId, originalToken, config, {
turnCompleted: !isIdleResumeTurnComplete(event),
turnSucceeded: turnCompletedSuccessfully(event),
})
}
return
}
let agentActive: boolean
let activityStarted = false
if (isAgentGenerationEvent(event)) {
activityStarted = !(await redisStream.setAgentActive(true))
agentActive = true
} else {
agentActive = await redisStream.getAgentActive()
}
if (!agentActive) {
return
}
const claimed = await redisStream.claimAgentActiveHeartbeat(HEARTBEAT_THROTTLE_SECONDS)
if (!claimed && !activityStarted) {
return
}
fireCallback(runId, 'heartbeat', true, taskId, teamId, originalToken, config, { activityStarted })
}