Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

20.5 KB
Raw Blame
// Side-effect dispatch for the ingest handler.
//
// After each accepted ingest event the handler calls heartbeatWorkflowIfNeeded.
// Redis state mutations (setAgentActive, claimAgentActiveHeartbeat) are awaited
// because they gate the callback decision. The Django HTTP callback is
// fire-and-forget: the fetch result is consumed in a detached promise with
// error logging so failures never propagate into the ingest response.
//
// The Node service stays a pure streaming plane — Temporal signals and Celery
// push notifications go through a single internal Django endpoint. Django
// decides whether a run is interactive; Node always calls the callback and lets
// Django branch on mode.

import type { Config } from './config.js'
import { HEARTBEAT_THROTTLE_SECONDS } from './constants.js'
import { logger } from './logging.js'
import type { TaskRunRedisStream } from './redis-stream.js'
import type { SideEffectKind } from './types.js'

// ACP event field values (byte-identical to ee/hogai/sandbox/types.py)
const ACP_NOTIFICATION_TYPE = 'notification'
const TURN_COMPLETE_METHOD = '_posthog/turn_complete'
const STOP_REASON_END_TURN = 'end_turn'
const IDLE_RESUME_STOP_REASON = 'idle_resume'
const ACP_METHOD_SESSION_UPDATE = 'session/update'
const AGENT_COMMAND_DISPATCHED_METHOD = '_posthog/agent_command_dispatched'
const ACP_GENERATION_UPDATES = new Set([
    'agent_message',
    'agent_message_chunk',
    'agent_thought_chunk',
    'tool_call',
    'tool_call_update',
])
const PI_GENERATION_EVENTS = new Set([
    'assistant_message_chunk',
    'assistant_thought_chunk',
    'tool_call_started',
    'tool_call_updated',
])

// pi agent event shapes (byte-identical to ee/hogai/sandbox/types.py)
const PI_EVENT_TYPE = 'pi_event'
const PI_TURN_COMPLETED_TYPE = 'turn_completed'
const PI_STOP_REASON_ERROR = 'error'

function asPiTurnCompletedEvent(event: Record<string, unknown>): Record<string, unknown> | null {
    if (event['type'] !== PI_EVENT_TYPE) {
        return null
    }
    const piEvent = event['event']
    if (typeof piEvent !== 'object' || piEvent === null) {
        return null
    }
    const inner = piEvent as Record<string, unknown>
    return inner['type'] === PI_TURN_COMPLETED_TYPE ? inner : null
}

// isTurnComplete mirrors ee/hogai/sandbox/types.py:is_turn_complete exactly.
// Matches the raw ACP prompt response (result.stopReason == "end_turn"), the synthetic
// _posthog/turn_complete notification, and the pi-shaped turn_completed event.
//
// True for a pi turn that ended in a runtime error too — the turn is over either way. Check
// isPiTurnError to tell the two apart before treating this as a successful completion.
export function isTurnComplete(event: Record<string, unknown>): boolean {
    if (event['type'] === PI_EVENT_TYPE) {
        return asPiTurnCompletedEvent(event) !== null
    }
    if (event['type'] !== ACP_NOTIFICATION_TYPE) {
        return false
    }
    const notification = event['notification']
    if (typeof notification !== 'object' || notification === null) {
        return false
    }
    const notif = notification as Record<string, unknown>
    if (notif['method'] === TURN_COMPLETE_METHOD) {
        return true
    }
    const result = notif['result']
    return (
        typeof result === 'object' &&
        result !== null &&
        (result as Record<string, unknown>)['stopReason'] === STOP_REASON_END_TURN
    )
}

// isPiTurnError mirrors ee/hogai/sandbox/types.py:pi_turn_error exactly.
// True when a pi turn_completed event reports a terminal runtime failure, so the caller
// can route it to a failed run instead of a successful turn completion.
export function isPiTurnError(event: Record<string, unknown>): boolean {
    const piTurnCompleted = asPiTurnCompletedEvent(event)
    return piTurnCompleted !== null && piTurnCompleted['stopReason'] === PI_STOP_REASON_ERROR
}

export function turnCompletedSuccessfully(event: Record<string, unknown>): boolean {
    if (event['type'] === PI_EVENT_TYPE) {
        const piTurnCompleted = asPiTurnCompletedEvent(event)
        return piTurnCompleted !== null && piTurnCompleted['stopReason'] === STOP_REASON_END_TURN
    }
    if (event['type'] !== ACP_NOTIFICATION_TYPE) {
        return false
    }
    const notification = event['notification']
    if (typeof notification !== 'object' || notification === null) {
        return false
    }
    const notif = notification as Record<string, unknown>
    const payload = notif['method'] === TURN_COMPLETE_METHOD ? notif['params'] : notif['result']
    return (
        typeof payload === 'object' &&
        payload !== null &&
        (payload as Record<string, unknown>)['stopReason'] === STOP_REASON_END_TURN
    )
}

export function isIdleResumeTurnComplete(event: Record<string, unknown>): boolean {
    if (event['type'] !== ACP_NOTIFICATION_TYPE) {
        return false
    }
    const notification = event['notification']
    if (typeof notification !== 'object' || notification === null) {
        return false
    }
    const notif = notification as Record<string, unknown>
    if (notif['method'] !== TURN_COMPLETE_METHOD) {
        return false
    }
    const params = notif['params']
    return (
        typeof params === 'object' &&
        params !== null &&
        (params as Record<string, unknown>)['stopReason'] === IDLE_RESUME_STOP_REASON
    )
}

// isSessionUpdate mirrors event_ingest.py:_is_session_update exactly.
export function isSessionUpdate(event: Record<string, unknown>): boolean {
    if (event['type'] !== ACP_NOTIFICATION_TYPE) {
        return false
    }
    const notification = event['notification']
    if (typeof notification !== 'object' || notification === null) {
        return false
    }
    return (notification as Record<string, unknown>)['method'] === ACP_METHOD_SESSION_UPDATE
}

export function isAgentCommandDispatched(event: Record<string, unknown>): boolean {
    if (event['type'] !== ACP_NOTIFICATION_TYPE) {
        return false
    }
    const notification = event['notification']
    return (
        typeof notification === 'object' &&
        notification !== null &&
        (notification as Record<string, unknown>)['method'] === AGENT_COMMAND_DISPATCHED_METHOD
    )
}

export function isAgentGenerationEvent(event: Record<string, unknown>): boolean {
    if (event['type'] === 'pi_event') {
        const piEvent = event['event']
        return (
            typeof piEvent === 'object' &&
            piEvent !== null &&
            PI_GENERATION_EVENTS.has(String((piEvent as Record<string, unknown>)['type']))
        )
    }
    if (!isSessionUpdate(event)) {
        return false
    }
    const notification = event['notification'] as Record<string, unknown>
    const params = notification['params']
    if (typeof params !== 'object' || params === null) {
        return false
    }
    const update = (params as Record<string, unknown>)['update']
    return (
        typeof update === 'object' &&
        update !== null &&
        ACP_GENERATION_UPDATES.has(String((update as Record<string, unknown>)['sessionUpdate']))
    )
}

const CALLBACK_TIMEOUT_MS = 10_000
const RETRY_DELAY_MS = 1000
const RETRYABLE_KINDS: ReadonlySet<SideEffectKind> = new Set([
    'awaiting_input',
    'permission_requested',
    'turn_failed',
    'budget_steer',
    'process_killed',
])
const PROCESS_KILLED_METHOD = '_posthog/process_killed'
const PERMISSION_REQUEST_TYPE = 'permission_request'
const PERMISSION_REQUEST_METHOD = '_posthog/permission_request'

interface ProcessKilledPayload {
    comm: string
    signal: string
    tree_rss_bytes: number
    memory_current_bytes: number
    memory_limit_bytes: number
}

function fetchErrorCode(err: unknown): string | undefined {
    if (!(err instanceof Error)) {
        return undefined
    }
    const cause = (err as Error & { cause?: unknown }).cause
    if (typeof cause === 'object' && cause !== null && 'code' in cause) {
        return String((cause as { code: unknown }).code)
    }
    return err.name
}

async function postCallback(url: string, init: RequestInit, runId: string, kind: SideEffectKind): Promise<Response> {
    const attempt = (): Promise<Response> => fetch(url, { ...init, signal: AbortSignal.timeout(CALLBACK_TIMEOUT_MS) })
    try {
        const response = await attempt()
        if (!RETRYABLE_KINDS.has(kind) || response.status < 500) {
            return response
        }
        logger.warn('side_effect:retry', { run: runId, kind, status: response.status })
    } catch (err: unknown) {
        if (!RETRYABLE_KINDS.has(kind)) {
            throw err
        }
        logger.warn('side_effect:retry', { run: runId, kind, code: fetchErrorCode(err) })
    }
    await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS))
    return attempt()
}

// fireCallback issues a best-effort POST to the Django agent-proxy callback.
// The call is fire-and-forget: the promise is not returned to the caller.
// Any failure is logged but never thrown into the ingest path.
//
// The original sandbox_event_ingest JWT is forwarded as the Authorization
// header so Django can re-validate it with validate_sandbox_event_ingest_token
// (same RS256 public key, same audience).
function fireCallback(
    runId: string,
    kind: SideEffectKind,
    agentActive: boolean,
    taskId: string,
    teamId: number,
    originalToken: string,
    config: Config,
    options: {
        releaseClaim?: () => Promise<void>
        turnCompleted?: boolean
        turnSucceeded?: boolean
        activityStarted?: boolean
        budgetSteer?: { sequence: number; timestamp?: string; params: Record<string, unknown> }
        processKilled?: { sequence: number; payload: ProcessKilledPayload }
        permissionRequest?: { sequence: number; requestId: string }
    } = {}
): void {
    const {
        releaseClaim,
        turnCompleted,
        turnSucceeded,
        activityStarted,
        budgetSteer,
        processKilled,
        permissionRequest,
    } = options
    if (!config.djangoCallbackBaseUrl) {
        // Dev environment without AGENT_PROXY_DJANGO_CALLBACK_URL — skip silently.
        void releaseMilestoneClaim(releaseClaim, runId, kind)
        return
    }

    const url = `${config.djangoCallbackBaseUrl}/internal/tasks/runs/${runId}/agent-proxy-callback/`
    const body = JSON.stringify({
        ...(budgetSteer
            ? {
                  sequence: budgetSteer.sequence,
                  timestamp: budgetSteer.timestamp,
                  stage: budgetSteer.params['stage'],
                  mode: budgetSteer.params['mode'],
                  delivered: budgetSteer.params['delivered'],
                  spent_usd: budgetSteer.params['spent_usd'],
                  cap_usd: budgetSteer.params['cap_usd'],
                  threshold_spent_usd: budgetSteer.params['threshold_spent_usd'],
                  threshold_at: budgetSteer.params['threshold_at'],
                  delivered_at: budgetSteer.params['delivered_at'],
              }
            : {}),
        ...(processKilled ? { sequence: processKilled.sequence, process_killed: processKilled.payload } : {}),
        ...(permissionRequest ? { sequence: permissionRequest.sequence, request_id: permissionRequest.requestId } : {}),
        kind,
        agent_active: agentActive,
        task_id: taskId,
        team_id: teamId,
        turn_completed: turnCompleted,
        turn_succeeded: turnSucceeded,
        activity_started: activityStarted,
    })

    const headers: Record<string, string> = {
        'Content-Type': 'application/json',
        Authorization: `Bearer ${originalToken}`,
    }
    // Service-to-service secret proving this call came from the proxy, not directly from a sandbox
    // (which also holds the ingest JWT). Django enforces it only when it has the same secret configured.
    if (config.agentProxyCallbackSecret) {
        headers['X-Agent-Proxy-Secret'] = config.agentProxyCallbackSecret
    }

    logger.debug('side_effect:fire', { run: runId, kind, agentActive })

    // Detached promise — errors are swallowed and logged.
    postCallback(url, { method: 'POST', headers, body }, runId, kind)
        .then(async (response) => {
            if (!response.ok) {
                logger.warn('side_effect:non_2xx', { run: runId, kind, status: response.status })
                if (response.status >= 500) {
                    await releaseMilestoneClaim(releaseClaim, runId, kind)
                }
                return
            }
            const payload: unknown = await response.json().catch(() => null)
            const dispatched =
                typeof payload === 'object' &&
                payload !== null &&
                'dispatched' in payload &&
                payload.dispatched === true
            if (!dispatched) {
                if (kind === 'budget_steer' || kind === 'process_killed') {
                    logger.warn(`side_effect:${kind}_not_captured`, { run: runId })
                }
                await releaseMilestoneClaim(releaseClaim, runId, kind)
            }
        })
        .catch(async (err: unknown) => {
            const message = err instanceof Error ? err.message : String(err)
            logger.error('side_effect:failed', { run: runId, kind, error: message, code: fetchErrorCode(err) })
            await releaseMilestoneClaim(releaseClaim, runId, kind)
        })
}

export function captureBudgetSteerIfNeeded(
    runId: string,
    sequence: number,
    event: Record<string, unknown>,
    taskId: string,
    teamId: number,
    originalToken: string,
    config: Config
): void {
    if (event['type'] !== ACP_NOTIFICATION_TYPE) {
        return
    }
    const notification = event['notification']
    if (typeof notification !== 'object' || notification === null || Array.isArray(notification)) {
        return
    }
    const { method, params } = notification as Record<string, unknown>
    if (method !== '_posthog/budget_steer' || typeof params !== 'object' || params === null || Array.isArray(params)) {
        return
    }
    fireCallback(runId, 'budget_steer', false, taskId, teamId, originalToken, config, {
        budgetSteer: {
            sequence,
            ...(typeof event['timestamp'] === 'string' ? { timestamp: event['timestamp'] } : {}),
            params: params as Record<string, unknown>,
        },
    })
}

function byteCount(value: unknown): number | null {
    return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? Math.trunc(value) : null
}

function parseProcessKilled(event: Record<string, unknown>): ProcessKilledPayload | null {
    if (event['type'] !== ACP_NOTIFICATION_TYPE) {
        return null
    }
    const notification = event['notification']
    if (typeof notification !== 'object' || notification === null || Array.isArray(notification)) {
        return null
    }
    const { method, params } = notification as Record<string, unknown>
    if (method !== PROCESS_KILLED_METHOD || typeof params !== 'object' || params === null || Array.isArray(params)) {
        return null
    }
    const fields = params as Record<string, unknown>
    const comm = fields['comm']
    const signal = fields['signal']
    const treeRssBytes = byteCount(fields['treeRssBytes'])
    const memoryCurrentBytes = byteCount(fields['memoryCurrentBytes'])
    const memoryLimitBytes = byteCount(fields['memoryLimitBytes'])
    if (typeof comm !== 'string' || typeof signal !== 'string') {
        return null
    }
    if (treeRssBytes === null || memoryCurrentBytes === null || memoryLimitBytes === null) {
        return null
    }
    return {
        comm,
        signal,
        tree_rss_bytes: treeRssBytes,
        memory_current_bytes: memoryCurrentBytes,
        memory_limit_bytes: memoryLimitBytes,
    }
}

export function captureProcessKilledIfNeeded(
    runId: string,
    sequence: number,
    event: Record<string, unknown>,
    taskId: string,
    teamId: number,
    originalToken: string,
    config: Config
): void {
    const payload = parseProcessKilled(event)
    if (payload === null) {
        return
    }
    fireCallback(runId, 'process_killed', false, taskId, teamId, originalToken, config, {
        processKilled: { sequence, payload },
    })
}

function asObject(value: unknown): Record<string, unknown> | null {
    return typeof value === 'object' && value !== null && !Array.isArray(value)
        ? (value as Record<string, unknown>)
        : null
}

function permissionRequestPayload(event: Record<string, unknown>): Record<string, unknown> | null {
    if (event['type'] === PERMISSION_REQUEST_TYPE) {
        return event
    }
    const notification = asObject(event['notification'])
    if (notification === null || notification['method'] !== PERMISSION_REQUEST_METHOD) {
        return null
    }
    return asObject(notification['params'])
}

// permissionRequestId mirrors permission_broker.py:parse_permission_request exactly, so the proxy
// pushes for the same events as the Django ingest. Unlike isTurnComplete and the other ACP
// predicates, the notification shape does not need type 'notification'.
export function permissionRequestId(event: Record<string, unknown>): string | null {
    const payload = permissionRequestPayload(event)
    if (payload === null) {
        return null
    }
    const requestId = payload['requestId']
    const options = payload['options']
    const hasOption =
        Array.isArray(options) &&
        options.some((option) => {
            const optionId = asObject(option)?.['optionId']
            return typeof optionId === 'string' && optionId !== ''
        })
    if (typeof requestId !== 'string' || requestId === '' || asObject(payload['toolCall']) === null || !hasOption) {
        return null
    }
    return requestId
}

export function notifyPermissionRequestIfNeeded(
    runId: string,
    sequence: number,
    event: Record<string, unknown>,
    taskId: string,
    teamId: number,
    originalToken: string,
    config: Config
): void {
    const requestId = permissionRequestId(event)
    if (requestId === null) {
        return
    }
    fireCallback(runId, 'permission_requested', false, taskId, teamId, originalToken, config, {
        permissionRequest: { sequence, requestId },
    })
}

async function releaseMilestoneClaim(
    release: (() => Promise<void>) | undefined,
    runId: string,
    kind: SideEffectKind
): Promise<void> {
    if (!release) {
        return
    }
    try {
        await release()
    } catch (err: unknown) {
        const message = err instanceof Error ? err.message : String(err)
        logger.error('side_effect:claim_release_failed', { run: runId, kind, error: message })
    }
}

export async function heartbeatWorkflowIfNeeded(
    redisStream: TaskRunRedisStream,
    runId: string,
    event: Record<string, unknown>,
    taskId: string,
    teamId: number,
    originalToken: string,
    config: Config
): Promise<void> {
    if (isAgentCommandDispatched(event) && (await redisStream.claimFirstAgentCommand())) {
        fireCallback(runId, 'command_dispatched', false, taskId, teamId, originalToken, config, {
            releaseClaim: () => redisStream.releaseFirstAgentCommand(),
        })
    } else if (isAgentGenerationEvent(event) && (await redisStream.claimFirstAgentActivity())) {
        fireCallback(runId, 'agent_activity', true, taskId, teamId, originalToken, config, {
            releaseClaim: () => redisStream.releaseFirstAgentActivity(),
        })
    }

    if (isTurnComplete(event)) {
        await redisStream.setAgentActive(false)
        if (isPiTurnError(event)) {
            // A pi runtime error ends the turn but is not a successful completion —
            // fail the run outright rather than reporting the turn as answered.
            fireCallback(runId, 'turn_failed', false, taskId, teamId, originalToken, config)
        } else {
            // Let Django decide whether the run is interactive; it will only
            // dispatch the push notification for interactive mode runs.
            fireCallback(runId, 'awaiting_input', false, taskId, teamId, originalToken, config, {
                turnCompleted: !isIdleResumeTurnComplete(event),
                turnSucceeded: turnCompletedSuccessfully(event),
            })
        }
        return
    }

    let agentActive: boolean
    let activityStarted = false
    if (isAgentGenerationEvent(event)) {
        activityStarted = !(await redisStream.setAgentActive(true))
        agentActive = true
    } else {
        agentActive = await redisStream.getAgentActive()
    }

    if (!agentActive) {
        return
    }

    const claimed = await redisStream.claimAgentActiveHeartbeat(HEARTBEAT_THROTTLE_SECONDS)
    if (!claimed && !activityStarted) {
        return
    }

    fireCallback(runId, 'heartbeat', true, taskId, teamId, originalToken, config, { activityStarted })
}