Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(posthog-ai): report how long a task chat takes to load its history (#120753)

Shy Alter9e056165e1b5
diff --git a/products/posthog_ai/frontend/logics/runStreamLogic.test.ts b/products/posthog_ai/frontend/logics/runStreamLogic.test.ts
index 41fe317871d8..3f047c5b6961 100644
--- a/products/posthog_ai/frontend/logics/runStreamLogic.test.ts
+++ b/products/posthog_ai/frontend/logics/runStreamLogic.test.ts
@@ -2740,6 +2740,50 @@ describe('runStreamLogic', () => {
await expectLogic(logic).toFinishAllListeners()
})
+ it.each([
+ { name: 'an existing run', justCreatedRun: false, captured: 1 },
+ { name: 'a just-created run', justCreatedRun: true, captured: 0 },
+ ])('reports history load time once for $name', async ({ justCreatedRun, captured }) => {
+ const captureSpy = jest.spyOn(posthog, 'capture').mockImplementation(() => undefined as any)
+ jest.mocked(tasksRunsRetrieve).mockResolvedValue({ status: 'completed', state: {} } as any)
+ jest.spyOn(api.tasks.runs, 'getLogEntries').mockResolvedValue([
+ sessionUpdate({ sessionUpdate: 'agent_message', content: { type: 'text', text: 'Done.' } }),
+ ] as any)
+
+ await expectLogic(logic, () => {
+ logic.actions.bootstrapRun({ taskId: 'task-1', runId: 'run-1', justCreatedRun })
+ }).toFinishAllListeners()
+
+ const loaded = captureSpy.mock.calls.filter(([event]) => event === 'sandbox_run_history_loaded')
+ expect(loaded.map(([, properties]) => properties)).toEqual(
+ Array(captured).fill(
+ expect.objectContaining({ run_id: 'run-1', entry_count: 1, run_status: 'completed' })
+ )
+ )
+ })
+
+ it('times a retried history load from the retry, not the failed attempt', async () => {
+ jest.useFakeTimers()
+ try {
+ const captureSpy = jest.spyOn(posthog, 'capture').mockImplementation(() => undefined as any)
+ jest.mocked(tasksRunsRetrieve).mockResolvedValue({ status: 'completed', state: {} } as any)
+ const logs = jest.spyOn(api.tasks.runs, 'getLogEntries').mockRejectedValue({ status: 503 })
+
+ logic.actions.bootstrapRun({ taskId: 'task-1', runId: 'run-1' })
+ await jest.advanceTimersByTimeAsync(30_000)
+ expect(logic.values.bootstrapError?.retryable).toBe(true)
+ await jest.advanceTimersByTimeAsync(60_000)
+ logs.mockResolvedValue([])
+ logic.actions.retryConnection()
+ await jest.advanceTimersByTimeAsync(1_000)
+
+ const loaded = captureSpy.mock.calls.filter(([event]) => event === 'sandbox_run_history_loaded')
+ expect(loaded.map(([, properties]) => properties?.duration_ms < 5_000)).toEqual([true])
+ } finally {
+ jest.useRealTimers()
+ }
+ })
+
it('clears the bootstrap spinner for a terminal run whose history renders no rows', async () => {
jest.mocked(tasksRunsRetrieve).mockResolvedValue({ status: 'failed', state: {} } as any)
jest.spyOn(api.tasks.runs, 'getLogEntries').mockResolvedValue([])
@@ -6620,7 +6664,7 @@ describe('runStreamLogic', () => {
await flushPromises()
expect(logic.values.pendingPermissionRequest?.requestId).toEqual(actionable ? 'req-1' : undefined)
- expect(captureSpy).not.toHaveBeenCalled()
+ expect(captureSpy).not.toHaveBeenCalledWith('permission_requested', expect.anything())
logic.actions.respondToPermission({ requestId: 'req-1', optionId: 'allow_once' })
await flushPromises()
expect(tasksRunsCommandCreate).toHaveBeenCalledTimes(actionable ? 1 : 0)
diff --git a/products/posthog_ai/frontend/logics/runStreamLogic.ts b/products/posthog_ai/frontend/logics/runStreamLogic.ts
index e872fee45fed..7b71a7889116 100644
--- a/products/posthog_ai/frontend/logics/runStreamLogic.ts
+++ b/products/posthog_ai/frontend/logics/runStreamLogic.ts
@@ -3821,6 +3821,7 @@ export const runStreamLogic = kea<runStreamLogicType>([
if (!session.owns() || session.paused) {
return
}
+ cache.bootstrapStartedAt = undefined
const envelope =
isRecord(error) && typeof error.errorTitle === 'string'
? (error as unknown as StreamErrorEnvelope)
@@ -3992,6 +3993,18 @@ export const runStreamLogic = kea<runStreamLogicType>([
}
actions.setHistoryComplete(true)
actions.bootstrapLogReady()
+ if (cache.bootstrapStartedAt !== undefined) {
+ posthog.capture('sandbox_run_history_loaded', {
+ conversation_id: props.conversationId,
+ task_id: session.taskId,
+ run_id: session.runId,
+ run_status: values.currentRunStatus,
+ entry_count: entries.length,
+ duration_ms: Math.round(performance.now() - cache.bootstrapStartedAt),
+ execution_type: 'sandbox',
+ })
+ cache.bootstrapStartedAt = undefined
+ }
if (session.phase !== 'finalization') {
recordRecovery(session)
}
@@ -4054,6 +4067,7 @@ export const runStreamLogic = kea<runStreamLogicType>([
}
cache.retainedMessage = retainedMessage
cache.isBootstrapping = true
+ cache.bootstrapStartedAt = justCreatedRun ? undefined : performance.now()
actions.setHistoryComplete(false)
try {
if (hasEnded(session)) {
@@ -4789,6 +4803,7 @@ export const runStreamLogic = kea<runStreamLogicType>([
},
reset: () => {
cache.disposables.dispose('stream-session')
+ cache.bootstrapStartedAt = undefined
cache.recoverySession = undefined
cache.eventCoverage = undefined
cache.retainedMessage = undefined