Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): stop reporting expected offline evaluation 404s (#118057)

posthog[bot]af24c234cc6b
diff --git a/frontend/src/initKea.ts b/frontend/src/initKea.ts
index 128ca0ef25b4..38863a07b555 100644
--- a/frontend/src/initKea.ts
+++ b/frontend/src/initKea.ts
@@ -171,6 +171,14 @@ purpose, so each caller that degrades has to name itself here, next to the toast
const NOT_FOUND_SELF_HANDLED = new Set([
'loadRecordingMeta', // The player renders RecordingNotFound off sessionRecordingMetaLogic's isNotFound
'loadLineage', // A metric has no lineage node until the sync task runs; the panel says so and retries
+ 'loadOfflineExperiment', // Offline evaluation views render a stale or deleted link as an inline error
+ 'loadOfflineSummaries',
+ 'loadOfflineItems',
+ 'loadOfflineItem',
+ 'loadOfflineItemPayload',
+ 'loadOfflineItemResults',
+ 'loadOfflineSelectedResult',
+ 'loadOfflineResultPayload',
])
/*
diff --git a/products/ai_observability/frontend/offline-evaluations/offlineExperimentLogic.test.ts b/products/ai_observability/frontend/offline-evaluations/offlineExperimentLogic.test.ts
index 02ea86e09b5d..6e350ffe00e1 100644
--- a/products/ai_observability/frontend/offline-evaluations/offlineExperimentLogic.test.ts
+++ b/products/ai_observability/frontend/offline-evaluations/offlineExperimentLogic.test.ts
@@ -1,4 +1,5 @@
import { expectLogic } from 'kea-test-utils'
+import posthog from 'posthog-js'
import { ApiError } from 'lib/api-error'
@@ -69,6 +70,21 @@ describe('offlineExperimentLogic', () => {
expect(logic.values.cellBatches[0]?.state).toBe('loaded')
})
+ it.each([
+ [404, 0],
+ [500, 3],
+ ])('reports %s failures from the parallel experiment loads %s times', async (status, reports) => {
+ const captureException = jest.spyOn(posthog, 'captureException').mockImplementation(() => undefined)
+ readExperiment.mockRejectedValue(new ApiError('Failed', status))
+ readSummaries.mockRejectedValue(new ApiError('Failed', status))
+ readItems.mockRejectedValue(new ApiError('Failed', status))
+ const logic = offlineExperimentLogic({ teamId: 543, experimentId: detailExperiment.id })
+ logic.mount()
+ await expectLogic(logic).toFinishAllListeners()
+ expect(logic.values.experimentError).toBeTruthy()
+ expect(captureException).toHaveBeenCalledTimes(reports)
+ })
+
it('does not replace cells for a new item page with a late response from the previous page', async () => {
const logic = offlineExperimentLogic({ teamId: 997, experimentId: detailExperiment.id })
logic.mount()