Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(observability): measure query request latency phases (#115652)

Vasco de Krijgeradc2bc23ae0c
diff --git a/docs/published/handbook/engineering/conventions/backend-coding.md b/docs/published/handbook/engineering/conventions/backend-coding.md
index 885b2919b8f0..71491297f744 100644
--- a/docs/published/handbook/engineering/conventions/backend-coding.md
+++ b/docs/published/handbook/engineering/conventions/backend-coding.md
@@ -11,6 +11,14 @@ A matching ID links a request to its processing span. It does not prove that the
An asynchronous query can continue in a separate trace after the submission request ends.
Check request timing against browser performance metrics, and inspect child spans and self time before choosing a performance fix.
+Query API responses expose a validated HTTP request UUID through `X-Request-ID` and the `Server-Timing` metric named `request_id`.
+Query analytics include the same UUID as `http_request_id`.
+The browser records response wait, body read, JSON parsing, and available resource phases on existing query outcome events.
+Resource phases require one matching response UUID; missing, ambiguous, or truncated evidence remains explicit.
+Cross-origin resource details also require `Timing-Allow-Origin`.
+The pre-request phase includes connection setup and browser waiting; it does not measure server queueing.
+Automatic capture covers `performQuery` submission POSTs and the status poll that returns a completed async result; earlier polls and rejected-response body processing remain outside this capture.
+
#### Logging
As a general rule, we should have logs for every expected and unexpected actions of the application, using the appropriate _log level_.
diff --git a/frontend/src/lib/api.test.ts b/frontend/src/lib/api.test.ts
index 2caeb931ee8b..8d820aa8306d 100644
--- a/frontend/src/lib/api.test.ts
+++ b/frontend/src/lib/api.test.ts
@@ -4,6 +4,7 @@ import posthog from 'posthog-js'
import api, { ApiConfig, ApiError, ApiRequest, NetworkError, ResponseBodyReadError } from 'lib/api'
import { shouldReportApiFailure } from 'lib/api-error'
import { apiStatusLogic } from 'lib/logic/apiStatusLogic'
+import * as oauthClient from 'lib/oauth/oauthClient'
import { NodeKind } from '~/queries/schema/schema-general'
import { PropertyFilterType, PropertyOperator } from '~/types'
@@ -187,6 +188,118 @@ describe('API helper', () => {
})
describe('query endpoints', () => {
+ it('observes response phases without changing the query result', async () => {
+ const requestId = '11111111-1111-4111-8111-111111111111'
+ let clock = 10
+ const now = jest.spyOn(performance, 'now').mockImplementation(() => clock)
+ const observeTiming = jest.fn()
+ fakeFetch.mockImplementation(async () => {
+ clock = 110
+ return {
+ ok: true,
+ status: 200,
+ headers: new Headers({ 'X-Request-ID': requestId, 'X-Envoy-Upstream-Service-Time': '80' }),
+ text: async () => {
+ clock = 310
+ return JSON.stringify(FAKE_FETCH_RESULT)
+ },
+ }
+ })
+
+ try {
+ await expect(
+ api.query(
+ { kind: NodeKind.HogQLQuery, query: 'select 1' },
+ { requestOptions: { onRequestTiming: observeTiming } }
+ )
+ ).resolves.toEqual(FAKE_FETCH_RESULT)
+ expect(observeTiming).toHaveBeenCalledTimes(1)
+ expect(observeTiming).toHaveBeenCalledWith(
+ expect.objectContaining({
+ request_total_ms: 300,
+ time_to_headers_ms: 100,
+ body_read_ms: 200,
+ json_parse_ms: 0,
+ http_status: 200,
+ http_request_id: requestId,
+ envoy_upstream_ms: 80,
+ })
+ )
+ expect(observeTiming.mock.calls[0][0]).not.toHaveProperty('url')
+ expect(observeTiming.mock.calls[0][0]).not.toHaveProperty('query')
+ } finally {
+ now.mockRestore()
+ }
+ })
+
+ it('keeps successful queries successful when a timing observer throws', async () => {
+ await expect(
+ api.query(
+ { kind: NodeKind.HogQLQuery, query: 'select 1' },
+ {
+ requestOptions: {
+ onRequestTiming: () => {
+ throw new Error('observer failed')
+ },
+ },
+ }
+ )
+ ).resolves.toEqual(FAKE_FETCH_RESULT)
+ })
+
+ it('preserves a body-read cancellation when a timing observer throws', async () => {
+ const abort = new DOMException('Cancelled', 'AbortError')
+ fakeFetch.mockResolvedValue({
+ ok: true,
+ status: 200,
+ text: () => Promise.reject(abort),
+ })
+ const observer = jest.fn(() => {
+ throw new Error('observer failed')
+ })
+
+ await expect(
+ api.query(
+ { kind: NodeKind.HogQLQuery, query: 'select 1' },
+ { requestOptions: { onRequestTiming: observer } }
+ )
+ ).rejects.toBe(abort)
+ expect(observer).toHaveBeenCalledWith(
+ expect.objectContaining({ http_status: 200, body_read_ms: expect.any(Number) })
+ )
+ })
+
+ it('observes rejected response headers before the error body is consumed', async () => {
+ let clock = 10
+ const now = jest.spyOn(performance, 'now').mockImplementation(() => clock)
+ const observer = jest.fn()
+ fakeFetch.mockImplementation(async () => {
+ clock = 110
+ return {
+ ok: false,
+ status: 500,
+ headers: new Headers({ 'X-Request-ID': '11111111-1111-4111-8111-111111111111' }),
+ json: async () => {
+ clock = 310
+ return { detail: 'error' }
+ },
+ }
+ })
+ try {
+ await expect(
+ api.query(
+ { kind: NodeKind.HogQLQuery, query: 'select 1' },
+ { requestOptions: { onRequestTiming: observer } }
+ )
+ ).rejects.toMatchObject({ status: 500 })
+ expect(observer).toHaveBeenCalledWith(
+ expect.objectContaining({ request_total_ms: 300, time_to_headers_ms: 100, http_status: 500 })
+ )
+ } finally {
+ now.mockRestore()
+ }
+ })
+
it('adds query kind to the query URL when present', async () => {
await api.query({ kind: NodeKind.HogQLQuery, query: 'select 1' })
@@ -348,6 +461,37 @@ describe('API helper', () => {
window.localStorage.removeItem(OAUTH_SESSION_KEY)
})
+ it.each([new TypeError('Failed to fetch'), new DOMException('Cancelled', 'AbortError')])(
+ 'clears the preceding response identity when an auth retry throws %s',
+ async (failure) => {
+ const refresh = jest.spyOn(oauthClient, 'refreshAccessToken').mockResolvedValue('refreshed-token')
+ const observer = jest.fn()
+ fakeFetch
+ .mockResolvedValueOnce({
+ ok: false,
+ status: 401,
+ headers: new Headers({ 'X-Request-ID': '11111111-1111-4111-8111-111111111111' }),
+ })
+ .mockRejectedValueOnce(failure)
+ try {
+ await expect(
+ api.query(
+ { kind: NodeKind.HogQLQuery, query: 'select 1' },
+ { requestOptions: { onRequestTiming: observer } }
+ )
+ ).rejects.toBeInstanceOf(failure.name === 'AbortError' ? DOMException : NetworkError)
+ expect(fakeFetch).toHaveBeenCalledTimes(2)
+ expect(observer).toHaveBeenCalledTimes(1)
+ expect(observer.mock.calls[0][0]).toMatchObject({ resource_timing_status: 'missing_request_id' })
+ expect(observer.mock.calls[0][0].http_status).toBeUndefined()
+ expect(observer.mock.calls[0][0].http_request_id).toBeUndefined()
+ expect(observer.mock.calls[0][0].time_to_headers_ms).toBeUndefined()
+ } finally {
+ refresh.mockRestore()
+ }
+ }
+ )
+
it('attaches the bearer token to requests routed to the OAuth backend host', async () => {
await api.get('/api/projects/2/insights/')
const [url, options] = fakeFetch.mock.calls[0]
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts
index ef114b1894f4..8360468589f3 100644
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -15,6 +15,7 @@ import {
readableErrorMessage,
ResponseBodyReadError,
} from 'lib/api-error'
+import { ApiRequestTiming, ApiRequestTimingRecorder } from 'lib/apiRequestTiming'
import { ActivityLogProps } from 'lib/components/ActivityLog/ActivityLog'
import { ActivityLogItem } from 'lib/components/ActivityLog/humanizeActivity'
import { apiStatusLogic, awaitReauthentication, isUserActionInProgress } from 'lib/logic/apiStatusLogic'
@@ -311,6 +312,7 @@ export interface ActivityLogPaginatedResponse<T> extends PaginatedResponse<T> {
export interface ApiMethodOptions {
signal?: AbortSignal
headers?: Record<string, any>
+ onRequestTiming?: (timing: ApiRequestTiming) => void
}
export interface ApiUploadProgress {
@@ -398,7 +400,12 @@ function apiErrorFallback(response: Response, method: string, url: string): stri
* with unparsable content) throws `ResponseBodyReadError`, so it can be recognized as wire-level
* noise and left out of error tracking.
*/
-export async function getJSONFromSuccessResponse(response: Response, method: string, url: string): Promise<any> {
+export async function getJSONFromSuccessResponse(
+ response: Response,
+ method: string,
+ url: string,
+ timing?: ApiRequestTimingRecorder
+): Promise<any> {
const requestContext = (): string =>
`[${method} ${new URL(url, location.origin).pathname}] (status ${response.status})`
// A no-content response must not depend on reading its body: some engines (in our telemetry,
@@ -408,7 +415,7 @@ export async function getJSONFromSuccessResponse(response: Response, method: str
}
let text: string
try {
- text = await response.text()
+ text = timing ? await timing.readBody(response) : await response.text()
} catch (error) {
if (isAbortError(error)) {
throw error
@@ -432,7 +439,7 @@ export async function getJSONFromSuccessResponse(response: Response, method: str
return null
}
try {
- return JSON.parse(text)
+ return timing ? timing.parseJSON(text) : JSON.parse(text)
} catch {
throw new ApiError(`Malformed JSON response ${requestContext()}`)
}
@@ -6143,8 +6150,8 @@ const api = {
},
queryStatus: {
- async get(queryId: string, showProgress: boolean): Promise<QueryStatusResponse> {
- return await new ApiRequest().queryStatus(queryId, showProgress).get()
+ async get(queryId: string, showProgress: boolean, options?: ApiMethodOptions): Promise<QueryStatusResponse> {
+ return await new ApiRequest().queryStatus(queryId, showProgress).get(options)
},
},
@@ -6746,11 +6753,13 @@ const api = {
/** Fetch data from specified URL. The result already is JSON-parsed. */
async get<T = any>(url: string, options?: ApiMethodOptions): Promise<T> {
- const res = await api.getResponse(url, options)
- return await getJSONFromSuccessResponse(res, 'GET', url)
+ return await withRequestTiming(options, async (timing) => {
+ const res = await api.getResponse(url, options, timing)
+ return await getJSONFromSuccessResponse(res, 'GET', url, timing)
+ })
},
- async getResponse(url: string, options?: ApiMethodOptions): Promise<Response> {
+ async getResponse(url: string, options?: ApiMethodOptions, timing?: ApiRequestTimingRecorder): Promise<Response> {
url = prepareUrl(url)
ensureProjectIdNotInvalid(url)
@@ -6762,7 +6771,8 @@ const api = {
}
return await handleFetch(url, 'GET', async () => {
- return fetch(url, {
+ timing?.startFetch(url)
+ const response = await fetch(url, {
signal: options?.signal,
headers: {
...objectClean(options?.headers ?? {}),
@@ -6771,6 +6781,8 @@ const api = {
...authHeaders,
},
})
+ timing?.recordHeaders(response)
+ return response
})
},
@@ -6811,17 +6823,25 @@ const api = {
},
async create<T = any, P = any>(url: string, data?: P, options?: ApiMethodOptions): Promise<T> {
- const res = await api.createResponse(url, data, options)
- return await getJSONFromSuccessResponse(res, 'POST', url)
+ return await withRequestTiming(options, async (timing) => {
+ const res = await api.createResponse(url, data, options, timing)
+ return await getJSONFromSuccessResponse(res, 'POST', url, timing)
+ })
},
- async createResponse(url: string, data?: any, options?: ApiMethodOptions): Promise<Response> {
+ async createResponse(
+ url: string,
+ data?: any,
+ options?: ApiMethodOptions,
+ timing?: ApiRequestTimingRecorder
+ ): Promise<Response> {
url = prepareUrl(url)
ensureProjectIdNotInvalid(url)
const isFormData = data instanceof FormData
- return await handleFetch(url, 'POST', async () =>
- fetch(url, {
+ return await handleFetch(url, 'POST', async () => {
+ timing?.startFetch(url)
+ const response = await fetch(url, {
method: 'POST',
headers: {
...objectClean(options?.headers ?? {}),
@@ -6833,7 +6853,9 @@ const api = {
body: data ? (isFormData ? data : JSON.stringify(data)) : undefined,
signal: options?.signal,
})
- )
+ timing?.recordHeaders(response)
+ return response
+ })
},
/**
@@ -7228,6 +7250,31 @@ async function isStaleSessionResponse(response: Response): Promise<boolean> {
}
}
+async function withRequestTiming<T>(
+ options: ApiMethodOptions | undefined,
+ request: (timing?: ApiRequestTimingRecorder) => Promise<T>
+): Promise<T> {
+ const onRequestTiming = options?.onRequestTiming
+ if (!onRequestTiming) {
+ return await request()
+ }
+ const timing = new ApiRequestTimingRecorder()
+ try {
+ return await request(timing)
+ } catch (error) {
+ if (error instanceof ApiError) {
+ timing.recordFailure(error)
+ }
+ throw error
+ } finally {
+ try {
+ onRequestTiming(timing.finish())
+ } catch {
+ // Observability must not change request success, failure, or cancellation.
+ }
+ }
+}
+
async function handleFetch(
url: string,
method: string,
diff --git a/frontend/src/lib/apiRequestTiming.test.ts b/frontend/src/lib/apiRequestTiming.test.ts
new file mode 100644
index 000000000000..1c5baa356aed
--- /dev/null
+++ b/frontend/src/lib/apiRequestTiming.test.ts
@@ -0,0 +1,254 @@
+import { ApiRequestTimingRecorder } from './apiRequestTiming'
+
+const REQUEST_ID = '11111111-1111-4111-8111-111111111111'
+const OTHER_REQUEST_ID = '22222222-2222-4222-8222-222222222222'
+
+describe('API request timing', () => {
+ const url = 'https://example.com/api/query/'
+ let now: number
+ let pending: PerformanceEntry[]
+ let observer: ResourceObserver
+ const originalObserver = Object.getOwnPropertyDescriptor(globalThis, 'PerformanceObserver')
+
+ class ResourceObserver {
+ observe = jest.fn()
+ disconnect = jest.fn()
+ takeRecords = jest.fn(() => pending.splice(0))
+
+ constructor(private readonly callback: PerformanceObserverCallback) {
+ observer = this
+ }
+
+ deliver(entries: PerformanceEntry[]): void {
+ this.callback(
+ { getEntries: () => entries } as PerformanceObserverEntryList,
+ this as unknown as PerformanceObserver
+ )
+ }
+ }
+
+ const resource = (requestId = REQUEST_ID): PerformanceResourceTiming =>
+ ({
+ name: url,
+ entryType: 'resource',
+ startTime: 10,
+ requestStart: 110,
+ responseStart: 210,
+ responseEnd: 310,
+ serverTiming: [{ name: 'request_id', description: requestId, duration: 0 }],
+ }) as unknown as PerformanceResourceTiming
+
+ beforeEach(() => {
+ now = 10
+ pending = []
+ Object.defineProperty(globalThis, 'PerformanceObserver', { configurable: true, value: ResourceObserver })
+ })
+
+ afterEach(() => {
+ if (originalObserver) {
+ Object.defineProperty(globalThis, 'PerformanceObserver', originalObserver)
+ } else {
+ delete (globalThis as { PerformanceObserver?: unknown }).PerformanceObserver
+ }
+ })
+
+ describe('API request resource timing attribution', () => {
+ let recorder: ApiRequestTimingRecorder
+
+ beforeEach(() => {
+ recorder = new ApiRequestTimingRecorder({ now: () => now })
+ now = 210
+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })
+ now = 320
+ })
+
+ it('uses the exact response ID even when another request to the same URL overlaps', () => {
+ pending = [resource(OTHER_REQUEST_ID), resource()]
+ expect(recorder.finish()).toEqual(
+ expect.objectContaining({
+ resource_timing_status: 'matched',
+ resource_pre_request_ms: 100,
+ resource_ttfb_ms: 100,
+ resource_download_ms: 100,
+ resource_callback_ms: 10,
+ })
+ )
+ })
+
+ it('does not associate an entry by URL or proximity when its ID differs', () => {
+ pending = [resource(OTHER_REQUEST_ID)]
+ expect(recorder.finish()).toEqual(expect.objectContaining({ resource_timing_status: 'not_found' }))
+ expect(recorder.finish()).not.toHaveProperty('resource_ttfb_ms')
+ })
+
+ it('rejects ambiguous matches instead of selecting the most recent request', () => {
+ pending = [resource(), resource()]
+ expect(recorder.finish()).toEqual(expect.objectContaining({ resource_timing_status: 'ambiguous' }))
+ expect(recorder.finish()).not.toHaveProperty('resource_pre_request_ms')
+ })
+
+ it.each([OTHER_REQUEST_ID, REQUEST_ID, 'invalid-id'])(
+ 'rejects a resource containing the response identity alongside %s',
+ (secondIdentity) => {
+ pending = [
+ {
+ ...resource(),
+ serverTiming: [
+ { name: 'request_id', description: REQUEST_ID, duration: 0 },
+ { name: 'request_id', description: secondIdentity, duration: 0 },
+ ],
+ } as unknown as PerformanceResourceTiming,
+ ]
+ const timing = recorder.finish()
+ expect(timing.resource_timing_status).toBe('ambiguous')
+ expect(timing.resource_ttfb_ms).toBeUndefined()
+ }
+ )
+
+ it('leaves unavailable cross-origin phases unknown', () => {
+ pending = [{ ...resource(), requestStart: 0, responseStart: 0 } as PerformanceResourceTiming]
+ expect(recorder.finish()).toEqual(expect.objectContaining({ resource_timing_status: 'restricted' }))
+ expect(recorder.finish()).not.toHaveProperty('resource_ttfb_ms')
+ })
+
+ it.each([
+ ['stale', { startTime: 1 }],
+ ['out-of-order', { responseEnd: 400 }],
+ ])('rejects %s timing entries', (_name, override) => {
+ pending = [{ ...resource(), ...override } as PerformanceResourceTiming]
+ expect(recorder.finish()).toEqual(expect.objectContaining({ resource_timing_status: 'invalid' }))
+ })
+
+ it('does not use malformed request IDs or upstream durations', () => {
+ recorder.recordHeaders({
+ status: 200,
+ url,
+ headers: new Headers({ 'X-Request-ID': 'not-a-uuid', 'X-Envoy-Upstream-Service-Time': '-10' }),
+ })
+ pending = [resource()]
+ expect(recorder.finish()).toEqual(expect.objectContaining({ resource_timing_status: 'missing_request_id' }))
+ expect(recorder.finish()).not.toHaveProperty('http_request_id')
+ expect(recorder.finish()).not.toHaveProperty('envoy_upstream_ms')
+ })
+
+ it('retains application timings when resource timing is unavailable', () => {
+ Object.defineProperty(globalThis, 'PerformanceObserver', {
+ configurable: true,
+ value: class {
+ constructor() {
+ throw new Error('Unavailable')
+ }
+ },
+ })
+ recorder = new ApiRequestTimingRecorder({ now: () => now })
+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })
+ expect(recorder.finish()).toEqual(
+ expect.objectContaining({
+ resource_timing_status: 'unavailable',
+ http_request_id: REQUEST_ID,
+ })
+ )
+ })
+
+ it.each(['abort', 'network'])('does not attribute an auth retry %s to the preceding response', (failure) => {
+ pending = [resource()]
+ recorder.startFetch(url)
+ if (failure === 'network') {
+ recorder.recordFailure({})
+ }
+ const timing = recorder.finish()
+ expect(timing.resource_timing_status).toBe('missing_request_id')
+ expect(timing.http_request_id).toBeUndefined()
+ expect(timing.http_status).toBeUndefined()
+ expect(timing.time_to_headers_ms).toBeUndefined()
+ expect(timing.resource_ttfb_ms).toBeUndefined()
+ })
+
+ it('rejects the preceding auth response when a successful retry reuses its request ID', () => {
+ pending = [resource()]
+ now = 500
+ recorder.startFetch(url)
+ now = 600
+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })
+ now = 700
+ const timing = recorder.finish()
+ expect(timing.request_total_ms).toBe(690)
+ expect(timing.time_to_headers_ms).toBe(590)
+ expect(timing.resource_timing_status).toBe('invalid')
+ expect(timing.resource_ttfb_ms).toBeUndefined()
+ })
+ })
+
+ describe('resource observation during an active request', () => {
+ it('attributes a completed request even when the global resource buffer has stopped recording', () => {
+ const recorder = new ApiRequestTimingRecorder({ now: () => now })
+ recorder.startFetch(url)
+ now = 210
+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })
+ // Resource callbacks may still be pending when the fetch body has completed.
+ pending = [resource()]
+ now = 320
+ expect(recorder.finish()).toEqual(
+ expect.objectContaining({ resource_timing_status: 'matched', resource_download_ms: 100 })
+ )
+ expect(observer.disconnect).toHaveBeenCalledTimes(1)
+ })
+
+ it('retains the matching identity when unrelated resources complete during the request', () => {
+ const recorder = new ApiRequestTimingRecorder({ now: () => now })
+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })
+ observer.deliver([resource(), ...Array.from({ length: 1000 }, () => resource(OTHER_REQUEST_ID))])
+ now = 320
+ expect(recorder.finish().resource_timing_status).toBe('matched')
+ })
+
+ it.each([
+ ['unidentifiable resources', 40, (): PerformanceResourceTiming => ({ ...resource(), serverTiming: [] })],
+ [
+ 'identified requests to other URLs',
+ 33,
+ (): PerformanceResourceTiming => ({
+ ...resource(OTHER_REQUEST_ID),
+ name: 'https://example.com/api/environments/1/query/other/',
+ }),
+ ],
+ ])('retains resource phases when %s complete before response headers', (_name, count, unrelated) => {
+ const recorder = new ApiRequestTimingRecorder({ now: () => now })
+ recorder.startFetch(url)
+ observer.deliver([...Array.from({ length: count }, unrelated), resource()])
+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })
+ now = 320
+ expect(recorder.finish()).toEqual(
+ expect.objectContaining({ resource_timing_status: 'matched', resource_ttfb_ms: 100 })
+ )
+ })
+
+ it('rejects duplicate identities observed within the request', () => {
+ const recorder = new ApiRequestTimingRecorder({ now: () => now })
+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })
+ observer.deliver([resource(), resource()])
+ now = 320
+ expect(recorder.finish().resource_timing_status).toBe('ambiguous')
+ })
+
+ it('leaves ownership unknown when bounded collection could have discarded a duplicate identity', () => {
+ const recorder = new ApiRequestTimingRecorder({ now: () => now })
+ recorder.startFetch(url)
+ observer.deliver([resource(), ...Array.from({ length: 64 }, () => resource(OTHER_REQUEST_ID)), resource()])
+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })
+ now = 320
+ const timing = recorder.finish()
+ expect(timing.resource_timing_status).toBe('overflow')
+ expect(timing.resource_ttfb_ms).toBeUndefined()
+ expect(observer.disconnect).toHaveBeenCalledTimes(1)
+ })
+
+ it('disconnects after a request fails before receiving an identity', () => {
+ const recorder = new ApiRequestTimingRecorder({ now: () => now })
+ recorder.startFetch(url)
+ recorder.recordFailure({})
+ expect(recorder.finish().resource_timing_status).toBe('missing_request_id')
+ expect(observer.disconnect).toHaveBeenCalledTimes(1)
+ })
+ })
+})
diff --git a/frontend/src/lib/apiRequestTiming.ts b/frontend/src/lib/apiRequestTiming.ts
new file mode 100644
index 000000000000..a7141f608518
--- /dev/null
+++ b/frontend/src/lib/apiRequestTiming.ts
@@ -0,0 +1,234 @@
+import { isUUIDLike } from 'lib/utils/guards'
+
+export interface ApiRequestTiming {
+ request_total_ms: number
+ // This includes API wrapper work and authentication retries before a successful response.
+ time_to_headers_ms?: number
+ body_read_ms?: number
+ json_parse_ms?: number
+ http_status?: number
+ http_request_id?: string
+ envoy_upstream_ms?: number
+ resource_timing_status:
+ | 'matched'
+ | 'missing_request_id'
+ | 'not_found'
+ | 'ambiguous'
+ | 'unavailable'
+ | 'restricted'
+ | 'invalid'
+ | 'overflow'
+ // Pre-request includes connection setup and browser waiting; it is not a queue measurement.
+ resource_pre_request_ms?: number
+ resource_ttfb_ms?: number
+ resource_download_ms?: number
+ // Includes browser scheduling, body decoding, and parsing after network completion.
+ resource_callback_ms?: number
+}
+
+const MAX_PENDING_RESOURCES = 32
+
+type ResourceTiming = Pick<
+ ApiRequestTiming,
+ | 'resource_timing_status'
+ | 'resource_pre_request_ms'
+ | 'resource_ttfb_ms'
+ | 'resource_download_ms'
+ | 'resource_callback_ms'
+>
+
+function serverRequestIds(entry: PerformanceResourceTiming): string[] {
+ return (entry.serverTiming ?? [])
+ .filter((metric) => metric.name === 'request_id')
+ .map((metric) => metric.description.toLowerCase())
+}
+
+/** One API request, with resource phases attributed only by a server-provided request ID. */
+export class ApiRequestTimingRecorder {
+ private readonly startedAt: number
+ private fetchStartedAt: number
+ private requestUrl?: string
+ private timing: Omit<ApiRequestTiming, 'request_total_ms' | 'resource_timing_status'> = {}
+ private responseUrl?: string
+ private observer?: PerformanceObserver
+ private resourceEntries: PerformanceResourceTiming[] = []
+ private resourceCollectionOverflowed = false
+
+ constructor(private readonly clock: Pick<Performance, 'now'> = performance) {
+ this.startedAt = clock.now()
+ this.fetchStartedAt = this.startedAt
+ try {
+ this.observer = new PerformanceObserver((list) => this.collectResources(list.getEntries()))
+ // Observe only new entries: the global buffer can fill during a long-lived app session.
+ this.observer.observe({ type: 'resource' })
+ } catch {
+ this.observer?.disconnect()
+ this.observer = undefined
+ }
+ }
+
+ startFetch(url: string): void {
+ // Authentication can retry internally; only the terminal fetch may supply response identity.
+ this.timing = {}
+ this.responseUrl = undefined
+ this.resourceEntries = []
+ this.resourceCollectionOverflowed = false
+ this.fetchStartedAt = this.clock.now()
+ try {
+ this.requestUrl = new URL(url, location.href).href
+ } catch {
+ this.requestUrl = undefined
+ }
+ }
+
+ recordHeaders(response: { status?: number; headers?: Headers; url?: string }): void {
+ this.timing.time_to_headers_ms = this.clock.now() - this.startedAt
+ this.recordResponseMetadata(response)
+ }
+
+ recordFailure(error: { status?: number; headers?: Headers }): void {
+ if (this.timing.http_status === undefined) {
+ this.recordResponseMetadata(error)
+ }
+ }
+
+ private recordResponseMetadata(response: { status?: number; headers?: Headers; url?: string }): void {
+ this.timing.http_status = response.status
+ const requestId = response.headers?.get('X-Request-ID')
+ if (requestId && isUUIDLike(requestId)) {
+ this.timing.http_request_id = requestId.toLowerCase()
+ } else {
+ delete this.timing.http_request_id
+ }
+ const upstream = response.headers?.get('X-Envoy-Upstream-Service-Time')
+ if (upstream && /^\d+(\.\d+)?$/.test(upstream) && Number.isFinite(Number(upstream))) {
+ this.timing.envoy_upstream_ms = Number(upstream)
+ } else {
+ delete this.timing.envoy_upstream_ms
+ }
+ this.responseUrl = response.url
+ if (this.timing.http_request_id && this.responseUrl) {
+ this.resourceEntries = this.resourceEntries.filter((entry) => this.isResponseEntry(entry)).slice(0, 2)
+ }
+ }
+
+ private isResponseEntry(entry: PerformanceResourceTiming): boolean {
+ return (
+ entry.name === this.responseUrl &&
+ serverRequestIds(entry).some((requestId) => requestId === this.timing.http_request_id)
+ )
+ }
+
+ private collectResources(entries: PerformanceEntry[]): void {
+ for (const entry of entries as PerformanceResourceTiming[]) {
+ if (entry.entryType !== 'resource') {
+ continue
+ }
+ if (this.timing.http_request_id && this.responseUrl) {
+ // Two candidates already prove ambiguity; unrelated resources need no retention.
+ if (this.resourceEntries.length < 2 && this.isResponseEntry(entry)) {
+ this.resourceEntries.push(entry)
+ }
+ } else if (this.isPendingRequestEntry(entry)) {
+ if (this.resourceEntries.length === MAX_PENDING_RESOURCES) {
+ this.resourceCollectionOverflowed = true
+ this.resourceEntries.shift()
+ }
+ this.resourceEntries.push(entry)
+ }
+ }
+ }
+
+ private isPendingRequestEntry(entry: PerformanceResourceTiming): boolean {
+ if (entry.name !== this.requestUrl || entry.startTime < this.fetchStartedAt - 1) {
+ return false
+ }
+ return serverRequestIds(entry).some(isUUIDLike)
+ }
+
+ async readBody(response: Response): Promise<string> {
+ const startedAt = this.clock.now()
+ try {
+ return await response.text()
+ } finally {
+ this.timing.body_read_ms = this.clock.now() - startedAt
+ }
+ }
+
+ parseJSON(text: string): unknown {
+ const startedAt = this.clock.now()
+ try {
+ return JSON.parse(text)
+ } finally {
+ this.timing.json_parse_ms = this.clock.now() - startedAt
+ }
+ }
+
+ finish(): ApiRequestTiming {
+ const finishedAt = this.clock.now()
+ const timing: ApiRequestTiming = {
+ ...this.timing,
+ request_total_ms: finishedAt - this.startedAt,
+ resource_timing_status: 'missing_request_id',
+ }
+ try {
+ // Fetch/body promises can resolve before the observer callback is delivered.
+ this.collectResources(this.observer?.takeRecords() ?? [])
+ if (timing.http_request_id) {
+ Object.assign(timing, this.resourceTiming(timing.http_request_id, finishedAt))
+ }
+ } catch {
+ timing.resource_timing_status = 'unavailable'
+ } finally {
+ this.observer?.disconnect()
+ }
+ return timing
+ }
+
+ private resourceTiming(requestId: string, finishedAt: number): ResourceTiming {
+ if (!this.observer) {
+ return { resource_timing_status: 'unavailable' }
+ }
+ if (this.resourceCollectionOverflowed) {
+ // Truncation may have hidden a duplicate identity; retained entries cannot prove ownership.
+ return { resource_timing_status: 'overflow' }
+ }
+ const matches = this.resourceEntries.filter((entry) => serverRequestIds(entry).includes(requestId))
+ if (matches.length !== 1) {
+ return { resource_timing_status: matches.length > 1 ? 'ambiguous' : 'not_found' }
+ }
+ const entry = matches[0]
+ const status = this.matchedEntryStatus(entry, finishedAt)
+ if (status !== 'matched') {
+ return { resource_timing_status: status }
+ }
+ return {
+ resource_timing_status: 'matched',
+ resource_pre_request_ms: entry.requestStart - entry.startTime,
+ resource_ttfb_ms: entry.responseStart - entry.requestStart,
+ resource_download_ms: entry.responseEnd - entry.responseStart,
+ resource_callback_ms: finishedAt - entry.responseEnd,
+ }
+ }
+
+ private matchedEntryStatus(
+ entry: PerformanceResourceTiming,
+ finishedAt: number
+ ): ApiRequestTiming['resource_timing_status'] {
+ const identities = serverRequestIds(entry)
+ if (identities.length !== 1 || !isUUIDLike(identities[0])) {
+ return 'ambiguous'
+ }
+ if (entry.requestStart === 0 || entry.responseStart === 0) {
+ return 'restricted'
+ }
+ const phases = [entry.startTime, entry.requestStart, entry.responseStart, entry.responseEnd, finishedAt]
+ if (
+ entry.startTime < this.fetchStartedAt - 1 ||
+ phases.some((phase, index) => !Number.isFinite(phase) || (index > 0 && phase < phases[index - 1]))
+ ) {
+ return 'invalid'
+ }
+ return 'matched'
+ }
+}
diff --git a/frontend/src/queries/query.test.ts b/frontend/src/queries/query.test.ts
index 15e9e3ccaeaf..2f37de08e80e 100644
--- a/frontend/src/queries/query.test.ts
+++ b/frontend/src/queries/query.test.ts
@@ -1,3 +1,4 @@
+import { HttpResponse } from 'msw'
import posthog from 'posthog-js'
import api, { ApiError } from 'lib/api'
@@ -110,7 +111,21 @@ describe('query', () => {
retry_wait_ms: expect.any(Number),
retry_status: null,
retry_recovered: false,
+ submit_request_timings: [
+ expect.objectContaining({
+ request_total_ms: expect.any(Number),
+ time_to_headers_ms: expect.any(Number),
+ body_read_ms: expect.any(Number),
+ json_parse_ms: expect.any(Number),
+ http_status: 200,
+ }),
+ ],
+ time_to_headers_ms: expect.any(Number),
+ body_read_ms: expect.any(Number),
+ json_parse_ms: expect.any(Number),
+ resource_timing_status: 'missing_request_id',
})
+ expect(queryCompletedCalls[0][1]).not.toHaveProperty('result_poll_timing')
})
it('emits a specific event on a HogQLQuery', async () => {
@@ -131,6 +146,87 @@ describe('query', () => {
})
})
+ it('keeps each retry timing on the existing completion event and flattens the final attempt', async () => {
+ const requestIds = ['11111111-1111-4111-8111-111111111111', '22222222-2222-4222-8222-222222222222']
+ let attempts = 0
+ useMocks({
+ post: {
+ '/api/environments/:team_id/query/:kind': () => {
+ const headers = { 'X-Request-ID': requestIds[attempts] }
+ return ++attempts === 1
+ ? HttpResponse.json({ detail: 'temporarily unavailable' }, { status: 502, headers })
+ : HttpResponse.json({ results: [] }, { headers })
+ },
+ },
+ })
+ const captureSpy = jest.spyOn(posthog, 'capture').mockClear()
+ const query: HogQLQuery = { kind: NodeKind.HogQLQuery, query: 'select 1' }
+
+ await expect(performQuery(query)).resolves.toMatchObject({ results: [] })
+
+ const completed = captureSpy.mock.calls.filter(([event]) => event === 'query completed')
+ expect(completed).toHaveLength(1)
+ expect(completed[0][1]).toMatchObject({
+ submit_attempts: 2,
+ submit_request_timings: [
+ expect.objectContaining({ http_status: 502, http_request_id: requestIds[0] }),
+ expect.objectContaining({ http_status: 200, http_request_id: requestIds[1] }),
+ ],
+ http_request_id: requestIds[1],
+ body_read_ms: expect.any(Number),
+ })
+ })
+
+ it('records only the poll that returns the completed async result', async () => {
+ const pollRequestIds = ['33333333-3333-4333-8333-333333333333', '44444444-4444-4444-8444-444444444444']
+ let polls = 0
+ useMocks({
+ post: {
+ '/api/environments/:team_id/query/:kind': () => [
+ 200,
+ { query_status: { id: 'async-query', complete: false } },
+ ],
+ },
+ get: {
+ '/api/environments/:team_id/query/:id': () => {
+ const complete = polls === 1
+ const headers = { 'X-Request-ID': pollRequestIds[polls++] }
+ return HttpResponse.json(
+ { query_status: { id: 'async-query', complete, results: complete ? { results: [] } : null } },
+ { headers }
+ )
+ },
+ },
+ })
+ jest.useFakeTimers()
+ const captureSpy = jest.spyOn(posthog, 'capture').mockClear()
+ const query: HogQLQuery = { kind: NodeKind.HogQLQuery, query: 'select 1' }
+
+ try {
+ const promise = performQuery(query, undefined, 'async')
+ await jest.advanceTimersByTimeAsync(1000)
+ await expect(promise).resolves.toEqual({ results: [] })
+ } finally {
+ jest.useRealTimers()
+ }
+
+ expect(polls).toBe(2)
+ const completed = captureSpy.mock.calls.filter(([event]) => event === 'query completed')
+ expect(completed).toHaveLength(1)
+ expect(completed[0][1]).toMatchObject({
+ result_poll_timing: expect.objectContaining({
+ http_status: 200,
+ http_request_id: pollRequestIds[1],
+ body_read_ms: expect.any(Number),
+ json_parse_ms: expect.any(Number),
+ }),
+ result_poll_http_request_id: pollRequestIds[1],
+ result_poll_body_read_ms: expect.any(Number),
+ result_poll_json_parse_ms: expect.any(Number),
+ })
+ expect(completed[0][1]?.submit_request_timings).toHaveLength(1)
+ })
+
it.each([
[true, 1],
[undefined, 0],
@@ -206,6 +302,11 @@ describe('query', () => {
duration: expect.any(Number),
error_status: 500,
error_code: null,
+ submit_request_timings: [
+ expect.objectContaining({ request_total_ms: expect.any(Number), http_status: 500 }),
+ ],
+ time_to_headers_ms: expect.any(Number),
+ resource_timing_status: 'missing_request_id',
})
// Raw error text must stay out of telemetry
expect(queryFailedCalls[0][1]).not.toHaveProperty('error_message')
diff --git a/frontend/src/queries/query.ts b/frontend/src/queries/query.ts
index abea2c35fc29..e1c94639b068 100644
--- a/frontend/src/queries/query.ts
+++ b/frontend/src/queries/query.ts
@@ -1,5 +1,6 @@
import api, { ApiMethodOptions, isAbortError } from 'lib/api'
import { ApiError, NetworkError } from 'lib/api-error'
+import { ApiRequestTiming } from 'lib/apiRequestTiming'
import posthog from 'lib/posthog-typed'
import { delay, retryWithBackoff } from 'lib/utils/async'
import { uuid } from 'lib/utils/dom'
@@ -108,6 +109,7 @@ const MANAGED_WAREHOUSE_UNAVAILABLE_CODE = 'managed_warehouse_connection_unavail
const TRANSIENT_SUBMIT_ATTEMPTS = 3
const TRANSIENT_SUBMIT_DELAY_MS = 600
const TRANSIENT_SUBMIT_RETRY_BUDGET_MS = 20_000
+const MAX_RECORDED_SUBMIT_TIMINGS = 6
type QuerySubmitTelemetry = {
client_query_id?: string
@@ -115,6 +117,33 @@ type QuerySubmitTelemetry = {
retry_wait_ms: number
retry_status: number | null
retry_recovered: boolean
+ submit_request_timings: ApiRequestTiming[]
+ result_poll_timing?: ApiRequestTiming
+}
+
+const FLATTENED_REQUEST_TIMING_FIELDS = [
+ 'http_request_id',
+ 'time_to_headers_ms',
+ 'body_read_ms',
+ 'json_parse_ms',
+ 'resource_timing_status',
+] as const
+
+function flattenRequestTiming(
+ timing: ApiRequestTiming | undefined,
+ prefix: '' | 'result_poll_'
+): Record<string, string | number | undefined> {
+ if (!timing) {
+ return {}
+ }
+ return Object.fromEntries(FLATTENED_REQUEST_TIMING_FIELDS.map((field) => [`${prefix}${field}`, timing[field]]))
+}
+
+function requestTimingProperties(telemetry: QuerySubmitTelemetry): Record<string, string | number | undefined> {
+ return {
+ ...flattenRequestTiming(telemetry.submit_request_timings.at(-1), ''),
+ ...flattenRequestTiming(telemetry.result_poll_timing, 'result_poll_'),
+ }
}
function capacityWaitMs(error: unknown): number | undefined {
@@ -199,10 +228,23 @@ export function queryExportContext<N extends DataNode>(
return { source: query }
}
+function annotatePollError(e: any, queryId: string): void {
+ // Parse error message to extract clean message and code if present
+ const parsed = parseErrorMessage(e.data?.query_status?.error_message ?? e.data?.detail ?? e.detail)
+ e.detail = parsed.message
+
+ // Prefer the structured code from QueryStatus over one parsed out of the message
+ e.code = e.data?.query_status?.error_code ?? e.data?.code ?? parsed.code ?? e.code
+
+ // Attach queryId to error for downstream error handling
+ e.queryId = queryId
+}
+
export async function pollForResults(
queryId: string,
methodOptions?: ApiMethodOptions,
- onPoll?: (response: QueryStatus) => void
+ onPoll?: (response: QueryStatus) => void,
+ onResultPollTiming?: (timing: ApiRequestTiming) => void
): Promise<QueryStatus> {
// A hidden or offline tab must not consume the polling deadline while it cannot poll.
let activeElapsedMs = 0
@@ -217,8 +259,22 @@ export async function pollForResults(
await waitForPollingReady(methodOptions?.signal)
try {
- const statusResponse = (await api.queryStatus.get(queryId, true)).query_status
+ let pollTiming: ApiRequestTiming | undefined
+ const statusResponse = (
+ await api.queryStatus.get(
+ queryId,
+ true,
+ onResultPollTiming && {
+ onRequestTiming: (timing) => {
+ pollTiming = timing
+ },
+ }
+ )
+ ).query_status
if (statusResponse.complete) {
+ if (pollTiming) {
+ onResultPollTiming?.(pollTiming)
+ }
return statusResponse
}
if (onPoll) {
@@ -228,16 +284,7 @@ export async function pollForResults(
if (e instanceof NetworkError && e.reason === 'offline') {
continue
}
- // Parse error message to extract clean message and code if present
- const parsed = parseErrorMessage(e.data?.query_status?.error_message ?? e.data?.detail ?? e.detail)
- e.detail = parsed.message
-
- // Prefer the structured code from QueryStatus over one parsed out of the message
- e.code = e.data?.query_status?.error_code ?? e.data?.code ?? parsed.code ?? e.code
-
- // Attach queryId to error for downstream error handling
- e.queryId = queryId
-
+ annotatePollError(e, queryId)
throw e
}
}
@@ -293,7 +340,14 @@ async function executeQuery<N extends DataNode>(
const requestStartedAt = performance.now()
try {
return await api.query(queryNode, {
- requestOptions: methodOptions,
+ requestOptions: {
+ ...methodOptions,
+ onRequestTiming: (timing) => {
+ if (submitTelemetry.submit_request_timings.length < MAX_RECORDED_SUBMIT_TIMINGS) {
+ submitTelemetry.submit_request_timings.push(timing)
+ }
+ },
+ },
clientQueryId,
refresh: refreshParam,
filtersOverride,
@@ -364,7 +418,9 @@ async function executeQuery<N extends DataNode>(
let statusResponse: QueryStatus
try {
- statusResponse = await pollForResults(queryId, methodOptions, setPollResponse)
+ statusResponse = await pollForResults(queryId, methodOptions, setPollResponse, (timing) => {
+ submitTelemetry.result_poll_timing = timing
+ })
} catch (e: any) {
// A backgrounded tab can outlive the query's status. Submit once more to recover its result.
//
@@ -419,6 +475,7 @@ export async function performQuery<N extends DataNode>(
retry_wait_ms: 0,
retry_status: null,
retry_recovered: false,
+ submit_request_timings: [],
}
const startTime = performance.now()
@@ -471,6 +528,7 @@ export async function performQuery<N extends DataNode>(
data_warehouse_source_types: warehouseSources.map((s) => s.source_type).filter(Boolean),
...logParams,
...submitTelemetry,
+ ...requestTimingProperties(submitTelemetry),
})
const historySources = warehouseSources.filter((s) => s.uses_history_table)
if (historySources.length > 0) {
@@ -498,6 +556,7 @@ export async function performQuery<N extends DataNode>(
uses_data_warehouse_source: queryUsesDataWarehouse(queryNode),
...logParams,
...submitTelemetry,
+ ...requestTimingProperties(submitTelemetry),
retry_recovered: false,
})
}
diff --git a/posthog/api/query.py b/posthog/api/query.py
index b50b1f6e8fa6..a12b0de4d68c 100644
--- a/posthog/api/query.py
+++ b/posthog/api/query.py
@@ -3,7 +3,7 @@ from time import perf_counter
from typing import Any, NoReturn
from django.core.cache import cache
-from django.http import JsonResponse
+from django.http import HttpRequest, JsonResponse
from django.http.response import HttpResponseBase
import orjson
@@ -56,7 +56,7 @@ from posthog.api.utils import action, is_async_query, is_insight_actors_options_
from posthog.api_queries_budget import get_request_query_cost, reset_request_query_cost
from posthog.clickhouse.client.execute_async import QueryNotFoundError, cancel_query, get_query_status
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
-from posthog.clickhouse.query_tagging import get_query_tag_value, get_query_tags, tag_queries
+from posthog.clickhouse.query_tagging import get_http_request_id, get_query_tag_value, get_query_tags, tag_queries
from posthog.constants import AvailableFeature
from posthog.errors import (
GENERIC_INTERNAL_CH_ERROR_MESSAGE,
@@ -65,7 +65,12 @@ from posthog.errors import (
internal_ch_error_user_message,
look_up_clickhouse_error_code_meta,
)
-from posthog.event_usage import EventSource, get_request_analytics_properties, report_user_or_team_action
+from posthog.event_usage import (
+ AnalyticsProps,
+ EventSource,
+ get_request_analytics_properties,
+ report_user_or_team_action,
+)
from posthog.exceptions_capture import capture_exception
from posthog.hogql_queries.apply_dashboard_filters import apply_dashboard_filters, apply_dashboard_variables
from posthog.hogql_queries.hogql_query_runner import HogQLQueryRunner
@@ -245,6 +250,23 @@ def required_scopes_for_query_payload(query: object) -> list[str] | None:
return None
+def add_query_response_identity_headers(response: Response) -> Response:
+ if http_request_id := get_http_request_id():
+ response["X-Request-ID"] = http_request_id
+ request_id_timing = f'request_id;desc="{http_request_id}"'
+ if existing_server_timing := response.get("Server-Timing"):
+ request_id_timing = f"{existing_server_timing}, {request_id_timing}"
+ response["Server-Timing"] = request_id_timing
+ return response
+
+
+def get_query_request_analytics_properties(request: HttpRequest | Request) -> AnalyticsProps:
+ analytics_props = get_request_analytics_properties(request)
+ if http_request_id := get_http_request_id():
+ analytics_props["http_request_id"] = http_request_id
+ return analytics_props
+
+
class QueryViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet):
# NOTE: Do we need to override the scopes for the "create"
scope_object = "query"
@@ -254,6 +276,9 @@ class QueryViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet)
scope_object_write_actions: list[str] = []
sharing_enabled_actions = ["retrieve"]
+ def finalize_response(self, request: Request, response: Response, *args: Any, **kwargs: Any) -> Response:
+ return add_query_response_identity_headers(super().finalize_response(request, response, *args, **kwargs))
+
def dangerously_get_required_scopes(self, request, view) -> list[str] | None:
if getattr(view, "action", None) != "create":
return None
@@ -322,7 +347,7 @@ class QueryViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet)
_mark_explicit_date_boundaries(query)
self._tag_client_query_id(client_query_id)
- analytics_props = get_request_analytics_properties(request)
+ analytics_props = get_query_request_analytics_properties(request)
query_dict = query.model_dump()
if data.limit_context == SchemaLimitContext.POSTHOG_AI:
diff --git a/posthog/api/test/test_query.py b/posthog/api/test/test_query.py
index aefc615d419e..12eb6eae6d87 100644
--- a/posthog/api/test/test_query.py
+++ b/posthog/api/test/test_query.py
@@ -93,6 +93,31 @@ class TestQueryTraceCorrelation(SimpleTestCase):
class TestQuery(ClickhouseTestMixin, APIBaseTest):
ENDPOINT = "query"
+ @patch("posthog.api.query.report_user_or_team_action")
+ @patch("posthog.api.query.process_query_model")
+ def test_query_response_identity_matches_query_analytics(self, mock_process_query_model, mock_report_action):
+ request_id = "01234567-89ab-4def-8123-456789abcdef"
+ client_query_id = "11111111-1111-4111-8111-111111111111"
+ mock_process_query_model.return_value = {"results": [[1]]}
+
+ response = self.client.post(
+ f"/api/environments/{self.team.id}/query/",
+ {
+ "query": {"kind": "HogQLQuery", "query": "select 1"},
+ "client_query_id": client_query_id,
+ },
+ HTTP_X_REQUEST_ID=request_id,
+ )
+
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+ self.assertEqual(response.json(), {"results": [[1]]})
+ self.assertEqual(response["X-Request-ID"], request_id)
+ self.assertIn(f'request_id;desc="{request_id}"', response["Server-Timing"])
+ self.assertEqual(mock_process_query_model.call_args.kwargs["analytics_props"]["http_request_id"], request_id)
+ self.assertEqual(mock_report_action.call_args.args[0], "query api response")
+ self.assertEqual(mock_report_action.call_args.args[1]["client_query_id"], client_query_id)
+ self.assertEqual(mock_report_action.call_args.kwargs["analytics_props"]["http_request_id"], request_id)
+
@parameterized.expand(
[
("ascii", "abc"),
diff --git a/posthog/clickhouse/query_tagging.py b/posthog/clickhouse/query_tagging.py
index 791ec6fedede..167f8f1506d8 100644
--- a/posthog/clickhouse/query_tagging.py
+++ b/posthog/clickhouse/query_tagging.py
@@ -690,6 +690,11 @@ def get_query_tag_value(key: str) -> Optional[Any]:
return None
+def get_http_request_id() -> str | None:
+ http_request_id = get_query_tag_value("http_request_id")
+ return str(http_request_id) if isinstance(http_request_id, uuid.UUID) else None
+
+
# Tag snapshots are isolated copy-on-write: every mutation goes through a top-level attribute
# assignment on a fresh shallow copy (see update/with_temporal/with_dagster), and nested tag
# objects are only ever replaced, never mutated in place. That makes shallow model_copy()
diff --git a/posthog/event_usage.py b/posthog/event_usage.py
index 988b81315ac6..f6fefab18065 100644
--- a/posthog/event_usage.py
+++ b/posthog/event_usage.py
@@ -433,6 +433,7 @@ AnalyticsProps = TypedDict(
"$session_id": NotRequired[str | None],
"was_impersonated": NotRequired[bool],
"access_method": NotRequired[str | None],
+ "http_request_id": NotRequired[str],
"user_agent": NotRequired[str | None],
"mcp_user_agent": NotRequired[str | None],
"mcp_client_name": NotRequired[str | None],
diff --git a/posthog/middleware.py b/posthog/middleware.py
index a8a95453f5bf..851ad203d8e7 100644
--- a/posthog/middleware.py
+++ b/posthog/middleware.py
@@ -565,11 +565,13 @@ class CHQueries:
user = cast(User, request.user)
- with suppress(Exception):
- if request_id := structlog.contextvars.get_contextvars().get("request_id"):
- tag_queries(http_request_id=uuid.UUID(request_id))
+ http_request_id: uuid.UUID | None = None
+ with suppress(ValueError):
+ if isinstance(request_id := structlog.contextvars.get_contextvars().get("request_id"), str):
+ http_request_id = uuid.UUID(request_id)
tag_queries(
+ http_request_id=http_request_id,
user_id=user.pk,
kind="request",
id=request.path,
@@ -649,7 +651,7 @@ class QueryTimeCountingMiddleware:
with connection.execute_wrapper(pg_query_counter), clickhouse_query_counter(ch_query_counter):
response: HttpResponse = self.get_response(request)
- response.headers["Server-Timing"] = self._construct_header(
+ server_timing = self._construct_header(
durations_ms={
"django": (time.perf_counter() - start_time) * 1000,
"pg": pg_query_counter.query_time_ms,
@@ -664,6 +666,9 @@ class QueryTimeCountingMiddleware:
"ch_slow": ch_query_counter.slow_count,
},
)
+ if existing_server_timing := response.headers.get("Server-Timing"):
+ server_timing = f"{existing_server_timing}, {server_timing}"
+ response.headers["Server-Timing"] = server_timing
return response
def _construct_header(self, durations_ms: dict[str, float], counts: dict[str, int]) -> str:
diff --git a/posthog/test/test_middleware.py b/posthog/test/test_middleware.py
index 3c019f27a76e..ffa25ff7be37 100644
--- a/posthog/test/test_middleware.py
+++ b/posthog/test/test_middleware.py
@@ -2908,6 +2908,81 @@ def test_chqueries_middleware_api_latency_records_on_raise():
assert sample_count() - before == 1.0, "exception path must still record latency at status_class=error"
+ [
+ ("success", "01234567-89ab-4def-8123-456789abcdef", 200, "01234567-89ab-4def-8123-456789abcdef"),
+ ("async", "0123456789AB4DEF8123456789ABCDEF", 202, "01234567-89ab-4def-8123-456789abcdef"),
+ ("auth_error", "01234567-89ab-4def-8123-456789abcdef", 401, "01234567-89ab-4def-8123-456789abcdef"),
+ ("invalid", '[email protected];desc="unsafe"', 400, None),
+ ("missing", None, 200, None),
+ ]
+)
+def test_chqueries_middleware_correlates_query_response(
+ name: str, request_id: str | None, status_code: int, expected_id: str | None
+) -> None:
+ from django.contrib.auth.models import AnonymousUser
+ from django.contrib.sessions.backends.signed_cookies import SessionStore
+ from django.http.response import HttpResponseBase
+ from django.urls import path
+
+ from rest_framework.request import Request
+ from rest_framework.response import Response
+
+ from posthog.api.query import QueryViewSet, get_query_request_analytics_properties
+ from posthog.event_usage import get_request_analytics_properties
+ from posthog.middleware import CHQueries
+
+ analytics_props: dict[str, object] = {}
+ view = QueryViewSet()
+ view.headers = {}
+
+ class QueryResponseRoutes:
+ urlpatterns = [path("api/projects/@current/query/", QueryViewSet.as_view({"post": "create"}))]
+
+ def get_response(request: HttpRequest) -> HttpResponseBase:
+ assert "http_request_id" not in get_request_analytics_properties(request)
+ analytics_props.update(get_query_request_analytics_properties(request))
+ response = Response(
+ {"results": []},
+ status=status_code,
+ headers={"Server-Timing": "existing;dur=5"},
+ )
+ return view.finalize_response(Request(request), response)
+
+ request = RequestFactory().get("/api/projects/@current/query/")
+ request.user = AnonymousUser()
+ request.session = SessionStore()
+ with (
+ override_settings(ROOT_URLCONF=QueryResponseRoutes),
+ structlog.contextvars.bound_contextvars(request_id=request_id),
+ ):
+ response = CHQueries(get_response)(request)
+
+ assert isinstance(response, Response)
+ assert response.status_code == status_code
+ assert response.data == {"results": []}
+ assert analytics_props.get("http_request_id") == expected_id
+ if expected_id:
+ assert response["X-Request-ID"] == expected_id
+ assert response["Server-Timing"] == f'existing;dur=5, request_id;desc="{expected_id}"'
+ else:
+ assert "X-Request-ID" not in response
+ assert response["Server-Timing"] == "existing;dur=5"
+
+
+@override_settings(CAPTURE_TIME_TO_SEE_DATA=True)
+def test_query_time_counting_middleware_preserves_request_identity() -> None:
+ from posthog.middleware import QueryTimeCountingMiddleware
+
+ request_id_timing = 'request_id;desc="01234567-89ab-4def-8123-456789abcdef"'
+ middleware = QueryTimeCountingMiddleware(
+ get_response=lambda request: HttpResponse(headers={"Server-Timing": request_id_timing})
+ )
+ response = middleware(RequestFactory().get("/api/projects/@current/insights/"))
+
+ assert response["Server-Timing"].startswith(f"{request_id_timing}, django;dur=")
+
+
def test_query_time_counting_middleware_emits_durations_in_milliseconds() -> None:
from posthog.middleware import QueryTimeCountingMiddleware
diff --git a/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py b/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py
index 70330bad3ec3..7841236b093f 100644
--- a/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py
+++ b/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py
@@ -1317,8 +1317,10 @@ class TestAccountsTableQueryAPI(APIBaseTest):
assert response.status_code == status.HTTP_400_BAD_REQUEST
- def test_accounts_table_query_endpoint_requires_account_scope_and_dispatches(self) -> None:
+ @patch("posthog.api.query.report_user_or_team_action")
+ def test_accounts_table_query_endpoint_requires_account_scope_and_dispatches(self, mock_report_action) -> None:
account = create_account(team_id=self.team.id, name="Acme")
+ request_id = "01234567-89ab-4def-8123-456789abcdef"
endpoint = f"/api/projects/{self.team.id}/accounts_table_query/"
accounts_query = AccountsTableQuery(columns=[], filters=[]).model_dump()
payload = {"query": accounts_query, "refresh": "force_blocking"}
@@ -1335,10 +1337,13 @@ class TestAccountsTableQueryAPI(APIBaseTest):
endpoint,
payload,
format="json",
- headers={"authorization": f"Bearer {self._token(['account:read'])}"},
+ headers={"authorization": f"Bearer {self._token(['account:read'])}", "x-request-id": request_id},
)
assert allowed.status_code == status.HTTP_200_OK, allowed.content
assert [row["id"] for row in allowed.json()["results"]] == [str(account.id)]
+ assert allowed["X-Request-ID"] == request_id
+ assert f'request_id;desc="{request_id}"' in allowed["Server-Timing"]
+ assert mock_report_action.call_args.kwargs["analytics_props"]["http_request_id"] == request_id
def test_accounts_table_query_endpoint_rejects_other_query_kinds(self) -> None:
response = self.client.post(
diff --git a/products/customer_analytics/backend/presentation/views/accounts_table_query.py b/products/customer_analytics/backend/presentation/views/accounts_table_query.py
index 1bb1c440859b..94c076fa9d03 100644
--- a/products/customer_analytics/backend/presentation/views/accounts_table_query.py
+++ b/products/customer_analytics/backend/presentation/views/accounts_table_query.py
@@ -18,7 +18,7 @@ from posthog.schema import (
from posthog.api.documentation import _FallbackSerializer
from posthog.api.mixins import PydanticModelMixin
-from posthog.api.query import QueryViewSet
+from posthog.api.query import QueryViewSet, add_query_response_identity_headers
from posthog.api.routing import TeamAndOrgViewSetMixin
@@ -62,6 +62,9 @@ class AccountsTableQueryViewSet(
_try_format_for_llm = QueryViewSet._try_format_for_llm
_validate_query_kind = QueryViewSet._validate_query_kind
+ def finalize_response(self, request: Request, response: Response, *args: Any, **kwargs: Any) -> Response:
+ return add_query_response_identity_headers(super().finalize_response(request, response, *args, **kwargs))
+
def dangerously_get_required_scopes(self, _request: Request, _view: Any) -> list[str] | None:
if self.action == "create":
return ["account:read"]