:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(observability): measure query request latency phases (#115652)
Vasco de Krijgeradc2bc23ae0c
diff --git a/docs/published/handbook/engineering/conventions/backend-coding.md b/docs/published/handbook/engineering/conventions/backend-coding.mdindex 885b2919b8f0..71491297f744 100644--- a/docs/published/handbook/engineering/conventions/backend-coding.md+++ b/docs/published/handbook/engineering/conventions/backend-coding.md@@ -11,6 +11,14 @@ A matching ID links a request to its processing span. It does not prove that theAn asynchronous query can continue in a separate trace after the submission request ends.Check request timing against browser performance metrics, and inspect child spans and self time before choosing a performance fix.+Query API responses expose a validated HTTP request UUID through `X-Request-ID` and the `Server-Timing` metric named `request_id`.+Query analytics include the same UUID as `http_request_id`.+The browser records response wait, body read, JSON parsing, and available resource phases on existing query outcome events.+Resource phases require one matching response UUID; missing, ambiguous, or truncated evidence remains explicit.+Cross-origin resource details also require `Timing-Allow-Origin`.+The pre-request phase includes connection setup and browser waiting; it does not measure server queueing.+Automatic capture covers `performQuery` submission POSTs and the status poll that returns a completed async result; earlier polls and rejected-response body processing remain outside this capture.+#### LoggingAs a general rule, we should have logs for every expected and unexpected actions of the application, using the appropriate _log level_.diff --git a/frontend/src/lib/api.test.ts b/frontend/src/lib/api.test.tsindex 2caeb931ee8b..8d820aa8306d 100644--- a/frontend/src/lib/api.test.ts+++ b/frontend/src/lib/api.test.ts@@ -4,6 +4,7 @@ import posthog from 'posthog-js'import api, { ApiConfig, ApiError, ApiRequest, NetworkError, ResponseBodyReadError } from 'lib/api'import { shouldReportApiFailure } from 'lib/api-error'import { apiStatusLogic } from 'lib/logic/apiStatusLogic'+import * as oauthClient from 'lib/oauth/oauthClient'import { NodeKind } from '~/queries/schema/schema-general'import { PropertyFilterType, PropertyOperator } from '~/types'@@ -187,6 +188,118 @@ describe('API helper', () => {})describe('query endpoints', () => {+ it('observes response phases without changing the query result', async () => {+ const requestId = '11111111-1111-4111-8111-111111111111'+ let clock = 10+ const now = jest.spyOn(performance, 'now').mockImplementation(() => clock)+ const observeTiming = jest.fn()+ fakeFetch.mockImplementation(async () => {+ clock = 110+ return {+ ok: true,+ status: 200,+ headers: new Headers({ 'X-Request-ID': requestId, 'X-Envoy-Upstream-Service-Time': '80' }),+ text: async () => {+ clock = 310+ return JSON.stringify(FAKE_FETCH_RESULT)+ },+ }+ })++ try {+ await expect(+ api.query(+ { kind: NodeKind.HogQLQuery, query: 'select 1' },+ { requestOptions: { onRequestTiming: observeTiming } }+ )+ ).resolves.toEqual(FAKE_FETCH_RESULT)+ expect(observeTiming).toHaveBeenCalledTimes(1)+ expect(observeTiming).toHaveBeenCalledWith(+ expect.objectContaining({+ request_total_ms: 300,+ time_to_headers_ms: 100,+ body_read_ms: 200,+ json_parse_ms: 0,+ http_status: 200,+ http_request_id: requestId,+ envoy_upstream_ms: 80,+ })+ )+ expect(observeTiming.mock.calls[0][0]).not.toHaveProperty('url')+ expect(observeTiming.mock.calls[0][0]).not.toHaveProperty('query')+ } finally {+ now.mockRestore()+ }+ })++ it('keeps successful queries successful when a timing observer throws', async () => {+ await expect(+ api.query(+ { kind: NodeKind.HogQLQuery, query: 'select 1' },+ {+ requestOptions: {+ onRequestTiming: () => {+ throw new Error('observer failed')+ },+ },+ }+ )+ ).resolves.toEqual(FAKE_FETCH_RESULT)+ })++ it('preserves a body-read cancellation when a timing observer throws', async () => {+ const abort = new DOMException('Cancelled', 'AbortError')+ fakeFetch.mockResolvedValue({+ ok: true,+ status: 200,+ text: () => Promise.reject(abort),+ })+ const observer = jest.fn(() => {+ throw new Error('observer failed')+ })++ await expect(+ api.query(+ { kind: NodeKind.HogQLQuery, query: 'select 1' },+ { requestOptions: { onRequestTiming: observer } }+ )+ ).rejects.toBe(abort)+ expect(observer).toHaveBeenCalledWith(+ expect.objectContaining({ http_status: 200, body_read_ms: expect.any(Number) })+ )+ })++ it('observes rejected response headers before the error body is consumed', async () => {+ let clock = 10+ const now = jest.spyOn(performance, 'now').mockImplementation(() => clock)+ const observer = jest.fn()+ fakeFetch.mockImplementation(async () => {+ clock = 110+ return {+ ok: false,+ status: 500,+ headers: new Headers({ 'X-Request-ID': '11111111-1111-4111-8111-111111111111' }),+ json: async () => {+ clock = 310+ return { detail: 'error' }+ },+ }+ })+ try {+ await expect(+ api.query(+ { kind: NodeKind.HogQLQuery, query: 'select 1' },+ { requestOptions: { onRequestTiming: observer } }+ )+ ).rejects.toMatchObject({ status: 500 })+ expect(observer).toHaveBeenCalledWith(+ expect.objectContaining({ request_total_ms: 300, time_to_headers_ms: 100, http_status: 500 })+ )+ } finally {+ now.mockRestore()+ }+ })+it('adds query kind to the query URL when present', async () => {await api.query({ kind: NodeKind.HogQLQuery, query: 'select 1' })@@ -348,6 +461,37 @@ describe('API helper', () => {window.localStorage.removeItem(OAUTH_SESSION_KEY)})+ it.each([new TypeError('Failed to fetch'), new DOMException('Cancelled', 'AbortError')])(+ 'clears the preceding response identity when an auth retry throws %s',+ async (failure) => {+ const refresh = jest.spyOn(oauthClient, 'refreshAccessToken').mockResolvedValue('refreshed-token')+ const observer = jest.fn()+ fakeFetch+ .mockResolvedValueOnce({+ ok: false,+ status: 401,+ headers: new Headers({ 'X-Request-ID': '11111111-1111-4111-8111-111111111111' }),+ })+ .mockRejectedValueOnce(failure)+ try {+ await expect(+ api.query(+ { kind: NodeKind.HogQLQuery, query: 'select 1' },+ { requestOptions: { onRequestTiming: observer } }+ )+ ).rejects.toBeInstanceOf(failure.name === 'AbortError' ? DOMException : NetworkError)+ expect(fakeFetch).toHaveBeenCalledTimes(2)+ expect(observer).toHaveBeenCalledTimes(1)+ expect(observer.mock.calls[0][0]).toMatchObject({ resource_timing_status: 'missing_request_id' })+ expect(observer.mock.calls[0][0].http_status).toBeUndefined()+ expect(observer.mock.calls[0][0].http_request_id).toBeUndefined()+ expect(observer.mock.calls[0][0].time_to_headers_ms).toBeUndefined()+ } finally {+ refresh.mockRestore()+ }+ }+ )+it('attaches the bearer token to requests routed to the OAuth backend host', async () => {await api.get('/api/projects/2/insights/')const [url, options] = fakeFetch.mock.calls[0]diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.tsindex ef114b1894f4..8360468589f3 100644--- a/frontend/src/lib/api.ts+++ b/frontend/src/lib/api.ts@@ -15,6 +15,7 @@ import {readableErrorMessage,ResponseBodyReadError,} from 'lib/api-error'+import { ApiRequestTiming, ApiRequestTimingRecorder } from 'lib/apiRequestTiming'import { ActivityLogProps } from 'lib/components/ActivityLog/ActivityLog'import { ActivityLogItem } from 'lib/components/ActivityLog/humanizeActivity'import { apiStatusLogic, awaitReauthentication, isUserActionInProgress } from 'lib/logic/apiStatusLogic'@@ -311,6 +312,7 @@ export interface ActivityLogPaginatedResponse<T> extends PaginatedResponse<T> {export interface ApiMethodOptions {signal?: AbortSignalheaders?: Record<string, any>+ onRequestTiming?: (timing: ApiRequestTiming) => void}export interface ApiUploadProgress {@@ -398,7 +400,12 @@ function apiErrorFallback(response: Response, method: string, url: string): stri* with unparsable content) throws `ResponseBodyReadError`, so it can be recognized as wire-level* noise and left out of error tracking.*/-export async function getJSONFromSuccessResponse(response: Response, method: string, url: string): Promise<any> {+export async function getJSONFromSuccessResponse(+ response: Response,+ method: string,+ url: string,+ timing?: ApiRequestTimingRecorder+): Promise<any> {const requestContext = (): string =>`[${method} ${new URL(url, location.origin).pathname}] (status ${response.status})`// A no-content response must not depend on reading its body: some engines (in our telemetry,@@ -408,7 +415,7 @@ export async function getJSONFromSuccessResponse(response: Response, method: str}let text: stringtry {- text = await response.text()+ text = timing ? await timing.readBody(response) : await response.text()} catch (error) {if (isAbortError(error)) {throw error@@ -432,7 +439,7 @@ export async function getJSONFromSuccessResponse(response: Response, method: strreturn null}try {- return JSON.parse(text)+ return timing ? timing.parseJSON(text) : JSON.parse(text)} catch {throw new ApiError(`Malformed JSON response ${requestContext()}`)}@@ -6143,8 +6150,8 @@ const api = {},queryStatus: {- async get(queryId: string, showProgress: boolean): Promise<QueryStatusResponse> {- return await new ApiRequest().queryStatus(queryId, showProgress).get()+ async get(queryId: string, showProgress: boolean, options?: ApiMethodOptions): Promise<QueryStatusResponse> {+ return await new ApiRequest().queryStatus(queryId, showProgress).get(options)},},@@ -6746,11 +6753,13 @@ const api = {/** Fetch data from specified URL. The result already is JSON-parsed. */async get<T = any>(url: string, options?: ApiMethodOptions): Promise<T> {- const res = await api.getResponse(url, options)- return await getJSONFromSuccessResponse(res, 'GET', url)+ return await withRequestTiming(options, async (timing) => {+ const res = await api.getResponse(url, options, timing)+ return await getJSONFromSuccessResponse(res, 'GET', url, timing)+ })},- async getResponse(url: string, options?: ApiMethodOptions): Promise<Response> {+ async getResponse(url: string, options?: ApiMethodOptions, timing?: ApiRequestTimingRecorder): Promise<Response> {url = prepareUrl(url)ensureProjectIdNotInvalid(url)@@ -6762,7 +6771,8 @@ const api = {}return await handleFetch(url, 'GET', async () => {- return fetch(url, {+ timing?.startFetch(url)+ const response = await fetch(url, {signal: options?.signal,headers: {...objectClean(options?.headers ?? {}),@@ -6771,6 +6781,8 @@ const api = {...authHeaders,},})+ timing?.recordHeaders(response)+ return response})},@@ -6811,17 +6823,25 @@ const api = {},async create<T = any, P = any>(url: string, data?: P, options?: ApiMethodOptions): Promise<T> {- const res = await api.createResponse(url, data, options)- return await getJSONFromSuccessResponse(res, 'POST', url)+ return await withRequestTiming(options, async (timing) => {+ const res = await api.createResponse(url, data, options, timing)+ return await getJSONFromSuccessResponse(res, 'POST', url, timing)+ })},- async createResponse(url: string, data?: any, options?: ApiMethodOptions): Promise<Response> {+ async createResponse(+ url: string,+ data?: any,+ options?: ApiMethodOptions,+ timing?: ApiRequestTimingRecorder+ ): Promise<Response> {url = prepareUrl(url)ensureProjectIdNotInvalid(url)const isFormData = data instanceof FormData- return await handleFetch(url, 'POST', async () =>- fetch(url, {+ return await handleFetch(url, 'POST', async () => {+ timing?.startFetch(url)+ const response = await fetch(url, {method: 'POST',headers: {...objectClean(options?.headers ?? {}),@@ -6833,7 +6853,9 @@ const api = {body: data ? (isFormData ? data : JSON.stringify(data)) : undefined,signal: options?.signal,})- )+ timing?.recordHeaders(response)+ return response+ })},/**@@ -7228,6 +7250,31 @@ async function isStaleSessionResponse(response: Response): Promise<boolean> {}}+async function withRequestTiming<T>(+ options: ApiMethodOptions | undefined,+ request: (timing?: ApiRequestTimingRecorder) => Promise<T>+): Promise<T> {+ const onRequestTiming = options?.onRequestTiming+ if (!onRequestTiming) {+ return await request()+ }+ const timing = new ApiRequestTimingRecorder()+ try {+ return await request(timing)+ } catch (error) {+ if (error instanceof ApiError) {+ timing.recordFailure(error)+ }+ throw error+ } finally {+ try {+ onRequestTiming(timing.finish())+ } catch {+ // Observability must not change request success, failure, or cancellation.+ }+ }+}+async function handleFetch(url: string,method: string,diff --git a/frontend/src/lib/apiRequestTiming.test.ts b/frontend/src/lib/apiRequestTiming.test.tsnew file mode 100644index 000000000000..1c5baa356aed--- /dev/null+++ b/frontend/src/lib/apiRequestTiming.test.ts@@ -0,0 +1,254 @@+import { ApiRequestTimingRecorder } from './apiRequestTiming'++const REQUEST_ID = '11111111-1111-4111-8111-111111111111'+const OTHER_REQUEST_ID = '22222222-2222-4222-8222-222222222222'++describe('API request timing', () => {+ const url = 'https://example.com/api/query/'+ let now: number+ let pending: PerformanceEntry[]+ let observer: ResourceObserver+ const originalObserver = Object.getOwnPropertyDescriptor(globalThis, 'PerformanceObserver')++ class ResourceObserver {+ observe = jest.fn()+ disconnect = jest.fn()+ takeRecords = jest.fn(() => pending.splice(0))++ constructor(private readonly callback: PerformanceObserverCallback) {+ observer = this+ }++ deliver(entries: PerformanceEntry[]): void {+ this.callback(+ { getEntries: () => entries } as PerformanceObserverEntryList,+ this as unknown as PerformanceObserver+ )+ }+ }++ const resource = (requestId = REQUEST_ID): PerformanceResourceTiming =>+ ({+ name: url,+ entryType: 'resource',+ startTime: 10,+ requestStart: 110,+ responseStart: 210,+ responseEnd: 310,+ serverTiming: [{ name: 'request_id', description: requestId, duration: 0 }],+ }) as unknown as PerformanceResourceTiming++ beforeEach(() => {+ now = 10+ pending = []+ Object.defineProperty(globalThis, 'PerformanceObserver', { configurable: true, value: ResourceObserver })+ })++ afterEach(() => {+ if (originalObserver) {+ Object.defineProperty(globalThis, 'PerformanceObserver', originalObserver)+ } else {+ delete (globalThis as { PerformanceObserver?: unknown }).PerformanceObserver+ }+ })++ describe('API request resource timing attribution', () => {+ let recorder: ApiRequestTimingRecorder++ beforeEach(() => {+ recorder = new ApiRequestTimingRecorder({ now: () => now })+ now = 210+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })+ now = 320+ })++ it('uses the exact response ID even when another request to the same URL overlaps', () => {+ pending = [resource(OTHER_REQUEST_ID), resource()]+ expect(recorder.finish()).toEqual(+ expect.objectContaining({+ resource_timing_status: 'matched',+ resource_pre_request_ms: 100,+ resource_ttfb_ms: 100,+ resource_download_ms: 100,+ resource_callback_ms: 10,+ })+ )+ })++ it('does not associate an entry by URL or proximity when its ID differs', () => {+ pending = [resource(OTHER_REQUEST_ID)]+ expect(recorder.finish()).toEqual(expect.objectContaining({ resource_timing_status: 'not_found' }))+ expect(recorder.finish()).not.toHaveProperty('resource_ttfb_ms')+ })++ it('rejects ambiguous matches instead of selecting the most recent request', () => {+ pending = [resource(), resource()]+ expect(recorder.finish()).toEqual(expect.objectContaining({ resource_timing_status: 'ambiguous' }))+ expect(recorder.finish()).not.toHaveProperty('resource_pre_request_ms')+ })++ it.each([OTHER_REQUEST_ID, REQUEST_ID, 'invalid-id'])(+ 'rejects a resource containing the response identity alongside %s',+ (secondIdentity) => {+ pending = [+ {+ ...resource(),+ serverTiming: [+ { name: 'request_id', description: REQUEST_ID, duration: 0 },+ { name: 'request_id', description: secondIdentity, duration: 0 },+ ],+ } as unknown as PerformanceResourceTiming,+ ]+ const timing = recorder.finish()+ expect(timing.resource_timing_status).toBe('ambiguous')+ expect(timing.resource_ttfb_ms).toBeUndefined()+ }+ )++ it('leaves unavailable cross-origin phases unknown', () => {+ pending = [{ ...resource(), requestStart: 0, responseStart: 0 } as PerformanceResourceTiming]+ expect(recorder.finish()).toEqual(expect.objectContaining({ resource_timing_status: 'restricted' }))+ expect(recorder.finish()).not.toHaveProperty('resource_ttfb_ms')+ })++ it.each([+ ['stale', { startTime: 1 }],+ ['out-of-order', { responseEnd: 400 }],+ ])('rejects %s timing entries', (_name, override) => {+ pending = [{ ...resource(), ...override } as PerformanceResourceTiming]+ expect(recorder.finish()).toEqual(expect.objectContaining({ resource_timing_status: 'invalid' }))+ })++ it('does not use malformed request IDs or upstream durations', () => {+ recorder.recordHeaders({+ status: 200,+ url,+ headers: new Headers({ 'X-Request-ID': 'not-a-uuid', 'X-Envoy-Upstream-Service-Time': '-10' }),+ })+ pending = [resource()]+ expect(recorder.finish()).toEqual(expect.objectContaining({ resource_timing_status: 'missing_request_id' }))+ expect(recorder.finish()).not.toHaveProperty('http_request_id')+ expect(recorder.finish()).not.toHaveProperty('envoy_upstream_ms')+ })++ it('retains application timings when resource timing is unavailable', () => {+ Object.defineProperty(globalThis, 'PerformanceObserver', {+ configurable: true,+ value: class {+ constructor() {+ throw new Error('Unavailable')+ }+ },+ })+ recorder = new ApiRequestTimingRecorder({ now: () => now })+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })+ expect(recorder.finish()).toEqual(+ expect.objectContaining({+ resource_timing_status: 'unavailable',+ http_request_id: REQUEST_ID,+ })+ )+ })++ it.each(['abort', 'network'])('does not attribute an auth retry %s to the preceding response', (failure) => {+ pending = [resource()]+ recorder.startFetch(url)+ if (failure === 'network') {+ recorder.recordFailure({})+ }+ const timing = recorder.finish()+ expect(timing.resource_timing_status).toBe('missing_request_id')+ expect(timing.http_request_id).toBeUndefined()+ expect(timing.http_status).toBeUndefined()+ expect(timing.time_to_headers_ms).toBeUndefined()+ expect(timing.resource_ttfb_ms).toBeUndefined()+ })++ it('rejects the preceding auth response when a successful retry reuses its request ID', () => {+ pending = [resource()]+ now = 500+ recorder.startFetch(url)+ now = 600+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })+ now = 700+ const timing = recorder.finish()+ expect(timing.request_total_ms).toBe(690)+ expect(timing.time_to_headers_ms).toBe(590)+ expect(timing.resource_timing_status).toBe('invalid')+ expect(timing.resource_ttfb_ms).toBeUndefined()+ })+ })++ describe('resource observation during an active request', () => {+ it('attributes a completed request even when the global resource buffer has stopped recording', () => {+ const recorder = new ApiRequestTimingRecorder({ now: () => now })+ recorder.startFetch(url)+ now = 210+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })+ // Resource callbacks may still be pending when the fetch body has completed.+ pending = [resource()]+ now = 320+ expect(recorder.finish()).toEqual(+ expect.objectContaining({ resource_timing_status: 'matched', resource_download_ms: 100 })+ )+ expect(observer.disconnect).toHaveBeenCalledTimes(1)+ })++ it('retains the matching identity when unrelated resources complete during the request', () => {+ const recorder = new ApiRequestTimingRecorder({ now: () => now })+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })+ observer.deliver([resource(), ...Array.from({ length: 1000 }, () => resource(OTHER_REQUEST_ID))])+ now = 320+ expect(recorder.finish().resource_timing_status).toBe('matched')+ })++ it.each([+ ['unidentifiable resources', 40, (): PerformanceResourceTiming => ({ ...resource(), serverTiming: [] })],+ [+ 'identified requests to other URLs',+ 33,+ (): PerformanceResourceTiming => ({+ ...resource(OTHER_REQUEST_ID),+ name: 'https://example.com/api/environments/1/query/other/',+ }),+ ],+ ])('retains resource phases when %s complete before response headers', (_name, count, unrelated) => {+ const recorder = new ApiRequestTimingRecorder({ now: () => now })+ recorder.startFetch(url)+ observer.deliver([...Array.from({ length: count }, unrelated), resource()])+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })+ now = 320+ expect(recorder.finish()).toEqual(+ expect.objectContaining({ resource_timing_status: 'matched', resource_ttfb_ms: 100 })+ )+ })++ it('rejects duplicate identities observed within the request', () => {+ const recorder = new ApiRequestTimingRecorder({ now: () => now })+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })+ observer.deliver([resource(), resource()])+ now = 320+ expect(recorder.finish().resource_timing_status).toBe('ambiguous')+ })++ it('leaves ownership unknown when bounded collection could have discarded a duplicate identity', () => {+ const recorder = new ApiRequestTimingRecorder({ now: () => now })+ recorder.startFetch(url)+ observer.deliver([resource(), ...Array.from({ length: 64 }, () => resource(OTHER_REQUEST_ID)), resource()])+ recorder.recordHeaders({ status: 200, url, headers: new Headers({ 'X-Request-ID': REQUEST_ID }) })+ now = 320+ const timing = recorder.finish()+ expect(timing.resource_timing_status).toBe('overflow')+ expect(timing.resource_ttfb_ms).toBeUndefined()+ expect(observer.disconnect).toHaveBeenCalledTimes(1)+ })++ it('disconnects after a request fails before receiving an identity', () => {+ const recorder = new ApiRequestTimingRecorder({ now: () => now })+ recorder.startFetch(url)+ recorder.recordFailure({})+ expect(recorder.finish().resource_timing_status).toBe('missing_request_id')+ expect(observer.disconnect).toHaveBeenCalledTimes(1)+ })+ })+})diff --git a/frontend/src/lib/apiRequestTiming.ts b/frontend/src/lib/apiRequestTiming.tsnew file mode 100644index 000000000000..a7141f608518--- /dev/null+++ b/frontend/src/lib/apiRequestTiming.ts@@ -0,0 +1,234 @@+import { isUUIDLike } from 'lib/utils/guards'++export interface ApiRequestTiming {+ request_total_ms: number+ // This includes API wrapper work and authentication retries before a successful response.+ time_to_headers_ms?: number+ body_read_ms?: number+ json_parse_ms?: number+ http_status?: number+ http_request_id?: string+ envoy_upstream_ms?: number+ resource_timing_status:+ | 'matched'+ | 'missing_request_id'+ | 'not_found'+ | 'ambiguous'+ | 'unavailable'+ | 'restricted'+ | 'invalid'+ | 'overflow'+ // Pre-request includes connection setup and browser waiting; it is not a queue measurement.+ resource_pre_request_ms?: number+ resource_ttfb_ms?: number+ resource_download_ms?: number+ // Includes browser scheduling, body decoding, and parsing after network completion.+ resource_callback_ms?: number+}++const MAX_PENDING_RESOURCES = 32++type ResourceTiming = Pick<+ ApiRequestTiming,+ | 'resource_timing_status'+ | 'resource_pre_request_ms'+ | 'resource_ttfb_ms'+ | 'resource_download_ms'+ | 'resource_callback_ms'+>++function serverRequestIds(entry: PerformanceResourceTiming): string[] {+ return (entry.serverTiming ?? [])+ .filter((metric) => metric.name === 'request_id')+ .map((metric) => metric.description.toLowerCase())+}++/** One API request, with resource phases attributed only by a server-provided request ID. */+export class ApiRequestTimingRecorder {+ private readonly startedAt: number+ private fetchStartedAt: number+ private requestUrl?: string+ private timing: Omit<ApiRequestTiming, 'request_total_ms' | 'resource_timing_status'> = {}+ private responseUrl?: string+ private observer?: PerformanceObserver+ private resourceEntries: PerformanceResourceTiming[] = []+ private resourceCollectionOverflowed = false++ constructor(private readonly clock: Pick<Performance, 'now'> = performance) {+ this.startedAt = clock.now()+ this.fetchStartedAt = this.startedAt+ try {+ this.observer = new PerformanceObserver((list) => this.collectResources(list.getEntries()))+ // Observe only new entries: the global buffer can fill during a long-lived app session.+ this.observer.observe({ type: 'resource' })+ } catch {+ this.observer?.disconnect()+ this.observer = undefined+ }+ }++ startFetch(url: string): void {+ // Authentication can retry internally; only the terminal fetch may supply response identity.+ this.timing = {}+ this.responseUrl = undefined+ this.resourceEntries = []+ this.resourceCollectionOverflowed = false+ this.fetchStartedAt = this.clock.now()+ try {+ this.requestUrl = new URL(url, location.href).href+ } catch {+ this.requestUrl = undefined+ }+ }++ recordHeaders(response: { status?: number; headers?: Headers; url?: string }): void {+ this.timing.time_to_headers_ms = this.clock.now() - this.startedAt+ this.recordResponseMetadata(response)+ }++ recordFailure(error: { status?: number; headers?: Headers }): void {+ if (this.timing.http_status === undefined) {+ this.recordResponseMetadata(error)+ }+ }++ private recordResponseMetadata(response: { status?: number; headers?: Headers; url?: string }): void {+ this.timing.http_status = response.status+ const requestId = response.headers?.get('X-Request-ID')+ if (requestId && isUUIDLike(requestId)) {+ this.timing.http_request_id = requestId.toLowerCase()+ } else {+ delete this.timing.http_request_id+ }+ const upstream = response.headers?.get('X-Envoy-Upstream-Service-Time')+ if (upstream && /^\d+(\.\d+)?$/.test(upstream) && Number.isFinite(Number(upstream))) {+ this.timing.envoy_upstream_ms = Number(upstream)+ } else {+ delete this.timing.envoy_upstream_ms+ }+ this.responseUrl = response.url+ if (this.timing.http_request_id && this.responseUrl) {+ this.resourceEntries = this.resourceEntries.filter((entry) => this.isResponseEntry(entry)).slice(0, 2)+ }+ }++ private isResponseEntry(entry: PerformanceResourceTiming): boolean {+ return (+ entry.name === this.responseUrl &&+ serverRequestIds(entry).some((requestId) => requestId === this.timing.http_request_id)+ )+ }++ private collectResources(entries: PerformanceEntry[]): void {+ for (const entry of entries as PerformanceResourceTiming[]) {+ if (entry.entryType !== 'resource') {+ continue+ }+ if (this.timing.http_request_id && this.responseUrl) {+ // Two candidates already prove ambiguity; unrelated resources need no retention.+ if (this.resourceEntries.length < 2 && this.isResponseEntry(entry)) {+ this.resourceEntries.push(entry)+ }+ } else if (this.isPendingRequestEntry(entry)) {+ if (this.resourceEntries.length === MAX_PENDING_RESOURCES) {+ this.resourceCollectionOverflowed = true+ this.resourceEntries.shift()+ }+ this.resourceEntries.push(entry)+ }+ }+ }++ private isPendingRequestEntry(entry: PerformanceResourceTiming): boolean {+ if (entry.name !== this.requestUrl || entry.startTime < this.fetchStartedAt - 1) {+ return false+ }+ return serverRequestIds(entry).some(isUUIDLike)+ }++ async readBody(response: Response): Promise<string> {+ const startedAt = this.clock.now()+ try {+ return await response.text()+ } finally {+ this.timing.body_read_ms = this.clock.now() - startedAt+ }+ }++ parseJSON(text: string): unknown {+ const startedAt = this.clock.now()+ try {+ return JSON.parse(text)+ } finally {+ this.timing.json_parse_ms = this.clock.now() - startedAt+ }+ }++ finish(): ApiRequestTiming {+ const finishedAt = this.clock.now()+ const timing: ApiRequestTiming = {+ ...this.timing,+ request_total_ms: finishedAt - this.startedAt,+ resource_timing_status: 'missing_request_id',+ }+ try {+ // Fetch/body promises can resolve before the observer callback is delivered.+ this.collectResources(this.observer?.takeRecords() ?? [])+ if (timing.http_request_id) {+ Object.assign(timing, this.resourceTiming(timing.http_request_id, finishedAt))+ }+ } catch {+ timing.resource_timing_status = 'unavailable'+ } finally {+ this.observer?.disconnect()+ }+ return timing+ }++ private resourceTiming(requestId: string, finishedAt: number): ResourceTiming {+ if (!this.observer) {+ return { resource_timing_status: 'unavailable' }+ }+ if (this.resourceCollectionOverflowed) {+ // Truncation may have hidden a duplicate identity; retained entries cannot prove ownership.+ return { resource_timing_status: 'overflow' }+ }+ const matches = this.resourceEntries.filter((entry) => serverRequestIds(entry).includes(requestId))+ if (matches.length !== 1) {+ return { resource_timing_status: matches.length > 1 ? 'ambiguous' : 'not_found' }+ }+ const entry = matches[0]+ const status = this.matchedEntryStatus(entry, finishedAt)+ if (status !== 'matched') {+ return { resource_timing_status: status }+ }+ return {+ resource_timing_status: 'matched',+ resource_pre_request_ms: entry.requestStart - entry.startTime,+ resource_ttfb_ms: entry.responseStart - entry.requestStart,+ resource_download_ms: entry.responseEnd - entry.responseStart,+ resource_callback_ms: finishedAt - entry.responseEnd,+ }+ }++ private matchedEntryStatus(+ entry: PerformanceResourceTiming,+ finishedAt: number+ ): ApiRequestTiming['resource_timing_status'] {+ const identities = serverRequestIds(entry)+ if (identities.length !== 1 || !isUUIDLike(identities[0])) {+ return 'ambiguous'+ }+ if (entry.requestStart === 0 || entry.responseStart === 0) {+ return 'restricted'+ }+ const phases = [entry.startTime, entry.requestStart, entry.responseStart, entry.responseEnd, finishedAt]+ if (+ entry.startTime < this.fetchStartedAt - 1 ||+ phases.some((phase, index) => !Number.isFinite(phase) || (index > 0 && phase < phases[index - 1]))+ ) {+ return 'invalid'+ }+ return 'matched'+ }+}diff --git a/frontend/src/queries/query.test.ts b/frontend/src/queries/query.test.tsindex 15e9e3ccaeaf..2f37de08e80e 100644--- a/frontend/src/queries/query.test.ts+++ b/frontend/src/queries/query.test.ts@@ -1,3 +1,4 @@+import { HttpResponse } from 'msw'import posthog from 'posthog-js'import api, { ApiError } from 'lib/api'@@ -110,7 +111,21 @@ describe('query', () => {retry_wait_ms: expect.any(Number),retry_status: null,retry_recovered: false,+ submit_request_timings: [+ expect.objectContaining({+ request_total_ms: expect.any(Number),+ time_to_headers_ms: expect.any(Number),+ body_read_ms: expect.any(Number),+ json_parse_ms: expect.any(Number),+ http_status: 200,+ }),+ ],+ time_to_headers_ms: expect.any(Number),+ body_read_ms: expect.any(Number),+ json_parse_ms: expect.any(Number),+ resource_timing_status: 'missing_request_id',})+ expect(queryCompletedCalls[0][1]).not.toHaveProperty('result_poll_timing')})it('emits a specific event on a HogQLQuery', async () => {@@ -131,6 +146,87 @@ describe('query', () => {})})+ it('keeps each retry timing on the existing completion event and flattens the final attempt', async () => {+ const requestIds = ['11111111-1111-4111-8111-111111111111', '22222222-2222-4222-8222-222222222222']+ let attempts = 0+ useMocks({+ post: {+ '/api/environments/:team_id/query/:kind': () => {+ const headers = { 'X-Request-ID': requestIds[attempts] }+ return ++attempts === 1+ ? HttpResponse.json({ detail: 'temporarily unavailable' }, { status: 502, headers })+ : HttpResponse.json({ results: [] }, { headers })+ },+ },+ })+ const captureSpy = jest.spyOn(posthog, 'capture').mockClear()+ const query: HogQLQuery = { kind: NodeKind.HogQLQuery, query: 'select 1' }++ await expect(performQuery(query)).resolves.toMatchObject({ results: [] })++ const completed = captureSpy.mock.calls.filter(([event]) => event === 'query completed')+ expect(completed).toHaveLength(1)+ expect(completed[0][1]).toMatchObject({+ submit_attempts: 2,+ submit_request_timings: [+ expect.objectContaining({ http_status: 502, http_request_id: requestIds[0] }),+ expect.objectContaining({ http_status: 200, http_request_id: requestIds[1] }),+ ],+ http_request_id: requestIds[1],+ body_read_ms: expect.any(Number),+ })+ })++ it('records only the poll that returns the completed async result', async () => {+ const pollRequestIds = ['33333333-3333-4333-8333-333333333333', '44444444-4444-4444-8444-444444444444']+ let polls = 0+ useMocks({+ post: {+ '/api/environments/:team_id/query/:kind': () => [+ 200,+ { query_status: { id: 'async-query', complete: false } },+ ],+ },+ get: {+ '/api/environments/:team_id/query/:id': () => {+ const complete = polls === 1+ const headers = { 'X-Request-ID': pollRequestIds[polls++] }+ return HttpResponse.json(+ { query_status: { id: 'async-query', complete, results: complete ? { results: [] } : null } },+ { headers }+ )+ },+ },+ })+ jest.useFakeTimers()+ const captureSpy = jest.spyOn(posthog, 'capture').mockClear()+ const query: HogQLQuery = { kind: NodeKind.HogQLQuery, query: 'select 1' }++ try {+ const promise = performQuery(query, undefined, 'async')+ await jest.advanceTimersByTimeAsync(1000)+ await expect(promise).resolves.toEqual({ results: [] })+ } finally {+ jest.useRealTimers()+ }++ expect(polls).toBe(2)+ const completed = captureSpy.mock.calls.filter(([event]) => event === 'query completed')+ expect(completed).toHaveLength(1)+ expect(completed[0][1]).toMatchObject({+ result_poll_timing: expect.objectContaining({+ http_status: 200,+ http_request_id: pollRequestIds[1],+ body_read_ms: expect.any(Number),+ json_parse_ms: expect.any(Number),+ }),+ result_poll_http_request_id: pollRequestIds[1],+ result_poll_body_read_ms: expect.any(Number),+ result_poll_json_parse_ms: expect.any(Number),+ })+ expect(completed[0][1]?.submit_request_timings).toHaveLength(1)+ })+it.each([[true, 1],[undefined, 0],@@ -206,6 +302,11 @@ describe('query', () => {duration: expect.any(Number),error_status: 500,error_code: null,+ submit_request_timings: [+ expect.objectContaining({ request_total_ms: expect.any(Number), http_status: 500 }),+ ],+ time_to_headers_ms: expect.any(Number),+ resource_timing_status: 'missing_request_id',})// Raw error text must stay out of telemetryexpect(queryFailedCalls[0][1]).not.toHaveProperty('error_message')diff --git a/frontend/src/queries/query.ts b/frontend/src/queries/query.tsindex abea2c35fc29..e1c94639b068 100644--- a/frontend/src/queries/query.ts+++ b/frontend/src/queries/query.ts@@ -1,5 +1,6 @@import api, { ApiMethodOptions, isAbortError } from 'lib/api'import { ApiError, NetworkError } from 'lib/api-error'+import { ApiRequestTiming } from 'lib/apiRequestTiming'import posthog from 'lib/posthog-typed'import { delay, retryWithBackoff } from 'lib/utils/async'import { uuid } from 'lib/utils/dom'@@ -108,6 +109,7 @@ const MANAGED_WAREHOUSE_UNAVAILABLE_CODE = 'managed_warehouse_connection_unavailconst TRANSIENT_SUBMIT_ATTEMPTS = 3const TRANSIENT_SUBMIT_DELAY_MS = 600const TRANSIENT_SUBMIT_RETRY_BUDGET_MS = 20_000+const MAX_RECORDED_SUBMIT_TIMINGS = 6type QuerySubmitTelemetry = {client_query_id?: string@@ -115,6 +117,33 @@ type QuerySubmitTelemetry = {retry_wait_ms: numberretry_status: number | nullretry_recovered: boolean+ submit_request_timings: ApiRequestTiming[]+ result_poll_timing?: ApiRequestTiming+}++const FLATTENED_REQUEST_TIMING_FIELDS = [+ 'http_request_id',+ 'time_to_headers_ms',+ 'body_read_ms',+ 'json_parse_ms',+ 'resource_timing_status',+] as const++function flattenRequestTiming(+ timing: ApiRequestTiming | undefined,+ prefix: '' | 'result_poll_'+): Record<string, string | number | undefined> {+ if (!timing) {+ return {}+ }+ return Object.fromEntries(FLATTENED_REQUEST_TIMING_FIELDS.map((field) => [`${prefix}${field}`, timing[field]]))+}++function requestTimingProperties(telemetry: QuerySubmitTelemetry): Record<string, string | number | undefined> {+ return {+ ...flattenRequestTiming(telemetry.submit_request_timings.at(-1), ''),+ ...flattenRequestTiming(telemetry.result_poll_timing, 'result_poll_'),+ }}function capacityWaitMs(error: unknown): number | undefined {@@ -199,10 +228,23 @@ export function queryExportContext<N extends DataNode>(return { source: query }}+function annotatePollError(e: any, queryId: string): void {+ // Parse error message to extract clean message and code if present+ const parsed = parseErrorMessage(e.data?.query_status?.error_message ?? e.data?.detail ?? e.detail)+ e.detail = parsed.message++ // Prefer the structured code from QueryStatus over one parsed out of the message+ e.code = e.data?.query_status?.error_code ?? e.data?.code ?? parsed.code ?? e.code++ // Attach queryId to error for downstream error handling+ e.queryId = queryId+}+export async function pollForResults(queryId: string,methodOptions?: ApiMethodOptions,- onPoll?: (response: QueryStatus) => void+ onPoll?: (response: QueryStatus) => void,+ onResultPollTiming?: (timing: ApiRequestTiming) => void): Promise<QueryStatus> {// A hidden or offline tab must not consume the polling deadline while it cannot poll.let activeElapsedMs = 0@@ -217,8 +259,22 @@ export async function pollForResults(await waitForPollingReady(methodOptions?.signal)try {- const statusResponse = (await api.queryStatus.get(queryId, true)).query_status+ let pollTiming: ApiRequestTiming | undefined+ const statusResponse = (+ await api.queryStatus.get(+ queryId,+ true,+ onResultPollTiming && {+ onRequestTiming: (timing) => {+ pollTiming = timing+ },+ }+ )+ ).query_statusif (statusResponse.complete) {+ if (pollTiming) {+ onResultPollTiming?.(pollTiming)+ }return statusResponse}if (onPoll) {@@ -228,16 +284,7 @@ export async function pollForResults(if (e instanceof NetworkError && e.reason === 'offline') {continue}- // Parse error message to extract clean message and code if present- const parsed = parseErrorMessage(e.data?.query_status?.error_message ?? e.data?.detail ?? e.detail)- e.detail = parsed.message-- // Prefer the structured code from QueryStatus over one parsed out of the message- e.code = e.data?.query_status?.error_code ?? e.data?.code ?? parsed.code ?? e.code-- // Attach queryId to error for downstream error handling- e.queryId = queryId-+ annotatePollError(e, queryId)throw e}}@@ -293,7 +340,14 @@ async function executeQuery<N extends DataNode>(const requestStartedAt = performance.now()try {return await api.query(queryNode, {- requestOptions: methodOptions,+ requestOptions: {+ ...methodOptions,+ onRequestTiming: (timing) => {+ if (submitTelemetry.submit_request_timings.length < MAX_RECORDED_SUBMIT_TIMINGS) {+ submitTelemetry.submit_request_timings.push(timing)+ }+ },+ },clientQueryId,refresh: refreshParam,filtersOverride,@@ -364,7 +418,9 @@ async function executeQuery<N extends DataNode>(let statusResponse: QueryStatustry {- statusResponse = await pollForResults(queryId, methodOptions, setPollResponse)+ statusResponse = await pollForResults(queryId, methodOptions, setPollResponse, (timing) => {+ submitTelemetry.result_poll_timing = timing+ })} catch (e: any) {// A backgrounded tab can outlive the query's status. Submit once more to recover its result.//@@ -419,6 +475,7 @@ export async function performQuery<N extends DataNode>(retry_wait_ms: 0,retry_status: null,retry_recovered: false,+ submit_request_timings: [],}const startTime = performance.now()@@ -471,6 +528,7 @@ export async function performQuery<N extends DataNode>(data_warehouse_source_types: warehouseSources.map((s) => s.source_type).filter(Boolean),...logParams,...submitTelemetry,+ ...requestTimingProperties(submitTelemetry),})const historySources = warehouseSources.filter((s) => s.uses_history_table)if (historySources.length > 0) {@@ -498,6 +556,7 @@ export async function performQuery<N extends DataNode>(uses_data_warehouse_source: queryUsesDataWarehouse(queryNode),...logParams,...submitTelemetry,+ ...requestTimingProperties(submitTelemetry),retry_recovered: false,})}diff --git a/posthog/api/query.py b/posthog/api/query.pyindex b50b1f6e8fa6..a12b0de4d68c 100644--- a/posthog/api/query.py+++ b/posthog/api/query.py@@ -3,7 +3,7 @@ from time import perf_counterfrom typing import Any, NoReturnfrom django.core.cache import cache-from django.http import JsonResponse+from django.http import HttpRequest, JsonResponsefrom django.http.response import HttpResponseBaseimport orjson@@ -56,7 +56,7 @@ from posthog.api.utils import action, is_async_query, is_insight_actors_options_from posthog.api_queries_budget import get_request_query_cost, reset_request_query_costfrom posthog.clickhouse.client.execute_async import QueryNotFoundError, cancel_query, get_query_statusfrom posthog.clickhouse.client.limit import ConcurrencyLimitExceeded-from posthog.clickhouse.query_tagging import get_query_tag_value, get_query_tags, tag_queries+from posthog.clickhouse.query_tagging import get_http_request_id, get_query_tag_value, get_query_tags, tag_queriesfrom posthog.constants import AvailableFeaturefrom posthog.errors import (GENERIC_INTERNAL_CH_ERROR_MESSAGE,@@ -65,7 +65,12 @@ from posthog.errors import (internal_ch_error_user_message,look_up_clickhouse_error_code_meta,)-from posthog.event_usage import EventSource, get_request_analytics_properties, report_user_or_team_action+from posthog.event_usage import (+ AnalyticsProps,+ EventSource,+ get_request_analytics_properties,+ report_user_or_team_action,+)from posthog.exceptions_capture import capture_exceptionfrom posthog.hogql_queries.apply_dashboard_filters import apply_dashboard_filters, apply_dashboard_variablesfrom posthog.hogql_queries.hogql_query_runner import HogQLQueryRunner@@ -245,6 +250,23 @@ def required_scopes_for_query_payload(query: object) -> list[str] | None:return None+def add_query_response_identity_headers(response: Response) -> Response:+ if http_request_id := get_http_request_id():+ response["X-Request-ID"] = http_request_id+ request_id_timing = f'request_id;desc="{http_request_id}"'+ if existing_server_timing := response.get("Server-Timing"):+ request_id_timing = f"{existing_server_timing}, {request_id_timing}"+ response["Server-Timing"] = request_id_timing+ return response+++def get_query_request_analytics_properties(request: HttpRequest | Request) -> AnalyticsProps:+ analytics_props = get_request_analytics_properties(request)+ if http_request_id := get_http_request_id():+ analytics_props["http_request_id"] = http_request_id+ return analytics_props++class QueryViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet):# NOTE: Do we need to override the scopes for the "create"scope_object = "query"@@ -254,6 +276,9 @@ class QueryViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet)scope_object_write_actions: list[str] = []sharing_enabled_actions = ["retrieve"]+ def finalize_response(self, request: Request, response: Response, *args: Any, **kwargs: Any) -> Response:+ return add_query_response_identity_headers(super().finalize_response(request, response, *args, **kwargs))+def dangerously_get_required_scopes(self, request, view) -> list[str] | None:if getattr(view, "action", None) != "create":return None@@ -322,7 +347,7 @@ class QueryViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet)_mark_explicit_date_boundaries(query)self._tag_client_query_id(client_query_id)- analytics_props = get_request_analytics_properties(request)+ analytics_props = get_query_request_analytics_properties(request)query_dict = query.model_dump()if data.limit_context == SchemaLimitContext.POSTHOG_AI:diff --git a/posthog/api/test/test_query.py b/posthog/api/test/test_query.pyindex aefc615d419e..12eb6eae6d87 100644--- a/posthog/api/test/test_query.py+++ b/posthog/api/test/test_query.py@@ -93,6 +93,31 @@ class TestQueryTraceCorrelation(SimpleTestCase):class TestQuery(ClickhouseTestMixin, APIBaseTest):ENDPOINT = "query"+ @patch("posthog.api.query.report_user_or_team_action")+ @patch("posthog.api.query.process_query_model")+ def test_query_response_identity_matches_query_analytics(self, mock_process_query_model, mock_report_action):+ request_id = "01234567-89ab-4def-8123-456789abcdef"+ client_query_id = "11111111-1111-4111-8111-111111111111"+ mock_process_query_model.return_value = {"results": [[1]]}++ response = self.client.post(+ f"/api/environments/{self.team.id}/query/",+ {+ "query": {"kind": "HogQLQuery", "query": "select 1"},+ "client_query_id": client_query_id,+ },+ HTTP_X_REQUEST_ID=request_id,+ )++ self.assertEqual(response.status_code, status.HTTP_200_OK)+ self.assertEqual(response.json(), {"results": [[1]]})+ self.assertEqual(response["X-Request-ID"], request_id)+ self.assertIn(f'request_id;desc="{request_id}"', response["Server-Timing"])+ self.assertEqual(mock_process_query_model.call_args.kwargs["analytics_props"]["http_request_id"], request_id)+ self.assertEqual(mock_report_action.call_args.args[0], "query api response")+ self.assertEqual(mock_report_action.call_args.args[1]["client_query_id"], client_query_id)+ self.assertEqual(mock_report_action.call_args.kwargs["analytics_props"]["http_request_id"], request_id)+@parameterized.expand([("ascii", "abc"),diff --git a/posthog/clickhouse/query_tagging.py b/posthog/clickhouse/query_tagging.pyindex 791ec6fedede..167f8f1506d8 100644--- a/posthog/clickhouse/query_tagging.py+++ b/posthog/clickhouse/query_tagging.py@@ -690,6 +690,11 @@ def get_query_tag_value(key: str) -> Optional[Any]:return None+def get_http_request_id() -> str | None:+ http_request_id = get_query_tag_value("http_request_id")+ return str(http_request_id) if isinstance(http_request_id, uuid.UUID) else None++# Tag snapshots are isolated copy-on-write: every mutation goes through a top-level attribute# assignment on a fresh shallow copy (see update/with_temporal/with_dagster), and nested tag# objects are only ever replaced, never mutated in place. That makes shallow model_copy()diff --git a/posthog/event_usage.py b/posthog/event_usage.pyindex 988b81315ac6..f6fefab18065 100644--- a/posthog/event_usage.py+++ b/posthog/event_usage.py@@ -433,6 +433,7 @@ AnalyticsProps = TypedDict("$session_id": NotRequired[str | None],"was_impersonated": NotRequired[bool],"access_method": NotRequired[str | None],+ "http_request_id": NotRequired[str],"user_agent": NotRequired[str | None],"mcp_user_agent": NotRequired[str | None],"mcp_client_name": NotRequired[str | None],diff --git a/posthog/middleware.py b/posthog/middleware.pyindex a8a95453f5bf..851ad203d8e7 100644--- a/posthog/middleware.py+++ b/posthog/middleware.py@@ -565,11 +565,13 @@ class CHQueries:user = cast(User, request.user)- with suppress(Exception):- if request_id := structlog.contextvars.get_contextvars().get("request_id"):- tag_queries(http_request_id=uuid.UUID(request_id))+ http_request_id: uuid.UUID | None = None+ with suppress(ValueError):+ if isinstance(request_id := structlog.contextvars.get_contextvars().get("request_id"), str):+ http_request_id = uuid.UUID(request_id)tag_queries(+ http_request_id=http_request_id,user_id=user.pk,kind="request",id=request.path,@@ -649,7 +651,7 @@ class QueryTimeCountingMiddleware:with connection.execute_wrapper(pg_query_counter), clickhouse_query_counter(ch_query_counter):response: HttpResponse = self.get_response(request)- response.headers["Server-Timing"] = self._construct_header(+ server_timing = self._construct_header(durations_ms={"django": (time.perf_counter() - start_time) * 1000,"pg": pg_query_counter.query_time_ms,@@ -664,6 +666,9 @@ class QueryTimeCountingMiddleware:"ch_slow": ch_query_counter.slow_count,},)+ if existing_server_timing := response.headers.get("Server-Timing"):+ server_timing = f"{existing_server_timing}, {server_timing}"+ response.headers["Server-Timing"] = server_timingreturn responsedef _construct_header(self, durations_ms: dict[str, float], counts: dict[str, int]) -> str:diff --git a/posthog/test/test_middleware.py b/posthog/test/test_middleware.pyindex 3c019f27a76e..ffa25ff7be37 100644--- a/posthog/test/test_middleware.py+++ b/posthog/test/test_middleware.py@@ -2908,6 +2908,81 @@ def test_chqueries_middleware_api_latency_records_on_raise():assert sample_count() - before == 1.0, "exception path must still record latency at status_class=error"+ [+ ("success", "01234567-89ab-4def-8123-456789abcdef", 200, "01234567-89ab-4def-8123-456789abcdef"),+ ("async", "0123456789AB4DEF8123456789ABCDEF", 202, "01234567-89ab-4def-8123-456789abcdef"),+ ("auth_error", "01234567-89ab-4def-8123-456789abcdef", 401, "01234567-89ab-4def-8123-456789abcdef"),+ ("invalid", '[email protected];desc="unsafe"', 400, None),+ ("missing", None, 200, None),+ ]+)+def test_chqueries_middleware_correlates_query_response(+ name: str, request_id: str | None, status_code: int, expected_id: str | None+) -> None:+ from django.contrib.auth.models import AnonymousUser+ from django.contrib.sessions.backends.signed_cookies import SessionStore+ from django.http.response import HttpResponseBase+ from django.urls import path++ from rest_framework.request import Request+ from rest_framework.response import Response++ from posthog.api.query import QueryViewSet, get_query_request_analytics_properties+ from posthog.event_usage import get_request_analytics_properties+ from posthog.middleware import CHQueries++ analytics_props: dict[str, object] = {}+ view = QueryViewSet()+ view.headers = {}++ class QueryResponseRoutes:+ urlpatterns = [path("api/projects/@current/query/", QueryViewSet.as_view({"post": "create"}))]++ def get_response(request: HttpRequest) -> HttpResponseBase:+ assert "http_request_id" not in get_request_analytics_properties(request)+ analytics_props.update(get_query_request_analytics_properties(request))+ response = Response(+ {"results": []},+ status=status_code,+ headers={"Server-Timing": "existing;dur=5"},+ )+ return view.finalize_response(Request(request), response)++ request = RequestFactory().get("/api/projects/@current/query/")+ request.user = AnonymousUser()+ request.session = SessionStore()+ with (+ override_settings(ROOT_URLCONF=QueryResponseRoutes),+ structlog.contextvars.bound_contextvars(request_id=request_id),+ ):+ response = CHQueries(get_response)(request)++ assert isinstance(response, Response)+ assert response.status_code == status_code+ assert response.data == {"results": []}+ assert analytics_props.get("http_request_id") == expected_id+ if expected_id:+ assert response["X-Request-ID"] == expected_id+ assert response["Server-Timing"] == f'existing;dur=5, request_id;desc="{expected_id}"'+ else:+ assert "X-Request-ID" not in response+ assert response["Server-Timing"] == "existing;dur=5"+++@override_settings(CAPTURE_TIME_TO_SEE_DATA=True)+def test_query_time_counting_middleware_preserves_request_identity() -> None:+ from posthog.middleware import QueryTimeCountingMiddleware++ request_id_timing = 'request_id;desc="01234567-89ab-4def-8123-456789abcdef"'+ middleware = QueryTimeCountingMiddleware(+ get_response=lambda request: HttpResponse(headers={"Server-Timing": request_id_timing})+ )+ response = middleware(RequestFactory().get("/api/projects/@current/insights/"))++ assert response["Server-Timing"].startswith(f"{request_id_timing}, django;dur=")++def test_query_time_counting_middleware_emits_durations_in_milliseconds() -> None:from posthog.middleware import QueryTimeCountingMiddlewarediff --git a/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py b/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.pyindex 70330bad3ec3..7841236b093f 100644--- a/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py+++ b/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py@@ -1317,8 +1317,10 @@ class TestAccountsTableQueryAPI(APIBaseTest):assert response.status_code == status.HTTP_400_BAD_REQUEST- def test_accounts_table_query_endpoint_requires_account_scope_and_dispatches(self) -> None:+ @patch("posthog.api.query.report_user_or_team_action")+ def test_accounts_table_query_endpoint_requires_account_scope_and_dispatches(self, mock_report_action) -> None:account = create_account(team_id=self.team.id, name="Acme")+ request_id = "01234567-89ab-4def-8123-456789abcdef"endpoint = f"/api/projects/{self.team.id}/accounts_table_query/"accounts_query = AccountsTableQuery(columns=[], filters=[]).model_dump()payload = {"query": accounts_query, "refresh": "force_blocking"}@@ -1335,10 +1337,13 @@ class TestAccountsTableQueryAPI(APIBaseTest):endpoint,payload,format="json",- headers={"authorization": f"Bearer {self._token(['account:read'])}"},+ headers={"authorization": f"Bearer {self._token(['account:read'])}", "x-request-id": request_id},)assert allowed.status_code == status.HTTP_200_OK, allowed.contentassert [row["id"] for row in allowed.json()["results"]] == [str(account.id)]+ assert allowed["X-Request-ID"] == request_id+ assert f'request_id;desc="{request_id}"' in allowed["Server-Timing"]+ assert mock_report_action.call_args.kwargs["analytics_props"]["http_request_id"] == request_iddef test_accounts_table_query_endpoint_rejects_other_query_kinds(self) -> None:response = self.client.post(diff --git a/products/customer_analytics/backend/presentation/views/accounts_table_query.py b/products/customer_analytics/backend/presentation/views/accounts_table_query.pyindex 1bb1c440859b..94c076fa9d03 100644--- a/products/customer_analytics/backend/presentation/views/accounts_table_query.py+++ b/products/customer_analytics/backend/presentation/views/accounts_table_query.py@@ -18,7 +18,7 @@ from posthog.schema import (from posthog.api.documentation import _FallbackSerializerfrom posthog.api.mixins import PydanticModelMixin-from posthog.api.query import QueryViewSet+from posthog.api.query import QueryViewSet, add_query_response_identity_headersfrom posthog.api.routing import TeamAndOrgViewSetMixin@@ -62,6 +62,9 @@ class AccountsTableQueryViewSet(_try_format_for_llm = QueryViewSet._try_format_for_llm_validate_query_kind = QueryViewSet._validate_query_kind+ def finalize_response(self, request: Request, response: Response, *args: Any, **kwargs: Any) -> Response:+ return add_query_response_identity_headers(super().finalize_response(request, response, *args, **kwargs))+def dangerously_get_required_scopes(self, _request: Request, _view: Any) -> list[str] | None:if self.action == "create":return ["account:read"]