:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cdp): cap the response body a function fetch reads (#121966)
Daniel Marchukbb64eb1d5443
diff --git a/nodejs/src/cdp/cdp-e2e.serial.test.ts b/nodejs/src/cdp/cdp-e2e.serial.test.tsindex ad0d206a0511..b29004a886ef 100644--- a/nodejs/src/cdp/cdp-e2e.serial.test.ts+++ b/nodejs/src/cdp/cdp-e2e.serial.test.ts@@ -197,6 +197,7 @@ describe('CDP Consumer loop', () => {"headers": {"Authorization": "Bearer super-secret-token",},+ "maxResponseBytes": 5242880,"method": "POST",},]diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-plugins.test.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-plugins.test.tsindex 0cd0a382dfad..3a9e9b48d86d 100644--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-plugins.test.ts+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-plugins.test.ts@@ -155,6 +155,7 @@ describe('CdpCyclotronWorkerPlugins', () => {"Authorization": "Bearer 1234567890","Content-Type": "application/json",},+ "maxResponseBytes": 5242880,"method": "POST",},]@@ -169,6 +170,7 @@ describe('CdpCyclotronWorkerPlugins', () => {"Authorization": "Bearer 1234567890","Content-Type": "application/json",},+ "maxResponseBytes": 5242880,"method": "POST",},]diff --git a/nodejs/src/cdp/services/hog-executor-async.service.ts b/nodejs/src/cdp/services/hog-executor-async.service.tsindex 2c8878bc4bc2..c914db4cba34 100644--- a/nodejs/src/cdp/services/hog-executor-async.service.ts+++ b/nodejs/src/cdp/services/hog-executor-async.service.ts@@ -18,7 +18,12 @@ import type {} from '../types'import { createAddLogFunction, destinationE2eLagMsSummary } from '../utils'import { resolveAwsSigV4Credentials, signAwsRequest } from '../utils/aws-sigv4'-import { cdpTrackedFetch, fetchErrorDetail, isFetchResponseRetriable } from '../utils/cdp-fetch'+import {+ MAX_FETCH_RESPONSE_BYTES,+ cdpTrackedFetch,+ fetchErrorDetail,+ isFetchResponseRetriable,+} from '../utils/cdp-fetch'import { createInvocationResult } from '../utils/invocation-utils'import { isNonFailureStatus } from '../utils/non-failure-status-codes'import { ScopedServiceJwt } from '../utils/scoped-service-jwt'@@ -579,7 +584,12 @@ export class HogExecutorAsyncService {}}} catch (e) {- addLog('error', `Failed to parse response body: ${e.message}`)+ addLog(+ 'error',+ e.name === 'ResponseTooLargeError'+ ? `The response body is larger than ${MAX_FETCH_RESPONSE_BYTES / 1024 / 1024} MB, so it was not read.`+ : `Failed to parse response body: ${e.message}`+ )body = undefined}diff --git a/nodejs/src/cdp/services/legacy-plugin-executor.service.test.ts b/nodejs/src/cdp/services/legacy-plugin-executor.service.test.tsindex 55f24f637f07..ac6fe565a54a 100644--- a/nodejs/src/cdp/services/legacy-plugin-executor.service.test.ts+++ b/nodejs/src/cdp/services/legacy-plugin-executor.service.test.ts@@ -255,6 +255,7 @@ describe('LegacyPluginExecutorService', () => {"Authorization": "Basic MTIzNDU2Nzg5MDpjaW8tdG9rZW4=","User-Agent": "PostHog Customer.io App",},+ "maxResponseBytes": 5242880,"method": "GET",},],@@ -267,6 +268,7 @@ describe('LegacyPluginExecutorService', () => {"Content-Type": "application/json","User-Agent": "PostHog Customer.io App",},+ "maxResponseBytes": 5242880,"method": "PUT",},],@@ -279,6 +281,7 @@ describe('LegacyPluginExecutorService', () => {"Content-Type": "application/json","User-Agent": "PostHog Customer.io App",},+ "maxResponseBytes": 5242880,"method": "POST",},],diff --git a/nodejs/src/cdp/services/native-destination-executor.service.test.ts b/nodejs/src/cdp/services/native-destination-executor.service.test.tsindex 64a21c4a7480..114aa5b05c0b 100644--- a/nodejs/src/cdp/services/native-destination-executor.service.test.ts+++ b/nodejs/src/cdp/services/native-destination-executor.service.test.ts@@ -105,6 +105,7 @@ describe('NativeDestinationExecutorService', () => {"Content-Type": "application/json","User-Agent": "PostHog.com/1.0",},+ "maxResponseBytes": 5242880,"method": "POST",},]@@ -170,6 +171,7 @@ describe('NativeDestinationExecutorService', () => {"Content-Type": "application/json","User-Agent": "PostHog.com/1.0",},+ "maxResponseBytes": 5242880,"method": "POST",},]@@ -192,6 +194,31 @@ describe('NativeDestinationExecutorService', () => {})})+ it('retries a retryable status whose body is over the size cap', async () => {+ const invocation = createExampleNativeInvocation(+ createHogFunction({ name: 'Plugin test', template_id: 'native-webhook' }),+ inputs+ )+ const tooLarge = Object.assign(new Error('Response body is larger than the 5242880 byte limit'), {+ name: 'ResponseTooLargeError',+ })+ mockFetch.mockResolvedValue({+ status: 503,+ json: () => Promise.reject(tooLarge),+ text: () => Promise.reject(tooLarge),+ headers: {},+ dump: () => Promise.resolve(),+ })++ const result = await service.execute(invocation)++ expect(result.finished).toBe(false)+ expect(result.invocation.queueMetadata).toEqual({ tries: 1 })+ expect(result.logs.map((x) => x.message)).toContainEqual(+ expect.stringContaining('HTTP request failed with status 503 (Response body is larger than')+ )+ })+it('should retry retryable fetch errors', async () => {const fn = createHogFunction({name: 'Plugin test',@@ -231,6 +258,7 @@ describe('NativeDestinationExecutorService', () => {"Content-Type": "application/json","User-Agent": "PostHog.com/1.0",},+ "maxResponseBytes": 5242880,"method": "POST",},]diff --git a/nodejs/src/cdp/services/native-destination-executor.service.ts b/nodejs/src/cdp/services/native-destination-executor.service.tsindex 3fe920fa1bc0..6b35027f431f 100644--- a/nodejs/src/cdp/services/native-destination-executor.service.ts+++ b/nodejs/src/cdp/services/native-destination-executor.service.ts@@ -15,7 +15,13 @@ import {redactError,redactSensitiveValues,} from '../utils'-import { CdpFetchConfig, cdpTrackedFetch, getNextRetryTime, isFetchResponseRetriable } from '../utils/cdp-fetch'+import {+ CdpFetchConfig,+ cdpTrackedFetch,+ getNextRetryTime,+ isFetchResponseRetriable,+ readFetchResponseText,+} from '../utils/cdp-fetch'import { createInvocationResult } from '../utils/invocation-utils'const nativeDestinationExecutionDuration = new Histogram({@@ -175,7 +181,7 @@ export class NativeDestinationExecutorService {hogFunctionId: invocation.hogFunction.id,})- const fetchResponseText = (await fetchResponse?.text()) ?? 'unknown'+ const fetchResponseText = await readFetchResponseText(fetchResponse)if (fetchError || !fetchResponse || fetchResponse.status >= 400) {if (diff --git a/nodejs/src/cdp/services/segment-destination-executor.service.test.ts b/nodejs/src/cdp/services/segment-destination-executor.service.test.tsindex 439e068e5131..8659b951e8ef 100644--- a/nodejs/src/cdp/services/segment-destination-executor.service.test.ts+++ b/nodejs/src/cdp/services/segment-destination-executor.service.test.ts@@ -107,6 +107,7 @@ describe('SegmentDestinationExecutorService', () => {"headers": {"Content-Type": "application/json",},+ "maxResponseBytes": 5242880,"method": "POST",},]@@ -198,6 +199,7 @@ describe('SegmentDestinationExecutorService', () => {"headers": {"Content-Type": "application/json",},+ "maxResponseBytes": 5242880,"method": "POST",},]@@ -262,6 +264,7 @@ describe('SegmentDestinationExecutorService', () => {"headers": {"Content-Type": "application/json",},+ "maxResponseBytes": 5242880,"method": "POST",},]@@ -390,6 +393,7 @@ describe('SegmentDestinationExecutorService', () => {"Content-Type": "application/json","x-gb-agent": "GB/Segment",},+ "maxResponseBytes": 5242880,"method": "POST",},]@@ -466,6 +470,7 @@ describe('SegmentDestinationExecutorService', () => {{"body": undefined,"headers": {},+ "maxResponseBytes": 5242880,"method": "GET",},]@@ -478,6 +483,7 @@ describe('SegmentDestinationExecutorService', () => {"headers": {"Content-Type": "application/json",},+ "maxResponseBytes": 5242880,"method": "POST",},]diff --git a/nodejs/src/cdp/services/segment-destination-executor.service.ts b/nodejs/src/cdp/services/segment-destination-executor.service.tsindex d20d9f648a76..f7d95f674db9 100644--- a/nodejs/src/cdp/services/segment-destination-executor.service.ts+++ b/nodejs/src/cdp/services/segment-destination-executor.service.ts@@ -16,7 +16,13 @@ import {redactError,redactSensitiveValues,} from '../utils'-import { CdpFetchConfig, cdpTrackedFetch, getNextRetryTime, isFetchResponseRetriable } from '../utils/cdp-fetch'+import {+ CdpFetchConfig,+ cdpTrackedFetch,+ getNextRetryTime,+ isFetchResponseRetriable,+ readFetchResponseText,+} from '../utils/cdp-fetch'import { createInvocationResult } from '../utils/invocation-utils'const pluginExecutionDuration = new Histogram({@@ -310,7 +316,7 @@ export class SegmentDestinationExecutorService {hogFunctionId: invocation.hogFunction.id,})- const fetchResponseText = (await fetchResponse?.text()) ?? 'unknown'+ const fetchResponseText = await readFetchResponseText(fetchResponse)if (fetchError || !fetchResponse || fetchResponse.status >= 400) {if (diff --git a/nodejs/src/cdp/utils/cdp-fetch.ts b/nodejs/src/cdp/utils/cdp-fetch.tsindex 2464864cc8be..5f3fd0df2937 100644--- a/nodejs/src/cdp/utils/cdp-fetch.ts+++ b/nodejs/src/cdp/utils/cdp-fetch.ts@@ -9,6 +9,23 @@ import { tryCatch } from '~/common/utils/try-catch'import { PluginsServerConfig } from '../../types'export const MAX_FETCH_TIMEOUT_MS = 10000+// The response lands in the invocation state, which can be queued through Kafka with a 6 MB message limit.+export const MAX_FETCH_RESPONSE_BYTES = 5 * 1024 * 1024++// An error body is only logged, so an oversized one must not hide the status that decides the retry.+export async function readFetchResponseText(response: FetchResponse | null | undefined): Promise<string> {+ if (!response) {+ return 'unknown'+ }+ try {+ return await response.text()+ } catch (e) {+ if (e.name === 'ResponseTooLargeError' && response.status >= 400) {+ return e.message+ }+ throw e+ }+}/** Narrowed config type for CDP fetch retry settings, used by native/segment destination executors */export type CdpFetchConfig = Pick<@@ -134,7 +151,7 @@ export async function cdpTrackedFetch({// which run inside undici's connect flow where no caller context is otherwise available.const doFetch = () =>fetchAttribution.run({ teamId, hogFunctionId, templateId }, () =>- tryCatch(async () => await fetch(url, fetchParams))+ tryCatch(async () => await fetch(url, { maxResponseBytes: MAX_FETCH_RESPONSE_BYTES, ...fetchParams })))let [fetchError, fetchResponse] = await doFetch()diff --git a/nodejs/src/common/utils/request.test.ts b/nodejs/src/common/utils/request.test.tsindex f184d2ff8ab7..7c4037ac8416 100644--- a/nodejs/src/common/utils/request.test.ts+++ b/nodejs/src/common/utils/request.test.ts@@ -10,6 +10,7 @@ import { parseJSON } from './json-parse'import { logger } from './logger'import {FetchOptions,+ ResponseTooLargeError,SecureRequestError,fetch,internalFetch,@@ -38,6 +39,9 @@ beforeAll(async () => {} else if (url.pathname === '/status/404') {res.writeHead(404)res.end()+ } else if (url.pathname === '/bom') {+ res.writeHead(200, { 'content-type': 'application/json' })+ res.end('\ufeff{"ok":true}')} else if (url.pathname === '/stream/50') {res.writeHead(200, { 'content-type': 'application/json' })for (let i = 0; i < 50; i++) {@@ -728,6 +732,29 @@ describe('_fetch response body handling', () => {expect(response.headers['content-type']).toBeDefined()})+ const streamedBodyBytes = range(50).reduce((total, i) => total + JSON.stringify({ id: i }).length + 1, 0)++ it.each([+ ['under the limit', streamedBodyBytes + 1, false],+ ['exactly at the limit', streamedBodyBytes, false],+ ['over the limit', streamedBodyBytes - 1, true],+ ])('caps a streamed body %s', async (_name, maxResponseBytes, rejects) => {+ const response = await internalFetch(`${baseUrl}/stream/50`, { maxResponseBytes })+ if (rejects) {+ await expect(response.text()).rejects.toBeInstanceOf(ResponseTooLargeError)+ } else {+ expect(await response.text()).toHaveLength(streamedBodyBytes)+ }+ })++ it.each([+ ['uncapped', undefined],+ ['capped', 1024],+ ])('strips a leading BOM from a %s body', async (_name, maxResponseBytes) => {+ const response = await internalFetch(`${baseUrl}/bom`, { maxResponseBytes })+ expect(await response.json()).toEqual({ ok: true })+ })+it('should fully read streamed/chunked response bodies', async () => {const response = await internalFetch(`${baseUrl}/stream/50`)const text = await response.text()diff --git a/nodejs/src/common/utils/request.ts b/nodejs/src/common/utils/request.tsindex b8ca00ba7b71..d418a74378f2 100644--- a/nodejs/src/common/utils/request.ts+++ b/nodejs/src/common/utils/request.ts@@ -135,6 +135,7 @@ export type FetchOptions = {// Requests with different groups never share an HTTP/2 session. APNs can reject a provider token from one developer// team on a session that already carried another team's token. Requires `allowH2`.http2ConnectionGroup?: string+ maxResponseBytes?: number}export type FetchResponse = {@@ -161,6 +162,13 @@ export class InvalidRequestError extends Error {}}+export class ResponseTooLargeError extends Error {+ constructor(maxBytes: number) {+ super(`Response body is larger than the ${maxBytes} byte limit`)+ this.name = 'ResponseTooLargeError'+ }+}+export class ResolutionError extends Error {constructor(message: string) {super(message)@@ -750,7 +758,18 @@ export async function _fetch(const readBody = (): Promise<string> => {if (!bodyPromise) {- bodyPromise = readAndDestroyBody(result.body).finally(() => lifecycle?.onBodyDone())+ const maxBytes = options.maxResponseBytes+ const read =+ maxBytes === undefined+ ? readAndDestroyBody(result.body)+ : readCappedBody(result.body, maxBytes, false).then(({ bytes, overLimit }) => {+ if (overLimit) {+ throw new ResponseTooLargeError(maxBytes)+ }+ const text = bytes.toString('utf8')+ return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text+ })+ bodyPromise = read.finally(() => lifecycle?.onBodyDone())}return bodyPromise}