Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cdp): cap the response body a function fetch reads (#121966)

Daniel Marchukbb64eb1d5443
diff --git a/nodejs/src/cdp/cdp-e2e.serial.test.ts b/nodejs/src/cdp/cdp-e2e.serial.test.ts
index ad0d206a0511..b29004a886ef 100644
--- a/nodejs/src/cdp/cdp-e2e.serial.test.ts
+++ b/nodejs/src/cdp/cdp-e2e.serial.test.ts
@@ -197,6 +197,7 @@ describe('CDP Consumer loop', () => {
"headers": {
"Authorization": "Bearer super-secret-token",
},
+ "maxResponseBytes": 5242880,
"method": "POST",
},
]
diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-plugins.test.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-plugins.test.ts
index 0cd0a382dfad..3a9e9b48d86d 100644
--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-plugins.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-plugins.test.ts
@@ -155,6 +155,7 @@ describe('CdpCyclotronWorkerPlugins', () => {
"Authorization": "Bearer 1234567890",
"Content-Type": "application/json",
},
+ "maxResponseBytes": 5242880,
"method": "POST",
},
]
@@ -169,6 +170,7 @@ describe('CdpCyclotronWorkerPlugins', () => {
"Authorization": "Bearer 1234567890",
"Content-Type": "application/json",
},
+ "maxResponseBytes": 5242880,
"method": "POST",
},
]
diff --git a/nodejs/src/cdp/services/hog-executor-async.service.ts b/nodejs/src/cdp/services/hog-executor-async.service.ts
index 2c8878bc4bc2..c914db4cba34 100644
--- a/nodejs/src/cdp/services/hog-executor-async.service.ts
+++ b/nodejs/src/cdp/services/hog-executor-async.service.ts
@@ -18,7 +18,12 @@ import type {
} from '../types'
import { createAddLogFunction, destinationE2eLagMsSummary } from '../utils'
import { resolveAwsSigV4Credentials, signAwsRequest } from '../utils/aws-sigv4'
-import { cdpTrackedFetch, fetchErrorDetail, isFetchResponseRetriable } from '../utils/cdp-fetch'
+import {
+ MAX_FETCH_RESPONSE_BYTES,
+ cdpTrackedFetch,
+ fetchErrorDetail,
+ isFetchResponseRetriable,
+} from '../utils/cdp-fetch'
import { createInvocationResult } from '../utils/invocation-utils'
import { isNonFailureStatus } from '../utils/non-failure-status-codes'
import { ScopedServiceJwt } from '../utils/scoped-service-jwt'
@@ -579,7 +584,12 @@ export class HogExecutorAsyncService {
}
}
} catch (e) {
- addLog('error', `Failed to parse response body: ${e.message}`)
+ addLog(
+ 'error',
+ e.name === 'ResponseTooLargeError'
+ ? `The response body is larger than ${MAX_FETCH_RESPONSE_BYTES / 1024 / 1024} MB, so it was not read.`
+ : `Failed to parse response body: ${e.message}`
+ )
body = undefined
}
diff --git a/nodejs/src/cdp/services/legacy-plugin-executor.service.test.ts b/nodejs/src/cdp/services/legacy-plugin-executor.service.test.ts
index 55f24f637f07..ac6fe565a54a 100644
--- a/nodejs/src/cdp/services/legacy-plugin-executor.service.test.ts
+++ b/nodejs/src/cdp/services/legacy-plugin-executor.service.test.ts
@@ -255,6 +255,7 @@ describe('LegacyPluginExecutorService', () => {
"Authorization": "Basic MTIzNDU2Nzg5MDpjaW8tdG9rZW4=",
"User-Agent": "PostHog Customer.io App",
},
+ "maxResponseBytes": 5242880,
"method": "GET",
},
],
@@ -267,6 +268,7 @@ describe('LegacyPluginExecutorService', () => {
"Content-Type": "application/json",
"User-Agent": "PostHog Customer.io App",
},
+ "maxResponseBytes": 5242880,
"method": "PUT",
},
],
@@ -279,6 +281,7 @@ describe('LegacyPluginExecutorService', () => {
"Content-Type": "application/json",
"User-Agent": "PostHog Customer.io App",
},
+ "maxResponseBytes": 5242880,
"method": "POST",
},
],
diff --git a/nodejs/src/cdp/services/native-destination-executor.service.test.ts b/nodejs/src/cdp/services/native-destination-executor.service.test.ts
index 64a21c4a7480..114aa5b05c0b 100644
--- a/nodejs/src/cdp/services/native-destination-executor.service.test.ts
+++ b/nodejs/src/cdp/services/native-destination-executor.service.test.ts
@@ -105,6 +105,7 @@ describe('NativeDestinationExecutorService', () => {
"Content-Type": "application/json",
"User-Agent": "PostHog.com/1.0",
},
+ "maxResponseBytes": 5242880,
"method": "POST",
},
]
@@ -170,6 +171,7 @@ describe('NativeDestinationExecutorService', () => {
"Content-Type": "application/json",
"User-Agent": "PostHog.com/1.0",
},
+ "maxResponseBytes": 5242880,
"method": "POST",
},
]
@@ -192,6 +194,31 @@ describe('NativeDestinationExecutorService', () => {
})
})
+ it('retries a retryable status whose body is over the size cap', async () => {
+ const invocation = createExampleNativeInvocation(
+ createHogFunction({ name: 'Plugin test', template_id: 'native-webhook' }),
+ inputs
+ )
+ const tooLarge = Object.assign(new Error('Response body is larger than the 5242880 byte limit'), {
+ name: 'ResponseTooLargeError',
+ })
+ mockFetch.mockResolvedValue({
+ status: 503,
+ json: () => Promise.reject(tooLarge),
+ text: () => Promise.reject(tooLarge),
+ headers: {},
+ dump: () => Promise.resolve(),
+ })
+
+ const result = await service.execute(invocation)
+
+ expect(result.finished).toBe(false)
+ expect(result.invocation.queueMetadata).toEqual({ tries: 1 })
+ expect(result.logs.map((x) => x.message)).toContainEqual(
+ expect.stringContaining('HTTP request failed with status 503 (Response body is larger than')
+ )
+ })
+
it('should retry retryable fetch errors', async () => {
const fn = createHogFunction({
name: 'Plugin test',
@@ -231,6 +258,7 @@ describe('NativeDestinationExecutorService', () => {
"Content-Type": "application/json",
"User-Agent": "PostHog.com/1.0",
},
+ "maxResponseBytes": 5242880,
"method": "POST",
},
]
diff --git a/nodejs/src/cdp/services/native-destination-executor.service.ts b/nodejs/src/cdp/services/native-destination-executor.service.ts
index 3fe920fa1bc0..6b35027f431f 100644
--- a/nodejs/src/cdp/services/native-destination-executor.service.ts
+++ b/nodejs/src/cdp/services/native-destination-executor.service.ts
@@ -15,7 +15,13 @@ import {
redactError,
redactSensitiveValues,
} from '../utils'
-import { CdpFetchConfig, cdpTrackedFetch, getNextRetryTime, isFetchResponseRetriable } from '../utils/cdp-fetch'
+import {
+ CdpFetchConfig,
+ cdpTrackedFetch,
+ getNextRetryTime,
+ isFetchResponseRetriable,
+ readFetchResponseText,
+} from '../utils/cdp-fetch'
import { createInvocationResult } from '../utils/invocation-utils'
const nativeDestinationExecutionDuration = new Histogram({
@@ -175,7 +181,7 @@ export class NativeDestinationExecutorService {
hogFunctionId: invocation.hogFunction.id,
})
- const fetchResponseText = (await fetchResponse?.text()) ?? 'unknown'
+ const fetchResponseText = await readFetchResponseText(fetchResponse)
if (fetchError || !fetchResponse || fetchResponse.status >= 400) {
if (
diff --git a/nodejs/src/cdp/services/segment-destination-executor.service.test.ts b/nodejs/src/cdp/services/segment-destination-executor.service.test.ts
index 439e068e5131..8659b951e8ef 100644
--- a/nodejs/src/cdp/services/segment-destination-executor.service.test.ts
+++ b/nodejs/src/cdp/services/segment-destination-executor.service.test.ts
@@ -107,6 +107,7 @@ describe('SegmentDestinationExecutorService', () => {
"headers": {
"Content-Type": "application/json",
},
+ "maxResponseBytes": 5242880,
"method": "POST",
},
]
@@ -198,6 +199,7 @@ describe('SegmentDestinationExecutorService', () => {
"headers": {
"Content-Type": "application/json",
},
+ "maxResponseBytes": 5242880,
"method": "POST",
},
]
@@ -262,6 +264,7 @@ describe('SegmentDestinationExecutorService', () => {
"headers": {
"Content-Type": "application/json",
},
+ "maxResponseBytes": 5242880,
"method": "POST",
},
]
@@ -390,6 +393,7 @@ describe('SegmentDestinationExecutorService', () => {
"Content-Type": "application/json",
"x-gb-agent": "GB/Segment",
},
+ "maxResponseBytes": 5242880,
"method": "POST",
},
]
@@ -466,6 +470,7 @@ describe('SegmentDestinationExecutorService', () => {
{
"body": undefined,
"headers": {},
+ "maxResponseBytes": 5242880,
"method": "GET",
},
]
@@ -478,6 +483,7 @@ describe('SegmentDestinationExecutorService', () => {
"headers": {
"Content-Type": "application/json",
},
+ "maxResponseBytes": 5242880,
"method": "POST",
},
]
diff --git a/nodejs/src/cdp/services/segment-destination-executor.service.ts b/nodejs/src/cdp/services/segment-destination-executor.service.ts
index d20d9f648a76..f7d95f674db9 100644
--- a/nodejs/src/cdp/services/segment-destination-executor.service.ts
+++ b/nodejs/src/cdp/services/segment-destination-executor.service.ts
@@ -16,7 +16,13 @@ import {
redactError,
redactSensitiveValues,
} from '../utils'
-import { CdpFetchConfig, cdpTrackedFetch, getNextRetryTime, isFetchResponseRetriable } from '../utils/cdp-fetch'
+import {
+ CdpFetchConfig,
+ cdpTrackedFetch,
+ getNextRetryTime,
+ isFetchResponseRetriable,
+ readFetchResponseText,
+} from '../utils/cdp-fetch'
import { createInvocationResult } from '../utils/invocation-utils'
const pluginExecutionDuration = new Histogram({
@@ -310,7 +316,7 @@ export class SegmentDestinationExecutorService {
hogFunctionId: invocation.hogFunction.id,
})
- const fetchResponseText = (await fetchResponse?.text()) ?? 'unknown'
+ const fetchResponseText = await readFetchResponseText(fetchResponse)
if (fetchError || !fetchResponse || fetchResponse.status >= 400) {
if (
diff --git a/nodejs/src/cdp/utils/cdp-fetch.ts b/nodejs/src/cdp/utils/cdp-fetch.ts
index 2464864cc8be..5f3fd0df2937 100644
--- a/nodejs/src/cdp/utils/cdp-fetch.ts
+++ b/nodejs/src/cdp/utils/cdp-fetch.ts
@@ -9,6 +9,23 @@ import { tryCatch } from '~/common/utils/try-catch'
import { PluginsServerConfig } from '../../types'
export const MAX_FETCH_TIMEOUT_MS = 10000
+// The response lands in the invocation state, which can be queued through Kafka with a 6 MB message limit.
+export const MAX_FETCH_RESPONSE_BYTES = 5 * 1024 * 1024
+
+// An error body is only logged, so an oversized one must not hide the status that decides the retry.
+export async function readFetchResponseText(response: FetchResponse | null | undefined): Promise<string> {
+ if (!response) {
+ return 'unknown'
+ }
+ try {
+ return await response.text()
+ } catch (e) {
+ if (e.name === 'ResponseTooLargeError' && response.status >= 400) {
+ return e.message
+ }
+ throw e
+ }
+}
/** Narrowed config type for CDP fetch retry settings, used by native/segment destination executors */
export type CdpFetchConfig = Pick<
@@ -134,7 +151,7 @@ export async function cdpTrackedFetch({
// which run inside undici's connect flow where no caller context is otherwise available.
const doFetch = () =>
fetchAttribution.run({ teamId, hogFunctionId, templateId }, () =>
- tryCatch(async () => await fetch(url, fetchParams))
+ tryCatch(async () => await fetch(url, { maxResponseBytes: MAX_FETCH_RESPONSE_BYTES, ...fetchParams }))
)
let [fetchError, fetchResponse] = await doFetch()
diff --git a/nodejs/src/common/utils/request.test.ts b/nodejs/src/common/utils/request.test.ts
index f184d2ff8ab7..7c4037ac8416 100644
--- a/nodejs/src/common/utils/request.test.ts
+++ b/nodejs/src/common/utils/request.test.ts
@@ -10,6 +10,7 @@ import { parseJSON } from './json-parse'
import { logger } from './logger'
import {
FetchOptions,
+ ResponseTooLargeError,
SecureRequestError,
fetch,
internalFetch,
@@ -38,6 +39,9 @@ beforeAll(async () => {
} else if (url.pathname === '/status/404') {
res.writeHead(404)
res.end()
+ } else if (url.pathname === '/bom') {
+ res.writeHead(200, { 'content-type': 'application/json' })
+ res.end('\ufeff{"ok":true}')
} else if (url.pathname === '/stream/50') {
res.writeHead(200, { 'content-type': 'application/json' })
for (let i = 0; i < 50; i++) {
@@ -728,6 +732,29 @@ describe('_fetch response body handling', () => {
expect(response.headers['content-type']).toBeDefined()
})
+ const streamedBodyBytes = range(50).reduce((total, i) => total + JSON.stringify({ id: i }).length + 1, 0)
+
+ it.each([
+ ['under the limit', streamedBodyBytes + 1, false],
+ ['exactly at the limit', streamedBodyBytes, false],
+ ['over the limit', streamedBodyBytes - 1, true],
+ ])('caps a streamed body %s', async (_name, maxResponseBytes, rejects) => {
+ const response = await internalFetch(`${baseUrl}/stream/50`, { maxResponseBytes })
+ if (rejects) {
+ await expect(response.text()).rejects.toBeInstanceOf(ResponseTooLargeError)
+ } else {
+ expect(await response.text()).toHaveLength(streamedBodyBytes)
+ }
+ })
+
+ it.each([
+ ['uncapped', undefined],
+ ['capped', 1024],
+ ])('strips a leading BOM from a %s body', async (_name, maxResponseBytes) => {
+ const response = await internalFetch(`${baseUrl}/bom`, { maxResponseBytes })
+ expect(await response.json()).toEqual({ ok: true })
+ })
+
it('should fully read streamed/chunked response bodies', async () => {
const response = await internalFetch(`${baseUrl}/stream/50`)
const text = await response.text()
diff --git a/nodejs/src/common/utils/request.ts b/nodejs/src/common/utils/request.ts
index b8ca00ba7b71..d418a74378f2 100644
--- a/nodejs/src/common/utils/request.ts
+++ b/nodejs/src/common/utils/request.ts
@@ -135,6 +135,7 @@ export type FetchOptions = {
// Requests with different groups never share an HTTP/2 session. APNs can reject a provider token from one developer
// team on a session that already carried another team's token. Requires `allowH2`.
http2ConnectionGroup?: string
+ maxResponseBytes?: number
}
export type FetchResponse = {
@@ -161,6 +162,13 @@ export class InvalidRequestError extends Error {
}
}
+export class ResponseTooLargeError extends Error {
+ constructor(maxBytes: number) {
+ super(`Response body is larger than the ${maxBytes} byte limit`)
+ this.name = 'ResponseTooLargeError'
+ }
+}
+
export class ResolutionError extends Error {
constructor(message: string) {
super(message)
@@ -750,7 +758,18 @@ export async function _fetch(
const readBody = (): Promise<string> => {
if (!bodyPromise) {
- bodyPromise = readAndDestroyBody(result.body).finally(() => lifecycle?.onBodyDone())
+ const maxBytes = options.maxResponseBytes
+ const read =
+ maxBytes === undefined
+ ? readAndDestroyBody(result.body)
+ : readCappedBody(result.body, maxBytes, false).then(({ bytes, overLimit }) => {
+ if (overLimit) {
+ throw new ResponseTooLargeError(maxBytes)
+ }
+ const text = bytes.toString('utf8')
+ return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text
+ })
+ bodyPromise = read.finally(() => lifecycle?.onBodyDone())
}
return bodyPromise
}