:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { gunzipSync } from 'zlib'
/** Mirrors Django's `decompress` / `load_data_from_request`. A body Django accepts and this rejects
* is a device that never registers, and the SDK re-posts it on every app open for the life of the
* install, so the decoding has to match rather than be merely reasonable. */
export class RequestParsingError extends Error {}
/** Raised when a body with no declared compression parsed as neither JSON nor gzip. Django
* distinguishes it from RequestParsingError; the view answers both the same way. */
export class UnspecifiedCompressionFallbackParsingError extends Error {}
export class InflatedBodyTooLargeError extends Error {}
/** Django inflates without a bound. Here decoding runs on the one event loop that serves every
* request and its cost grows with the inflated size, so a 16 KiB body that inflates to 16 MiB would
* stall every other request on the process. A registration is under 1 KiB, so no SDK request comes
* near this. */
export const MAX_INFLATED_BYTES = 64 * 1024
const GZIP_COMPRESSIONS = new Set(['gzip', 'gzip-js'])
const BODY_CONTENT_TYPES = new Set(['', 'text/plain', 'application/json'])
export type RawRequest = {
method: string
body: Buffer
contentType?: string
contentEncoding?: string
/** Parsed from the URL. Django reads `compression` from the query string for POST. */
query?: URLSearchParams
/** Envoy appends `-shadow` to the Host of a mirrored copy. */
mirrored?: boolean
}
/** The decoded body, plus the form fields Django's `get_token` reads directly off `request.POST`. */
export type DecodedRequest = {
data: unknown
form?: URLSearchParams
}
export function decodeRequest(request: RawRequest): DecodedRequest {
const contentType = (request.contentType ?? '').split(';')[0].trim().toLowerCase()
const isForm = request.method === 'POST' && !BODY_CONTENT_TYPES.has(contentType)
const form = isForm ? parseForm(request.body, contentType) : undefined
// DELETE never reads form fields: Django's `load_data_from_request` only inspects the body for
// POST, so the view decompresses `request.body` directly for every other method.
const compression =
request.method === 'POST'
? (
request.query?.get('compression') ||
form?.get('compression') ||
request.contentEncoding ||
''
).toLowerCase()
: (request.contentEncoding ?? '').toLowerCase()
const payload: Buffer | string | null = isForm ? (form?.get('data') ?? null) : request.body
return { data: decompress(payload, compression), form }
}
export function decompress(data: Buffer | string | null, compression: string): unknown {
if (data === null || data.length === 0) {
return null
}
let current: Buffer | string = data
if (GZIP_COMPRESSIONS.has(compression)) {
if (Buffer.isBuffer(current) ? current.equals(Buffer.from('undefined')) : current === 'undefined') {
throw new RequestParsingError(
'data being loaded from the request body for decompression is the literal string "undefined"'
)
}
try {
current = gunzipSync(Buffer.isBuffer(current) ? current : Buffer.from(current), {
maxOutputLength: MAX_INFLATED_BYTES,
})
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ERR_BUFFER_TOO_LARGE') {
throw new InflatedBodyTooLargeError('Inflated body too large.')
}
throw new RequestParsingError(`Failed to decompress data. ${String(error)}`)
}
}
if (compression === 'lz64') {
// A posthog-js path. No mobile SDK sends it and web clients do not register devices, so
// this rejects it rather than carry a decompressor for traffic that does not exist.
throw new RequestParsingError('lz64 compression is not supported.')
}
const base64Decoded = tryBase64Decode(current)
if (base64Decoded !== null) {
current = base64Decoded
}
// `json.loads` on bytes decodes UTF-8 strictly, and the view treats that failure the same way
// as invalid JSON, so invalid bytes have to fail here rather than become replacement characters.
const text = Buffer.isBuffer(current) ? decodeUtf8Strict(current) : current
try {
if (text === null) {
throw new SyntaxError('body is not valid UTF-8')
}
return parseJsonLikePython(text)
} catch (error) {
if (compression !== '') {
throw new RequestParsingError(`Invalid JSON: ${String(error)}`)
}
// Django retries as gzip before giving up, so a body it accepts this way must not fail here.
try {
return decompress(current, 'gzip')
} catch (fallbackError) {
if (fallbackError instanceof InflatedBodyTooLargeError) {
throw fallbackError
}
throw new UnspecifiedCompressionFallbackParsingError(`Invalid JSON: ${String(error)}`)
}
}
}
/** Python's `json.loads` reads the bare tokens `NaN`, `Infinity` and `-Infinity` and this endpoint
* maps them to null, where `JSON.parse` rejects them outright. Only reached once strict parsing has
* failed, so a string whose contents are "NaN" is already parsed and never rewritten. */
function parseJsonLikePython(text: string): unknown {
try {
// oxlint-disable-next-line eslint-js/no-restricted-syntax
return JSON.parse(text)
} catch (error) {
const rewritten = replaceBareConstants(text)
if (rewritten === null) {
throw error
}
// oxlint-disable-next-line eslint-js/no-restricted-syntax
return JSON.parse(rewritten)
}
}
const BARE_CONSTANT = /NaN|-?Infinity/y
/** Replaces the non-JSON numeric constants outside string literals, or null when there are none. */
function replaceBareConstants(text: string): string | null {
let result = ''
let index = 0
let inString = false
let replaced = false
while (index < text.length) {
const character = text[index]
if (inString) {
if (character === '\\') {
result += text.slice(index, index + 2)
index += 2
continue
}
if (character === '"') {
inString = false
}
result += character
index += 1
continue
}
if (character === '"') {
inString = true
result += character
index += 1
continue
}
BARE_CONSTANT.lastIndex = index
const match = BARE_CONSTANT.exec(text)
if (match) {
result += 'null'
index += match[0].length
replaced = true
continue
}
result += character
index += 1
}
return replaced ? result : null
}
const BASE64_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
const BASE64_VALUES = new Map([...BASE64_ALPHABET].map((character, index) => [character, index]))
/** Django tries base64 on every body before parsing it. Mirrors `posthog.utils.base64_decode`, where
* each step can reject a body, and a body Python rejects here is one it goes on to parse unchanged,
* so the failures matter as much as the successes. */
function tryBase64Decode(data: Buffer | string): string | null {
const raw = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf8')
// Python encodes to ASCII first, so anything outside it raises and the attempt is abandoned.
if (raw.some((byte) => byte > 0x7f)) {
return null
}
let text = raw.toString('latin1')
const unquoted = percentDecodeToAscii(text)
if (unquoted === null) {
return null
}
if (raw.subarray(0, 5).toString('latin1') === 'data=') {
const separator = unquoted.indexOf('=')
text = separator === -1 ? '' : unquoted.slice(separator + 1)
} else {
text = unquoted
}
text = text.replace(/ /g, '')
const padding = text.length % 4
if (padding) {
text += '='.repeat(4 - padding)
}
const decoded = pythonB64Decode(text)
if (decoded === null) {
return null
}
return decodeUtf8Strict(decoded)
}
/** `urllib.parse.unquote` then `.encode("ascii")`: a non-ASCII byte or a malformed escape both leave
* a character the ASCII encode rejects, so either abandons the attempt. */
function percentDecodeToAscii(text: string): string | null {
let result = ''
let index = 0
while (index < text.length) {
if (text[index] === '%' && index + 2 < text.length) {
const hex = text.slice(index + 1, index + 3)
if (/^[0-9a-fA-F]{2}$/.test(hex)) {
const byte = parseInt(hex, 16)
if (byte > 0x7f) {
return null
}
result += String.fromCharCode(byte)
index += 3
continue
}
// Python leaves a malformed escape as the literal characters it found.
}
result += text[index]
index += 1
}
return result
}
/** CPython's lenient `binascii.a2b_base64`: non-alphabet characters and `=` are skipped inline, and
* only trailing padding can close a partial quad. Node's decoder accepts bodies Python rejects, and
* Python leaving a body untouched is what lets a plain JSON body parse as itself. */
function pythonB64Decode(text: string): Buffer | null {
const out: number[] = []
let quadPosition = 0
let trailingPads = 0
let carry = 0
for (const character of text) {
if (character === '=') {
trailingPads += 1
continue
}
const value = BASE64_VALUES.get(character)
if (value === undefined) {
continue
}
trailingPads = 0
switch (quadPosition) {
case 0:
carry = value
quadPosition = 1
break
case 1:
out.push(((carry << 2) | (value >> 4)) & 0xff)
carry = value & 0x0f
quadPosition = 2
break
case 2:
out.push(((carry << 4) | (value >> 2)) & 0xff)
carry = value & 0x03
quadPosition = 3
break
default:
out.push(((carry << 6) | value) & 0xff)
quadPosition = 0
break
}
}
if (quadPosition === 0) {
return Buffer.from(out)
}
// One data character past a quad can never be a byte, and a partial quad needs padding behind it.
if (quadPosition === 1 || quadPosition + trailingPads < 4) {
return null
}
return Buffer.from(out)
}
/** Python raises on bytes that are not valid UTF-8, and that exception is what makes the caller keep
* the original body. Node's default decoder would substitute U+FFFD and mangle it instead. */
function decodeUtf8Strict(buffer: Buffer): string | null {
try {
return new TextDecoder('utf-8', { fatal: true }).decode(buffer)
} catch {
return null
}
}
function parseForm(body: Buffer, contentType: string): URLSearchParams | undefined {
// Django also parses multipart here, but no client registers a device that way and a half-built
// parser would answer differently rather than not at all.
if (contentType !== 'application/x-www-form-urlencoded') {
return undefined
}
try {
return new URLSearchParams(body.toString('utf8'))
} catch {
return undefined
}
}