Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(logs): stop corrupt avro containers from stalling or poisoning decodes (#127604)

Frank Hamanda56e5969aae1
diff --git a/nodejs/src/logs/log-record-avro.test.ts b/nodejs/src/logs/log-record-avro.test.ts
index 40f66b3d1dae..8e44fc9396dd 100644
--- a/nodejs/src/logs/log-record-avro.test.ts
+++ b/nodejs/src/logs/log-record-avro.test.ts
@@ -1,3 +1,4 @@
+import { compress } from '@mongodb-js/zstd'
import avro from 'avsc'
import { parseJSON } from '~/common/utils/json-parse'
@@ -420,10 +421,54 @@ describe('log-record-avro', () => {
expect(decoded[0].bytes_uncompressed).toBe(789)
})
- it('rejects promise for invalid buffer', async () => {
- const invalidBuffer = Buffer.from('not avro data')
+ it.each([
+ [
+ 'bytes that are not a container',
+ () => Promise.resolve(Buffer.from('not avro data')),
+ 'invalid magic bytes',
+ ],
+ [
+ 'a trailing block count past 2^53',
+ async () =>
+ Buffer.concat([
+ await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', [createRecord()]),
+ Buffer.from([0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x20]),
+ ]),
+ 'potential precision loss',
+ ],
+ ])('rejects promise for %s', async (_name, build, message) => {
+ await expect(decodeLogRecords(await build())).rejects.toThrow(message)
+ })
+
+ it('keeps decoding bytes fields after a container whose bytes length is NaN', async () => {
+ // uuid null, then trace_id as bytes with a length varint whose float read gives NaN.
+ const craftedBlock = Buffer.concat([
+ Buffer.from([0x00, 0x02]),
+ Buffer.alloc(200, 0x80),
+ Buffer.from([0x00]),
+ ])
+ const crafted = await new Promise<Buffer>((resolve, reject) => {
+ const buffers: Buffer[] = []
+ const encoder = new avro.streams.BlockEncoder(LOG_RECORD_SCHEMA, {
+ codec: 'zstandard',
+ codecs: {
+ zstandard: (_buf: Buffer, cb: (err: Error | null, compressed?: Buffer) => void) => {
+ compress(craftedBlock, 1).then((compressed) => cb(null, compressed), cb)
+ },
+ },
+ })
+ encoder.on('data', (buf: Buffer) => buffers.push(buf))
+ encoder.on('end', () => resolve(Buffer.concat(buffers)))
+ encoder.on('error', reject)
+ encoder.end(createRecord())
+ })
+ const traceId = Buffer.from('0102030405060708090a0b0c0d0e0f10', 'hex')
+ const valid = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', [createRecord({ trace_id: traceId })])
+
+ await expect(decodeLogRecords(crafted)).rejects.toThrow()
+ const [, , decoded] = await decodeLogRecords(valid)
- await expect(decodeLogRecords(invalidBuffer)).rejects.toThrow()
+ expect(decoded[0].trace_id).toEqual(traceId)
})
})
diff --git a/nodejs/src/logs/log-record-avro.ts b/nodejs/src/logs/log-record-avro.ts
index 73e4db8b3e1c..a19586f9f0a7 100644
--- a/nodejs/src/logs/log-record-avro.ts
+++ b/nodejs/src/logs/log-record-avro.ts
@@ -1,7 +1,6 @@
import { compress, decompress } from '@mongodb-js/zstd'
import avro from 'avsc'
import { Counter, Histogram } from 'prom-client'
-import { Readable } from 'stream'
import { instrumented } from '~/common/tracing/tracing-utils'
import { parseJSON } from '~/common/utils/json-parse'
@@ -118,14 +117,6 @@ export async function decodeLogRecords(buffer: Buffer): Promise<[avro.Type | und
},
})
- const stream = new Readable()
- stream.on('error', (err: Error) => {
- reject(err)
- })
- stream.push(buffer)
- stream.push(null)
- stream.pipe(decoder)
-
let logRecordType: avro.Type | undefined
let compressionCodec: string = 'null'
@@ -150,6 +141,10 @@ export async function decodeLogRecords(buffer: Buffer): Promise<[avro.Type | und
decoder.on('error', (err: Error) => {
reject(err)
})
+
+ // Write directly rather than through a pipe: avsc throws from inside `write` when a
+ // block count is not a safe long, and only a synchronous call lets this catch see it.
+ decoder.end(buffer)
} catch (err) {
reject(err)
}
diff --git a/patches/[email protected] b/patches/[email protected]
new file mode 100644
index 000000000000..91f22101ae18
--- /dev/null
+++ b/patches/[email protected]
@@ -0,0 +1,19 @@
+diff --git a/lib/utils.js b/lib/utils.js
+index cd0cb41a24c57786e5cb4dd0061f3d8d9a7492b6..266afce261f2980000b02802b69f45f7d94af54b 100644
+--- a/lib/utils.js
++++ b/lib/utils.js
+@@ -548,7 +548,14 @@ Tap.prototype.readInt = Tap.prototype.readLong = function () {
+ // Switch to float arithmetic, otherwise we might overflow.
+ f = n;
+ fk = 268435456; // 2 ** 28.
++ var i = 4; // Bytes read so far.
+ do {
++ if (i++ === 10) {
++ // A long takes at most 10 bytes. A longer varint is invalid, and past about 150 bytes
++ // `fk` overflows, so a zero group adds `0 * Infinity` and the result is NaN.
++ this._invalidate();
++ return 0;
++ }
+ b = buf[this.pos++];
+ f += (b & 0x7f) * fk;
+ fk *= 128;
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 8bea95d26c58..ae9d35f86422 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -192,6 +192,7 @@ patchedDependencies:
'@base-ui/[email protected]': e0be9f964398f2e05b65153968c757a2869a3c5ddc6c2d3a32bf3140a4a8b34f
'@base-ui/[email protected]': c25c4734cc997d0eec882b628a0231f91805610478dba85f65a60f0ba6fbb80a
'@connectrpc/[email protected]': a8248109885bda6ff3ce8663332190cabec9ca5da4deab47603ea6b1dbca27c4
+ [email protected]: 4d6bb6760158ebf8090f1673f57fbf5adca9b37c2827c0a7de3303f4b3a0873f
[email protected]: 2e3e09d22bbc20d8961e0392c318f12ffada0babb6422f03b5538af8b2707852
[email protected]: d95e6638894935a75bd93b60b5243cb825892a1eedcc4c9a125a3e7dce8b64f0
[email protected]: f0a5786708aab63eb75c89a2c8de10a8da63fce02a5d9ab588fbe684cab8951e
@@ -1718,7 +1719,7 @@ importers:
version: 2.2.1
avsc:
specifier: ^5.7.9
- version: 5.7.9
+ version: 5.7.9(patch_hash=4d6bb6760158ebf8090f1673f57fbf5adca9b37c2827c0a7de3303f4b3a0873f)
aws-sdk:
specifier: ^2.927.0
version: 2.1366.0
@@ -35094,7 +35095,7 @@ snapshots:
dependencies:
possible-typed-array-names: 1.0.0
+ [email protected](patch_hash=4d6bb6760158ebf8090f1673f57fbf5adca9b37c2827c0a7de3303f4b3a0873f): {}
dependencies:
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index a6e7f28af316..067c12281823 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -119,6 +119,10 @@ overrides:
'@tiptap/extension-underline': 3.20.6
patchedDependencies:
+ # avsc reads a varint of any length. Past about 150 bytes its float path returns NaN, which passes
+ # every length check and corrupts the buffer pool all decoders share. The patch rejects a varint
+ # longer than 10 bytes, as the Avro spec requires. Remove it after an avsc release does the same.
# The drag library changes a body-wide selection style, which stalls dense dashboards.
# Remove this patch after an upstream release avoids that style change, or after the grid
# passes enableUserSelectHack=false to both drag and resize.