Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hog): bundle the source files a line program defines (#125181)

Rafael Audibert8c1ecd20aa4c
diff --git a/hog/README.md b/hog/README.md
index 91cc96aebd9c..e9d841ae402d 100644
--- a/hog/README.md
+++ b/hog/README.md
@@ -48,7 +48,6 @@ Then, once the commands above work:
Smaller follow-ups, none of them blocking:
-- [ ] `hog error-tracking upload --include-source`: read `DW_LNE_define_file` entries in the DWARF line program, so hog bundles the same source files as posthog-cli. Current compilers don't emit them, and DWARF 5 removed them.
- [ ] `hog error-tracking upload --include-source`: write ZIP64, so a binary that names more than 65,533 source files keeps its sources. hog uploads it without sources and warns.
Not planned:
diff --git a/hog/src/error-tracking/native/dwarf-fixture.ts b/hog/src/error-tracking/native/dwarf-fixture.ts
new file mode 100644
index 000000000000..d434a6731850
--- /dev/null
+++ b/hog/src/error-tracking/native/dwarf-fixture.ts
@@ -0,0 +1,113 @@
+// Builds a relocatable x86-64 ELF with one DWARF 4 compile unit and a hand-written line program,
+// for line-program cases no compiler emits. Tests only.
+
+const cstr = (text: string): number[] => [...Buffer.from(text, 'utf8'), 0]
+
+export function uleb(value: bigint | number): number[] {
+ let v = BigInt(value)
+ const out: number[] = []
+ do {
+ let byte = Number(v & 0x7fn)
+ v >>= 7n
+ if (v !== 0n) {
+ byte |= 0x80
+ }
+ out.push(byte)
+ } while (v !== 0n)
+ return out
+}
+
+const u16 = (v: number): number[] => [v & 0xff, (v >> 8) & 0xff]
+const u32 = (v: number): number[] => [v & 0xff, (v >> 8) & 0xff, (v >> 16) & 0xff, (v >>> 24) & 0xff]
+export const u64 = (v: bigint): number[] => Array.from({ length: 8 }, (_, i) => Number((v >> BigInt(8 * i)) & 0xffn))
+
+export const op = {
+ setAddress: (address: bigint): number[] => [0, 9, 0x02, ...u64(address)],
+ defineFile: (path: string, dir: number): number[] => {
+ const body = [0x03, ...cstr(path), ...uleb(dir), 0, 0]
+ return [0, ...uleb(body.length), ...body]
+ },
+ advancePc: (by: bigint): number[] => [0x02, ...uleb(by)],
+ copy: [0x01],
+ endSequence: [0, 1, 0x01],
+}
+
+export function dwarf4Object(program: number[]): Uint8Array {
+ const abbrev = [1, 0x11, 0, 0x03, 0x08, 0x1b, 0x08, 0x10, 0x17, 0x11, 0x01, 0, 0, 0]
+ const die = [1, ...cstr('/src/proj/main.c'), ...cstr('/src/proj'), ...u32(0), ...u64(0x1000n)]
+ const infoBody = [...u16(4), ...u32(0), 8, ...die]
+ const info = [...u32(infoBody.length), ...infoBody]
+
+ // minimum_instruction_length, maximum_operations_per_instruction, default_is_stmt,
+ // line_base (-5), line_range, opcode_base, then the standard opcode lengths.
+ const encoding = [1, 1, 1, 0xfb, 14, 13]
+ const opcodeLengths = [0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1]
+ const file = (name: string, dir: number): number[] => [...cstr(name), dir, 0, 0]
+ const header = [
+ ...encoding,
+ ...opcodeLengths,
+ ...cstr('/src/proj/include'),
+ 0,
+ ...file('main.c', 0),
+ ...file('util.h', 1),
+ 0,
+ ]
+ const lineBody = [...u16(4), ...u32(header.length), ...header, ...program]
+ const line = [...u32(lineBody.length), ...lineBody]
+
+ const names = ['', '.debug_abbrev', '.debug_info', '.debug_line', '.shstrtab']
+ const shstrtab: number[] = []
+ const nameOffsets = names.map((name) => {
+ const at = shstrtab.length
+ shstrtab.push(...cstr(name))
+ return at
+ })
+ const contents = [abbrev, info, line, shstrtab]
+ const body: number[] = []
+ const offsets = contents.map((data) => {
+ const at = 64 + body.length
+ body.push(...data)
+ return at
+ })
+ while ((64 + body.length) % 8 !== 0) {
+ body.push(0)
+ }
+ const sectionHeadersAt = 64 + body.length
+ const sectionHeader = (index: number): number[] => {
+ if (index === 0) {
+ return Array.from({ length: 64 }, () => 0)
+ }
+ const type = names[index] === '.shstrtab' ? 3 : 1
+ return [
+ ...u32(nameOffsets[index]!),
+ ...u32(type),
+ ...u64(0n),
+ ...u64(0n),
+ ...u64(BigInt(offsets[index - 1]!)),
+ ...u64(BigInt(contents[index - 1]!.length)),
+ ...u32(0),
+ ...u32(0),
+ ...u64(1n),
+ ...u64(0n),
+ ]
+ }
+ // ELF64, little-endian, version 1.
+ const ident = [0x7f, 0x45, 0x4c, 0x46, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+ const elfHeader = [
+ ...ident,
+ ...u16(1), // ET_REL
+ ...u16(62), // EM_X86_64
+ ...u32(1),
+ ...u64(0n),
+ ...u64(0n),
+ ...u64(BigInt(sectionHeadersAt)),
+ ...u32(0),
+ ...u16(64),
+ ...u16(0),
+ ...u16(0),
+ ...u16(64),
+ ...u16(names.length),
+ ...u16(names.length - 1),
+ ]
+ return Uint8Array.from([...elfHeader, ...body, ...names.flatMap((_, i) => sectionHeader(i))])
+}
diff --git a/hog/src/error-tracking/native/dwarf.ts b/hog/src/error-tracking/native/dwarf.ts
index a4a987f5e314..194b7e5418d2 100644
--- a/hog/src/error-tracking/native/dwarf.ts
+++ b/hog/src/error-tracking/native/dwarf.ts
@@ -548,7 +548,8 @@ export function parseLineHeader(
}
const headerLength = rest.offset(is64)
const hdr = rest.split(headerLength)
- if (hdr.u8() === 0) {
+ const minInstLength = hdr.u8()
+ if (minInstLength === 0) {
throw new DwarfError('minimum instruction length zero')
}
const maxOps = version >= 4 ? hdr.u8() : 1
@@ -557,14 +558,15 @@ export function parseLineHeader(
}
hdr.u8() // default_is_stmt
hdr.i8() // line_base
- if (hdr.u8() === 0) {
+ const lineRange = hdr.u8()
+ if (lineRange === 0) {
throw new DwarfError('line range zero')
}
const opcodeBase = hdr.u8()
if (opcodeBase === 0) {
throw new DwarfError('opcode base zero')
}
- hdr.split(opcodeBase - 1)
+ const opcodeLengths = hdr.split(opcodeBase - 1)
const enc = { is64, version, addressSize }
const includeDirs: AttrValue[] = []
@@ -587,6 +589,10 @@ export function parseLineHeader(
hdr.uleb() // length
files.push({ path: { k: 'string', v: p }, dirIndex })
}
+ const programStart = offset + lenSize + fixed + headerLength
+ const program = new Cursor(s.line.read(programStart, unitLength - fixed - headerLength), s.le)
+ const params = { minInstLength, maxOps, lineRange, opcodeBase, opcodeLengths, addressSize }
+ files.push(...definedFiles(program, params))
return { version, compDir, includeDirs, files }
}
const dirFormat = entryFormat(hdr)
@@ -620,6 +626,163 @@ export function parseLineHeader(
return { version, compDir: null, includeDirs, files }
}
+interface LineProgramParams {
+ minInstLength: number
+ maxOps: number
+ lineRange: number
+ opcodeBase: number
+ opcodeLengths: Cursor
+ addressSize: number
+}
+
+const U64_MAX = (1n << 64n) - 1n
+
+/** gimli `leb128::read::unsigned`, exact to 64 bits. */
+function uleb64(c: Cursor): bigint {
+ let result = 0n
+ let shift = 0n
+ for (;;) {
+ const byte = c.u8()
+ if (shift === 63n && byte !== 0x00 && byte !== 0x01) {
+ throw new DwarfError('bad unsigned LEB128')
+ }
+ result |= BigInt(byte & 0x7f) << shift
+ if ((byte & 0x80) === 0) {
+ return result & U64_MAX
+ }
+ shift += 7n
+ }
+}
+
+function uintBig(c: Cursor, size: number): bigint {
+ const bytes = c.split(size)
+ let value = 0n
+ for (let i = 0; i < size; i++) {
+ value = (value << 8n) | BigInt(bytes.buf[bytes.pos + (c.le ? size - 1 - i : i)]!)
+ }
+ return value
+}
+
+/** gimli `leb128::read::signed`. Only its errors matter here, so it returns nothing. */
+function skipSleb64(c: Cursor): void {
+ let shift = 0
+ for (;;) {
+ const byte = c.u8()
+ if (shift === 63 && byte !== 0x00 && byte !== 0x7f) {
+ throw new DwarfError('bad signed LEB128')
+ }
+ shift += 7
+ if ((byte & 0x80) === 0) {
+ return
+ }
+ }
+}
+
+/**
+ * The files `DW_LNE_define_file` adds while the program runs. symbolic runs every line program
+ * (`DwarfLineProgram::prepare`) until its end or its first error, and gimli appends each defined
+ * file to the header's list. Address arithmetic can raise that error, so it is tracked exactly.
+ */
+function definedFiles(c: Cursor, p: LineProgramParams): LineFile[] {
+ const files: LineFile[] = []
+ const ones = p.addressSize === 8 ? U64_MAX : (1n << BigInt(8 * p.addressSize)) - 1n
+ const minTombstone = (ones - 1n) & ones
+ let address = 0n
+ let opIndex = 0n
+ let tombstone = false
+ const addSized = (length: bigint): void => {
+ const sum = address + length
+ if (sum > U64_MAX || (sum & ~ones) !== 0n) {
+ throw new DwarfError('address overflow')
+ }
+ address = sum
+ }
+ const advance = (operationAdvance: bigint): void => {
+ if (tombstone) {
+ return
+ }
+ const minInst = BigInt(p.minInstLength)
+ const maxOps = BigInt(p.maxOps)
+ let addressAdvance: bigint
+ if (maxOps === 1n) {
+ opIndex = 0n
+ addressAdvance = (minInst * operationAdvance) & U64_MAX
+ } else {
+ const withAdvance = (opIndex + operationAdvance) & U64_MAX
+ opIndex = withAdvance % maxOps
+ addressAdvance = (minInst * (withAdvance / maxOps)) & U64_MAX
+ }
+ addSized(addressAdvance)
+ }
+ try {
+ while (!c.isEmpty()) {
+ const opcode = c.u8()
+ let endSequence = false
+ if (opcode === 0) {
+ const length = uleb64(c)
+ if (length > BigInt(c.remaining)) {
+ throw new DwarfEof()
+ }
+ const rest = c.split(Number(length))
+ const extended = rest.u8()
+ if (extended === 0x01) {
+ endSequence = true
+ } else if (extended === 0x02) {
+ if (p.addressSize !== 1 && p.addressSize !== 2 && p.addressSize !== 4 && p.addressSize !== 8) {
+ throw new DwarfError('unsupported address size')
+ }
+ const value = uintBig(rest, p.addressSize)
+ tombstone = value < address || value >= minTombstone
+ if (!tombstone) {
+ address = value
+ opIndex = 0n
+ }
+ } else if (extended === 0x03) {
+ const path = rest.cstr()
+ const dirIndex = uleb64(rest)
+ uleb64(rest) // mtime
+ uleb64(rest) // length
+ files.push({ path: { k: 'string', v: path }, dirIndex: Number(dirIndex) })
+ } else if (extended === 0x04) {
+ uleb64(rest)
+ }
+ } else if (opcode >= p.opcodeBase) {
+ const adjusted = BigInt(opcode - p.opcodeBase)
+ advance(adjusted / BigInt(p.lineRange))
+ } else if (opcode === 0x02) {
+ advance(uleb64(c))
+ } else if (opcode === 0x03) {
+ skipSleb64(c)
+ } else if (opcode === 0x04 || opcode === 0x05 || opcode === 0x0c) {
+ uleb64(c)
+ } else if (opcode === 0x08) {
+ advance(BigInt(255 - p.opcodeBase) / BigInt(p.lineRange))
+ } else if (opcode === 0x09) {
+ const operand = BigInt(c.u16())
+ if (!tombstone) {
+ addSized(operand)
+ opIndex = 0n
+ }
+ } else if (opcode > 0x0c) {
+ const args = p.opcodeLengths.buf[p.opcodeLengths.pos + opcode - 1]!
+ for (let i = 0; i < args; i++) {
+ uleb64(c)
+ }
+ }
+ if (endSequence) {
+ address = 0n
+ opIndex = 0n
+ tombstone = false
+ }
+ }
+ } catch (error) {
+ if (!(error instanceof DwarfError)) {
+ throw error
+ }
+ }
+ return files
+}
+
function entryFormat(c: Cursor): { type: number; form: number }[] {
const count = c.u8()
const out: { type: number; form: number }[] = []
diff --git a/hog/src/error-tracking/native/walk.test.ts b/hog/src/error-tracking/native/walk.test.ts
new file mode 100644
index 000000000000..b0f3eecc7c09
--- /dev/null
+++ b/hog/src/error-tracking/native/walk.test.ts
@@ -0,0 +1,59 @@
+import { describe, expect, test } from 'bun:test'
+
+import { dwarf4Object, op } from './dwarf-fixture'
+import { BufferSource } from './object'
+import { extractSourcePaths } from './walk'
+
+const MAX = (1n << 64n) - 1n
+const HEADER_FILES = ['/src/proj/include/util.h', '/src/proj/main.c']
+
+// The expected paths come from posthog-cli's extract_source_paths (symbolic) on the same bytes.
+describe('DW_LNE_define_file', () => {
+ test.each([
+ [
+ 'adds files the program defines',
+ [
+ ...op.setAddress(0x1000n),
+ ...op.defineFile('defined.c', 0),
+ ...op.copy,
+ ...op.defineFile('extra.h', 1),
+ ...op.endSequence,
+ ],
+ ['/src/proj/defined.c', '/src/proj/include/extra.h'],
+ ],
+ [
+ 'stops at the first address overflow',
+ [
+ ...op.setAddress(MAX - 0x10n),
+ ...op.defineFile('before.c', 0),
+ ...op.advancePc(0x20n),
+ ...op.defineFile('after.c', 0),
+ ],
+ ['/src/proj/before.c'],
+ ],
+ [
+ 'skips address arithmetic after a tombstone address',
+ [
+ ...op.setAddress(0x2000n),
+ ...op.copy,
+ ...op.setAddress(0x10n),
+ ...op.advancePc(MAX),
+ ...op.defineFile('tomb.c', 0),
+ ],
+ ['/src/proj/tomb.c'],
+ ],
+ [
+ 'resets the address after an end of sequence',
+ [...op.setAddress(MAX - 0x10n), ...op.endSequence, ...op.advancePc(0x20n), ...op.defineFile('reset.c', 0)],
+ ['/src/proj/reset.c'],
+ ],
+ [
+ 'keeps the files defined before a truncated instruction',
+ [...op.setAddress(0x1000n), ...op.defineFile('ok.c', 0), 0, 0x7f, 0x03],
+ ['/src/proj/ok.c'],
+ ],
+ ])('%s', (_name, program, defined) => {
+ const paths = extractSourcePaths(new BufferSource(dwarf4Object(program)))
+ expect(paths).toEqual([...HEADER_FILES, ...defined].sort())
+ })
+})