:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hog): bundle the source files a line program defines (#125181)
Rafael Audibert8c1ecd20aa4c
diff --git a/hog/README.md b/hog/README.mdindex 91cc96aebd9c..e9d841ae402d 100644--- a/hog/README.md+++ b/hog/README.md@@ -48,7 +48,6 @@ Then, once the commands above work:Smaller follow-ups, none of them blocking:-- [ ] `hog error-tracking upload --include-source`: read `DW_LNE_define_file` entries in the DWARF line program, so hog bundles the same source files as posthog-cli. Current compilers don't emit them, and DWARF 5 removed them.- [ ] `hog error-tracking upload --include-source`: write ZIP64, so a binary that names more than 65,533 source files keeps its sources. hog uploads it without sources and warns.Not planned:diff --git a/hog/src/error-tracking/native/dwarf-fixture.ts b/hog/src/error-tracking/native/dwarf-fixture.tsnew file mode 100644index 000000000000..d434a6731850--- /dev/null+++ b/hog/src/error-tracking/native/dwarf-fixture.ts@@ -0,0 +1,113 @@+// Builds a relocatable x86-64 ELF with one DWARF 4 compile unit and a hand-written line program,+// for line-program cases no compiler emits. Tests only.++const cstr = (text: string): number[] => [...Buffer.from(text, 'utf8'), 0]++export function uleb(value: bigint | number): number[] {+ let v = BigInt(value)+ const out: number[] = []+ do {+ let byte = Number(v & 0x7fn)+ v >>= 7n+ if (v !== 0n) {+ byte |= 0x80+ }+ out.push(byte)+ } while (v !== 0n)+ return out+}++const u16 = (v: number): number[] => [v & 0xff, (v >> 8) & 0xff]+const u32 = (v: number): number[] => [v & 0xff, (v >> 8) & 0xff, (v >> 16) & 0xff, (v >>> 24) & 0xff]+export const u64 = (v: bigint): number[] => Array.from({ length: 8 }, (_, i) => Number((v >> BigInt(8 * i)) & 0xffn))++export const op = {+ setAddress: (address: bigint): number[] => [0, 9, 0x02, ...u64(address)],+ defineFile: (path: string, dir: number): number[] => {+ const body = [0x03, ...cstr(path), ...uleb(dir), 0, 0]+ return [0, ...uleb(body.length), ...body]+ },+ advancePc: (by: bigint): number[] => [0x02, ...uleb(by)],+ copy: [0x01],+ endSequence: [0, 1, 0x01],+}++export function dwarf4Object(program: number[]): Uint8Array {+ const abbrev = [1, 0x11, 0, 0x03, 0x08, 0x1b, 0x08, 0x10, 0x17, 0x11, 0x01, 0, 0, 0]+ const die = [1, ...cstr('/src/proj/main.c'), ...cstr('/src/proj'), ...u32(0), ...u64(0x1000n)]+ const infoBody = [...u16(4), ...u32(0), 8, ...die]+ const info = [...u32(infoBody.length), ...infoBody]++ // minimum_instruction_length, maximum_operations_per_instruction, default_is_stmt,+ // line_base (-5), line_range, opcode_base, then the standard opcode lengths.+ const encoding = [1, 1, 1, 0xfb, 14, 13]+ const opcodeLengths = [0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1]+ const file = (name: string, dir: number): number[] => [...cstr(name), dir, 0, 0]+ const header = [+ ...encoding,+ ...opcodeLengths,+ ...cstr('/src/proj/include'),+ 0,+ ...file('main.c', 0),+ ...file('util.h', 1),+ 0,+ ]+ const lineBody = [...u16(4), ...u32(header.length), ...header, ...program]+ const line = [...u32(lineBody.length), ...lineBody]++ const names = ['', '.debug_abbrev', '.debug_info', '.debug_line', '.shstrtab']+ const shstrtab: number[] = []+ const nameOffsets = names.map((name) => {+ const at = shstrtab.length+ shstrtab.push(...cstr(name))+ return at+ })+ const contents = [abbrev, info, line, shstrtab]+ const body: number[] = []+ const offsets = contents.map((data) => {+ const at = 64 + body.length+ body.push(...data)+ return at+ })+ while ((64 + body.length) % 8 !== 0) {+ body.push(0)+ }+ const sectionHeadersAt = 64 + body.length+ const sectionHeader = (index: number): number[] => {+ if (index === 0) {+ return Array.from({ length: 64 }, () => 0)+ }+ const type = names[index] === '.shstrtab' ? 3 : 1+ return [+ ...u32(nameOffsets[index]!),+ ...u32(type),+ ...u64(0n),+ ...u64(0n),+ ...u64(BigInt(offsets[index - 1]!)),+ ...u64(BigInt(contents[index - 1]!.length)),+ ...u32(0),+ ...u32(0),+ ...u64(1n),+ ...u64(0n),+ ]+ }+ // ELF64, little-endian, version 1.+ const ident = [0x7f, 0x45, 0x4c, 0x46, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]+ const elfHeader = [+ ...ident,+ ...u16(1), // ET_REL+ ...u16(62), // EM_X86_64+ ...u32(1),+ ...u64(0n),+ ...u64(0n),+ ...u64(BigInt(sectionHeadersAt)),+ ...u32(0),+ ...u16(64),+ ...u16(0),+ ...u16(0),+ ...u16(64),+ ...u16(names.length),+ ...u16(names.length - 1),+ ]+ return Uint8Array.from([...elfHeader, ...body, ...names.flatMap((_, i) => sectionHeader(i))])+}diff --git a/hog/src/error-tracking/native/dwarf.ts b/hog/src/error-tracking/native/dwarf.tsindex a4a987f5e314..194b7e5418d2 100644--- a/hog/src/error-tracking/native/dwarf.ts+++ b/hog/src/error-tracking/native/dwarf.ts@@ -548,7 +548,8 @@ export function parseLineHeader(}const headerLength = rest.offset(is64)const hdr = rest.split(headerLength)- if (hdr.u8() === 0) {+ const minInstLength = hdr.u8()+ if (minInstLength === 0) {throw new DwarfError('minimum instruction length zero')}const maxOps = version >= 4 ? hdr.u8() : 1@@ -557,14 +558,15 @@ export function parseLineHeader(}hdr.u8() // default_is_stmthdr.i8() // line_base- if (hdr.u8() === 0) {+ const lineRange = hdr.u8()+ if (lineRange === 0) {throw new DwarfError('line range zero')}const opcodeBase = hdr.u8()if (opcodeBase === 0) {throw new DwarfError('opcode base zero')}- hdr.split(opcodeBase - 1)+ const opcodeLengths = hdr.split(opcodeBase - 1)const enc = { is64, version, addressSize }const includeDirs: AttrValue[] = []@@ -587,6 +589,10 @@ export function parseLineHeader(hdr.uleb() // lengthfiles.push({ path: { k: 'string', v: p }, dirIndex })}+ const programStart = offset + lenSize + fixed + headerLength+ const program = new Cursor(s.line.read(programStart, unitLength - fixed - headerLength), s.le)+ const params = { minInstLength, maxOps, lineRange, opcodeBase, opcodeLengths, addressSize }+ files.push(...definedFiles(program, params))return { version, compDir, includeDirs, files }}const dirFormat = entryFormat(hdr)@@ -620,6 +626,163 @@ export function parseLineHeader(return { version, compDir: null, includeDirs, files }}+interface LineProgramParams {+ minInstLength: number+ maxOps: number+ lineRange: number+ opcodeBase: number+ opcodeLengths: Cursor+ addressSize: number+}++const U64_MAX = (1n << 64n) - 1n++/** gimli `leb128::read::unsigned`, exact to 64 bits. */+function uleb64(c: Cursor): bigint {+ let result = 0n+ let shift = 0n+ for (;;) {+ const byte = c.u8()+ if (shift === 63n && byte !== 0x00 && byte !== 0x01) {+ throw new DwarfError('bad unsigned LEB128')+ }+ result |= BigInt(byte & 0x7f) << shift+ if ((byte & 0x80) === 0) {+ return result & U64_MAX+ }+ shift += 7n+ }+}++function uintBig(c: Cursor, size: number): bigint {+ const bytes = c.split(size)+ let value = 0n+ for (let i = 0; i < size; i++) {+ value = (value << 8n) | BigInt(bytes.buf[bytes.pos + (c.le ? size - 1 - i : i)]!)+ }+ return value+}++/** gimli `leb128::read::signed`. Only its errors matter here, so it returns nothing. */+function skipSleb64(c: Cursor): void {+ let shift = 0+ for (;;) {+ const byte = c.u8()+ if (shift === 63 && byte !== 0x00 && byte !== 0x7f) {+ throw new DwarfError('bad signed LEB128')+ }+ shift += 7+ if ((byte & 0x80) === 0) {+ return+ }+ }+}++/**+ * The files `DW_LNE_define_file` adds while the program runs. symbolic runs every line program+ * (`DwarfLineProgram::prepare`) until its end or its first error, and gimli appends each defined+ * file to the header's list. Address arithmetic can raise that error, so it is tracked exactly.+ */+function definedFiles(c: Cursor, p: LineProgramParams): LineFile[] {+ const files: LineFile[] = []+ const ones = p.addressSize === 8 ? U64_MAX : (1n << BigInt(8 * p.addressSize)) - 1n+ const minTombstone = (ones - 1n) & ones+ let address = 0n+ let opIndex = 0n+ let tombstone = false+ const addSized = (length: bigint): void => {+ const sum = address + length+ if (sum > U64_MAX || (sum & ~ones) !== 0n) {+ throw new DwarfError('address overflow')+ }+ address = sum+ }+ const advance = (operationAdvance: bigint): void => {+ if (tombstone) {+ return+ }+ const minInst = BigInt(p.minInstLength)+ const maxOps = BigInt(p.maxOps)+ let addressAdvance: bigint+ if (maxOps === 1n) {+ opIndex = 0n+ addressAdvance = (minInst * operationAdvance) & U64_MAX+ } else {+ const withAdvance = (opIndex + operationAdvance) & U64_MAX+ opIndex = withAdvance % maxOps+ addressAdvance = (minInst * (withAdvance / maxOps)) & U64_MAX+ }+ addSized(addressAdvance)+ }+ try {+ while (!c.isEmpty()) {+ const opcode = c.u8()+ let endSequence = false+ if (opcode === 0) {+ const length = uleb64(c)+ if (length > BigInt(c.remaining)) {+ throw new DwarfEof()+ }+ const rest = c.split(Number(length))+ const extended = rest.u8()+ if (extended === 0x01) {+ endSequence = true+ } else if (extended === 0x02) {+ if (p.addressSize !== 1 && p.addressSize !== 2 && p.addressSize !== 4 && p.addressSize !== 8) {+ throw new DwarfError('unsupported address size')+ }+ const value = uintBig(rest, p.addressSize)+ tombstone = value < address || value >= minTombstone+ if (!tombstone) {+ address = value+ opIndex = 0n+ }+ } else if (extended === 0x03) {+ const path = rest.cstr()+ const dirIndex = uleb64(rest)+ uleb64(rest) // mtime+ uleb64(rest) // length+ files.push({ path: { k: 'string', v: path }, dirIndex: Number(dirIndex) })+ } else if (extended === 0x04) {+ uleb64(rest)+ }+ } else if (opcode >= p.opcodeBase) {+ const adjusted = BigInt(opcode - p.opcodeBase)+ advance(adjusted / BigInt(p.lineRange))+ } else if (opcode === 0x02) {+ advance(uleb64(c))+ } else if (opcode === 0x03) {+ skipSleb64(c)+ } else if (opcode === 0x04 || opcode === 0x05 || opcode === 0x0c) {+ uleb64(c)+ } else if (opcode === 0x08) {+ advance(BigInt(255 - p.opcodeBase) / BigInt(p.lineRange))+ } else if (opcode === 0x09) {+ const operand = BigInt(c.u16())+ if (!tombstone) {+ addSized(operand)+ opIndex = 0n+ }+ } else if (opcode > 0x0c) {+ const args = p.opcodeLengths.buf[p.opcodeLengths.pos + opcode - 1]!+ for (let i = 0; i < args; i++) {+ uleb64(c)+ }+ }+ if (endSequence) {+ address = 0n+ opIndex = 0n+ tombstone = false+ }+ }+ } catch (error) {+ if (!(error instanceof DwarfError)) {+ throw error+ }+ }+ return files+}+function entryFormat(c: Cursor): { type: number; form: number }[] {const count = c.u8()const out: { type: number; form: number }[] = []diff --git a/hog/src/error-tracking/native/walk.test.ts b/hog/src/error-tracking/native/walk.test.tsnew file mode 100644index 000000000000..b0f3eecc7c09--- /dev/null+++ b/hog/src/error-tracking/native/walk.test.ts@@ -0,0 +1,59 @@+import { describe, expect, test } from 'bun:test'++import { dwarf4Object, op } from './dwarf-fixture'+import { BufferSource } from './object'+import { extractSourcePaths } from './walk'++const MAX = (1n << 64n) - 1n+const HEADER_FILES = ['/src/proj/include/util.h', '/src/proj/main.c']++// The expected paths come from posthog-cli's extract_source_paths (symbolic) on the same bytes.+describe('DW_LNE_define_file', () => {+ test.each([+ [+ 'adds files the program defines',+ [+ ...op.setAddress(0x1000n),+ ...op.defineFile('defined.c', 0),+ ...op.copy,+ ...op.defineFile('extra.h', 1),+ ...op.endSequence,+ ],+ ['/src/proj/defined.c', '/src/proj/include/extra.h'],+ ],+ [+ 'stops at the first address overflow',+ [+ ...op.setAddress(MAX - 0x10n),+ ...op.defineFile('before.c', 0),+ ...op.advancePc(0x20n),+ ...op.defineFile('after.c', 0),+ ],+ ['/src/proj/before.c'],+ ],+ [+ 'skips address arithmetic after a tombstone address',+ [+ ...op.setAddress(0x2000n),+ ...op.copy,+ ...op.setAddress(0x10n),+ ...op.advancePc(MAX),+ ...op.defineFile('tomb.c', 0),+ ],+ ['/src/proj/tomb.c'],+ ],+ [+ 'resets the address after an end of sequence',+ [...op.setAddress(MAX - 0x10n), ...op.endSequence, ...op.advancePc(0x20n), ...op.defineFile('reset.c', 0)],+ ['/src/proj/reset.c'],+ ],+ [+ 'keeps the files defined before a truncated instruction',+ [...op.setAddress(0x1000n), ...op.defineFile('ok.c', 0), 0, 0x7f, 0x03],+ ['/src/proj/ok.c'],+ ],+ ])('%s', (_name, program, defined) => {+ const paths = extractSourcePaths(new BufferSource(dwarf4Object(program)))+ expect(paths).toEqual([...HEADER_FILES, ...defined].sort())+ })+})