Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hog): write zip64 archives for binaries with many sources (#125182)

Rafael Audibertf0af36b88447
diff --git a/hog/README.md b/hog/README.md
index e9d841ae402d..6f2dc0fa3cc6 100644
--- a/hog/README.md
+++ b/hog/README.md
@@ -46,10 +46,6 @@ Then, once the commands above work:
- [ ] Callers: the SDKs, build plugins and scripts that run `posthog-cli` with its arguments. Move them to hog, or give hog a `posthog-cli` entry point that translates the arguments.
- [ ] Distribution: a binary for each platform, macOS signing and notarization, the npm package, the Homebrew formula, `hog update`, and CI on Linux and Windows.
-Smaller follow-ups, none of them blocking:
-
-- [ ] `hog error-tracking upload --include-source`: write ZIP64, so a binary that names more than 65,533 source files keeps its sources. hog uploads it without sources and warns.
-
Not planned:
- `posthog-cli exp query editor`, the interactive query editor in the terminal. hog won't port it.
diff --git a/hog/src/error-tracking/native/sources.ts b/hog/src/error-tracking/native/sources.ts
index c58fe49f1855..5c679b4260df 100644
--- a/hog/src/error-tracking/native/sources.ts
+++ b/hog/src/error-tracking/native/sources.ts
@@ -9,7 +9,6 @@ import type { ZipWriter } from './zip'
// manifest from DWARF path to zip path. posthog-cli's dsym/source_bundle.rs.
const MAX_SOURCE_FILE_BYTES = 64 * 1024 * 1024
-const MAX_ZIP_ENTRIES = 0xffff
export interface SourceBundle {
manifest: Map<string, string>
@@ -180,13 +179,5 @@ export function sourceStep(
return undefined
}
const bundle = collectSources(paths, warnings)
- // The sources, the manifest and the binary must fit a zip without ZIP64. posthog-cli's zip
- // crate switches to ZIP64 there, which hog doesn't write, so it uploads the binary alone.
- if (bundle.contents.size + 2 > MAX_ZIP_ENTRIES) {
- warnings.push(
- `The binary names ${bundle.contents.size} source files, more than one upload can hold, so hog uploads it without sources.`
- )
- return undefined
- }
return (zip) => addSources(zip, bundle)
}
diff --git a/hog/src/error-tracking/native/zip.test.ts b/hog/src/error-tracking/native/zip.test.ts
new file mode 100644
index 000000000000..c74db297f2d5
--- /dev/null
+++ b/hog/src/error-tracking/native/zip.test.ts
@@ -0,0 +1,28 @@
+import { describe, expect, test } from 'bun:test'
+import { mkdtempSync, readdirSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+import { SymbolDataType, writeSymbolData } from '../container'
+import { extractSymbolSet } from '../symbols'
+import { ZipWriter } from './zip'
+
+describe('ZipWriter', () => {
+ test('writes and reads an archive with more entries than a zip without ZIP64 holds', () => {
+ const zip = new ZipWriter()
+ zip.add('dwarf', Buffer.from('not really dwarf'))
+ zip.add('__source/manifest.json', Buffer.from('{}'))
+ for (let i = 0; i < 65534; i++) {
+ zip.add(`__source/f${i}.c`, Buffer.from(`int f${i};\n`))
+ }
+ const archive = zip.finish()
+
+ // The zip crate 2.4.2 posthog-cli uses writes these exact bytes for the same entries.
+ expect(new Bun.CryptoHasher('sha256').update(archive).digest('hex')).toBe(
+ '3d769dba09591dbe4236fc8b22551102700886babca1c94c6ff684bc5beea2f0'
+ )
+ const folder = mkdtempSync(join(tmpdir(), 'hog-zip64-'))
+ extractSymbolSet(writeSymbolData(SymbolDataType.AppleDsym, archive), 'Big', folder, folder)
+ expect(readdirSync(join(folder, 'Big/__source')).length).toBe(65535)
+ }, 30_000)
+})
diff --git a/hog/src/error-tracking/native/zip.ts b/hog/src/error-tracking/native/zip.ts
index 2f865440ea3e..22514a93ea35 100644
--- a/hog/src/error-tracking/native/zip.ts
+++ b/hog/src/error-tracking/native/zip.ts
@@ -7,6 +7,9 @@ import { deflateRaw } from 'pako'
const LOCAL_HEADER = 0x04034b50
const CENTRAL_HEADER = 0x02014b50
const END_OF_CENTRAL_DIRECTORY = 0x06054b50
+const ZIP64_END_OF_CENTRAL_DIRECTORY = 0x06064b50
+const ZIP64_END_LOCATOR = 0x07064b50
+const MAX_ENTRIES = 0xffff
const VERSION_NEEDED = 20
const VERSION_MADE_BY = 0x0314
const DEFLATED = 8
@@ -86,8 +89,10 @@ export class ZipWriter {
}
finish(): Buffer {
- if (this.entries.length > 0xffff || this.length > 0xffffffff) {
- throw new ZipError('Zip64 archives are not supported')
+ // More than 65,535 entries is the only ZIP64 case hog writes. Offsets past 4 GiB would also
+ // need ZIP64 fields in each entry.
+ if (this.length > 0xffffffff) {
+ throw new ZipError('Zip64 archives over 4 GiB are not supported')
}
const directoryOffset = this.length
for (const entry of this.entries) {
@@ -108,10 +113,27 @@ export class ZipWriter {
this.push(header, entry.name)
}
const directorySize = this.length - directoryOffset
+ if (this.entries.length > MAX_ENTRIES) {
+ const record = Buffer.alloc(56)
+ record.writeUInt32LE(ZIP64_END_OF_CENTRAL_DIRECTORY, 0)
+ record.writeBigUInt64LE(44n, 4)
+ record.writeUInt16LE(VERSION_NEEDED, 12)
+ record.writeUInt16LE(VERSION_NEEDED, 14)
+ record.writeBigUInt64LE(BigInt(this.entries.length), 24)
+ record.writeBigUInt64LE(BigInt(this.entries.length), 32)
+ record.writeBigUInt64LE(BigInt(directorySize), 40)
+ record.writeBigUInt64LE(BigInt(directoryOffset), 48)
+ const locator = Buffer.alloc(20)
+ locator.writeUInt32LE(ZIP64_END_LOCATOR, 0)
+ locator.writeBigUInt64LE(BigInt(directoryOffset + directorySize), 8)
+ locator.writeUInt32LE(1, 16)
+ this.push(record, locator)
+ }
+ const count = Math.min(this.entries.length, MAX_ENTRIES)
const end = Buffer.alloc(22)
end.writeUInt32LE(END_OF_CENTRAL_DIRECTORY, 0)
- end.writeUInt16LE(this.entries.length, 8)
- end.writeUInt16LE(this.entries.length, 10)
+ end.writeUInt16LE(count, 8)
+ end.writeUInt16LE(count, 10)
end.writeUInt32LE(directorySize, 12)
end.writeUInt32LE(directoryOffset, 16)
this.push(end)
diff --git a/hog/src/error-tracking/symbols.ts b/hog/src/error-tracking/symbols.ts
index e4efef21c69d..da78c4209c2d 100644
--- a/hog/src/error-tracking/symbols.ts
+++ b/hog/src/error-tracking/symbols.ts
@@ -128,8 +128,19 @@ function readZip(zip: Buffer): ZipEntry[] {
if (end < 0 || end + 22 > zip.length) {
throw unreadable('no end of central directory')
}
- const count = zip.readUInt16LE(end + 10)
+ let count = zip.readUInt16LE(end + 10)
let offset = zip.readUInt32LE(end + 16)
+ // An archive with more than 65,535 entries keeps the real count in the ZIP64 record, which the
+ // locator just before the end record points at.
+ const locator = end - 20
+ if (locator >= 0 && zip.readUInt32LE(locator) === 0x07064b50) {
+ const record = Number(zip.readBigUInt64LE(locator + 8))
+ if (record + 56 > zip.length || zip.readUInt32LE(record) !== 0x06064b50) {
+ throw unreadable('the ZIP64 end of central directory is out of bounds')
+ }
+ count = Number(zip.readBigUInt64LE(record + 32))
+ offset = Number(zip.readBigUInt64LE(record + 48))
+ }
let total = 0
const entries: ZipEntry[] = []
for (let index = 0; index < count; index++) {