:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hog): write zip64 archives for binaries with many sources (#125182)
Rafael Audibertf0af36b88447
diff --git a/hog/README.md b/hog/README.mdindex e9d841ae402d..6f2dc0fa3cc6 100644--- a/hog/README.md+++ b/hog/README.md@@ -46,10 +46,6 @@ Then, once the commands above work:- [ ] Callers: the SDKs, build plugins and scripts that run `posthog-cli` with its arguments. Move them to hog, or give hog a `posthog-cli` entry point that translates the arguments.- [ ] Distribution: a binary for each platform, macOS signing and notarization, the npm package, the Homebrew formula, `hog update`, and CI on Linux and Windows.-Smaller follow-ups, none of them blocking:--- [ ] `hog error-tracking upload --include-source`: write ZIP64, so a binary that names more than 65,533 source files keeps its sources. hog uploads it without sources and warns.-Not planned:- `posthog-cli exp query editor`, the interactive query editor in the terminal. hog won't port it.diff --git a/hog/src/error-tracking/native/sources.ts b/hog/src/error-tracking/native/sources.tsindex c58fe49f1855..5c679b4260df 100644--- a/hog/src/error-tracking/native/sources.ts+++ b/hog/src/error-tracking/native/sources.ts@@ -9,7 +9,6 @@ import type { ZipWriter } from './zip'// manifest from DWARF path to zip path. posthog-cli's dsym/source_bundle.rs.const MAX_SOURCE_FILE_BYTES = 64 * 1024 * 1024-const MAX_ZIP_ENTRIES = 0xffffexport interface SourceBundle {manifest: Map<string, string>@@ -180,13 +179,5 @@ export function sourceStep(return undefined}const bundle = collectSources(paths, warnings)- // The sources, the manifest and the binary must fit a zip without ZIP64. posthog-cli's zip- // crate switches to ZIP64 there, which hog doesn't write, so it uploads the binary alone.- if (bundle.contents.size + 2 > MAX_ZIP_ENTRIES) {- warnings.push(- `The binary names ${bundle.contents.size} source files, more than one upload can hold, so hog uploads it without sources.`- )- return undefined- }return (zip) => addSources(zip, bundle)}diff --git a/hog/src/error-tracking/native/zip.test.ts b/hog/src/error-tracking/native/zip.test.tsnew file mode 100644index 000000000000..c74db297f2d5--- /dev/null+++ b/hog/src/error-tracking/native/zip.test.ts@@ -0,0 +1,28 @@+import { describe, expect, test } from 'bun:test'+import { mkdtempSync, readdirSync } from 'node:fs'+import { tmpdir } from 'node:os'+import { join } from 'node:path'++import { SymbolDataType, writeSymbolData } from '../container'+import { extractSymbolSet } from '../symbols'+import { ZipWriter } from './zip'++describe('ZipWriter', () => {+ test('writes and reads an archive with more entries than a zip without ZIP64 holds', () => {+ const zip = new ZipWriter()+ zip.add('dwarf', Buffer.from('not really dwarf'))+ zip.add('__source/manifest.json', Buffer.from('{}'))+ for (let i = 0; i < 65534; i++) {+ zip.add(`__source/f${i}.c`, Buffer.from(`int f${i};\n`))+ }+ const archive = zip.finish()++ // The zip crate 2.4.2 posthog-cli uses writes these exact bytes for the same entries.+ expect(new Bun.CryptoHasher('sha256').update(archive).digest('hex')).toBe(+ '3d769dba09591dbe4236fc8b22551102700886babca1c94c6ff684bc5beea2f0'+ )+ const folder = mkdtempSync(join(tmpdir(), 'hog-zip64-'))+ extractSymbolSet(writeSymbolData(SymbolDataType.AppleDsym, archive), 'Big', folder, folder)+ expect(readdirSync(join(folder, 'Big/__source')).length).toBe(65535)+ }, 30_000)+})diff --git a/hog/src/error-tracking/native/zip.ts b/hog/src/error-tracking/native/zip.tsindex 2f865440ea3e..22514a93ea35 100644--- a/hog/src/error-tracking/native/zip.ts+++ b/hog/src/error-tracking/native/zip.ts@@ -7,6 +7,9 @@ import { deflateRaw } from 'pako'const LOCAL_HEADER = 0x04034b50const CENTRAL_HEADER = 0x02014b50const END_OF_CENTRAL_DIRECTORY = 0x06054b50+const ZIP64_END_OF_CENTRAL_DIRECTORY = 0x06064b50+const ZIP64_END_LOCATOR = 0x07064b50+const MAX_ENTRIES = 0xffffconst VERSION_NEEDED = 20const VERSION_MADE_BY = 0x0314const DEFLATED = 8@@ -86,8 +89,10 @@ export class ZipWriter {}finish(): Buffer {- if (this.entries.length > 0xffff || this.length > 0xffffffff) {- throw new ZipError('Zip64 archives are not supported')+ // More than 65,535 entries is the only ZIP64 case hog writes. Offsets past 4 GiB would also+ // need ZIP64 fields in each entry.+ if (this.length > 0xffffffff) {+ throw new ZipError('Zip64 archives over 4 GiB are not supported')}const directoryOffset = this.lengthfor (const entry of this.entries) {@@ -108,10 +113,27 @@ export class ZipWriter {this.push(header, entry.name)}const directorySize = this.length - directoryOffset+ if (this.entries.length > MAX_ENTRIES) {+ const record = Buffer.alloc(56)+ record.writeUInt32LE(ZIP64_END_OF_CENTRAL_DIRECTORY, 0)+ record.writeBigUInt64LE(44n, 4)+ record.writeUInt16LE(VERSION_NEEDED, 12)+ record.writeUInt16LE(VERSION_NEEDED, 14)+ record.writeBigUInt64LE(BigInt(this.entries.length), 24)+ record.writeBigUInt64LE(BigInt(this.entries.length), 32)+ record.writeBigUInt64LE(BigInt(directorySize), 40)+ record.writeBigUInt64LE(BigInt(directoryOffset), 48)+ const locator = Buffer.alloc(20)+ locator.writeUInt32LE(ZIP64_END_LOCATOR, 0)+ locator.writeBigUInt64LE(BigInt(directoryOffset + directorySize), 8)+ locator.writeUInt32LE(1, 16)+ this.push(record, locator)+ }+ const count = Math.min(this.entries.length, MAX_ENTRIES)const end = Buffer.alloc(22)end.writeUInt32LE(END_OF_CENTRAL_DIRECTORY, 0)- end.writeUInt16LE(this.entries.length, 8)- end.writeUInt16LE(this.entries.length, 10)+ end.writeUInt16LE(count, 8)+ end.writeUInt16LE(count, 10)end.writeUInt32LE(directorySize, 12)end.writeUInt32LE(directoryOffset, 16)this.push(end)diff --git a/hog/src/error-tracking/symbols.ts b/hog/src/error-tracking/symbols.tsindex e4efef21c69d..da78c4209c2d 100644--- a/hog/src/error-tracking/symbols.ts+++ b/hog/src/error-tracking/symbols.ts@@ -128,8 +128,19 @@ function readZip(zip: Buffer): ZipEntry[] {if (end < 0 || end + 22 > zip.length) {throw unreadable('no end of central directory')}- const count = zip.readUInt16LE(end + 10)+ let count = zip.readUInt16LE(end + 10)let offset = zip.readUInt32LE(end + 16)+ // An archive with more than 65,535 entries keeps the real count in the ZIP64 record, which the+ // locator just before the end record points at.+ const locator = end - 20+ if (locator >= 0 && zip.readUInt32LE(locator) === 0x07064b50) {+ const record = Number(zip.readBigUInt64LE(locator + 8))+ if (record + 56 > zip.length || zip.readUInt32LE(record) !== 0x06064b50) {+ throw unreadable('the ZIP64 end of central directory is out of bounds')+ }+ count = Number(zip.readBigUInt64LE(record + 32))+ offset = Number(zip.readBigUInt64LE(record + 48))+ }let total = 0const entries: ZipEntry[] = []for (let index = 0; index < count; index++) {