Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
fix(cli): verify codex-acp tarballs against the lockfile before packaging them
composio-zen[bot]2d409c7417d0
diff --git a/ts/packages/cli/scripts/_acp-adapters.ts b/ts/packages/cli/scripts/_acp-adapters.tsindex c5a83fbca..78cac9f47 100644--- a/ts/packages/cli/scripts/_acp-adapters.ts+++ b/ts/packages/cli/scripts/_acp-adapters.ts@@ -1,10 +1,16 @@import { chmod, copyFile, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises';import { createRequire } from 'node:module';import * as path from 'node:path';+import { fileURLToPath } from 'node:url';import {RUN_CODEX_ACP_BINARY_TARGETS,type RunCodexAcpBinaryTarget,} from '../src/services/run-companion-modules';+import {+ assertBytesMatchIntegrity,+ expectedIntegrityFor,+ loadLockfileIntegrity,+} from './_tarball-integrity';const GENERATED_ROOT_DIR = path.resolve('./.generated');const ACP_ADAPTERS_CACHE_DIR = path.join(GENERATED_ROOT_DIR, 'acp-adapters');@@ -231,6 +237,7 @@ const buildClaudeAdapterBundle = async (cacheDir: string): Promise<void> => {const extractCodexBinaryFromTarball = async ({binaryFileName,+ integrityByPackage,packageName,relativePath,stageCacheDir,@@ -238,12 +245,16 @@ const extractCodexBinaryFromTarball = async ({version,}: {readonly binaryFileName: string;+ readonly integrityByPackage: ReadonlyMap<string, string>;readonly packageName: string;readonly relativePath: string;readonly stageCacheDir: string;readonly stageRootDir: string;readonly version: string;}): Promise<void> => {+ // Read before the download so a missing pin fails before any bytes are fetched.+ const expectedIntegrity = expectedIntegrityFor(integrityByPackage, packageName, version);+const metadata = await fetchRegistryJson(packageName, version);const tarballUrl = metadata.dist?.tarball;if (!tarballUrl) {@@ -262,8 +273,14 @@ const extractCodexBinaryFromTarball = async ({);await downloadTarball(tarballUrl, tempArchivePath);++ // Checked against the lockfile rather than the registry's own `dist.integrity`:+ // whoever can serve the tarball can serve the metadata that vouches for it.+ const archiveBytes = await Bun.file(tempArchivePath).bytes();+ await assertBytesMatchIntegrity(archiveBytes, expectedIntegrity, `${packageName}@${version}`);+await mkdir(tempExtractDir, { recursive: true });- const archive = new Bun.Archive(await Bun.file(tempArchivePath).bytes());+ const archive = new Bun.Archive(archiveBytes);await archive.extract(tempExtractDir);const extractedBinaryPath = path.join(tempExtractDir, 'package', 'bin', binaryFileName);@@ -284,6 +301,10 @@ const materializeCodexAdapters = async (stageRootDir: string,stageCacheDir: string): Promise<void> => {+ const integrityByPackage = await loadLockfileIntegrity(+ path.dirname(fileURLToPath(import.meta.url))+ );+await Promise.all(manifest.codex.map(target =>extractCodexBinaryFromTarball({@@ -291,6 +312,7 @@ const materializeCodexAdapters = async (RUN_CODEX_ACP_BINARY_TARGETS.find(candidate => candidate.packageName === target.packageName)?.binaryFileName ?? 'codex-acp',+ integrityByPackage,packageName: target.packageName,relativePath: target.relativePath,stageCacheDir,diff --git a/ts/packages/cli/scripts/_tarball-integrity.ts b/ts/packages/cli/scripts/_tarball-integrity.tsnew file mode 100644index 000000000..3badcc194--- /dev/null+++ b/ts/packages/cli/scripts/_tarball-integrity.ts@@ -0,0 +1,173 @@+import { readFile, stat } from 'node:fs/promises';+import * as path from 'node:path';++/**+ * Integrity verification for tarballs this repo downloads outside of pnpm.+ *+ * `pnpm install --frozen-lockfile` verifies every dependency it installs+ * against the `integrity` recorded in `pnpm-lock.yaml`. A tarball fetched by a+ * build script gets none of that: the script asks the registry where the bytes+ * live and writes whatever comes back. The codex-acp binaries are fetched that+ * way — they must be present for all four platforms on a single-platform runner,+ * which pnpm will not install — and they are then packaged, executable, inside+ * the published CLI release archive.+ *+ * The lockfile already carries the expected hash for exactly those packages, so+ * the fix is to use it. Verification is fail-closed: a package with no lockfile+ * entry, or a lockfile that cannot be found, aborts the build rather than+ * shipping unverified bytes.+ */++const LOCKFILE_NAME = 'pnpm-lock.yaml';++/** ` '@scope/[email protected]':` or ` [email protected]:` at the top level of `packages:`. */+const PACKAGE_KEY_PATTERN = /^ {2}(?:'(.+)'|([^\s'][^\s]*?)):$/;++/** `integrity: sha512-…` inside a `resolution: { … }` mapping. */+const INTEGRITY_PATTERN = /^\s+resolution:\s*\{[^}]*\bintegrity:\s*(sha\d{3}-[A-Za-z0-9+/=]+)/;++export class TarballIntegrityError extends Error {+ constructor(message: string) {+ super(message);+ this.name = 'TarballIntegrityError';+ }+}++/**+ * Maps every `name@version` in a pnpm lockfile to its recorded integrity hash.+ *+ * Deliberately line-based rather than a YAML parse: the lockfile is large, the+ * shape being read is two adjacent lines, and this keeps the build script free+ * of a YAML dependency. Entries whose `resolution` carries no `integrity` — git+ * and direct-tarball resolutions — are skipped, so asking for one fails closed+ * in `expectedIntegrityFor` instead of silently passing.+ */+export const parseLockfileIntegrity = (lockfileText: string): ReadonlyMap<string, string> => {+ const integrityByPackage = new Map<string, string>();+ const lines = lockfileText.split('\n');++ for (let index = 0; index < lines.length - 1; index += 1) {+ const keyMatch = PACKAGE_KEY_PATTERN.exec(lines[index] ?? '');+ if (!keyMatch) {+ continue;+ }++ const integrityMatch = INTEGRITY_PATTERN.exec(lines[index + 1] ?? '');+ if (!integrityMatch) {+ continue;+ }++ const packageKey = keyMatch[1] ?? keyMatch[2];+ const integrity = integrityMatch[1];+ if (packageKey && integrity && !integrityByPackage.has(packageKey)) {+ integrityByPackage.set(packageKey, integrity);+ }+ }++ return integrityByPackage;+};++/**+ * The integrity hash the lockfile pins for `packageName@version`.+ *+ * Throws when the package is absent: a build script that downloads something+ * the lockfile never resolved has no pinned bytes to check against, which is+ * the situation this module exists to refuse.+ */+export const expectedIntegrityFor = (+ integrityByPackage: ReadonlyMap<string, string>,+ packageName: string,+ version: string+): string => {+ const packageKey = `${packageName}@${version}`;+ const integrity = integrityByPackage.get(packageKey);++ if (!integrity) {+ throw new TarballIntegrityError(+ `${packageKey} has no integrity hash in ${LOCKFILE_NAME}; refusing to package unverified bytes.`+ );+ }++ return integrity;+};++/** Lockfile hash prefix to the Web Crypto algorithm name that computes it. */+const SUBTLE_ALGORITHM_BY_PREFIX: Readonly<Record<string, string>> = {+ sha1: 'SHA-1',+ sha256: 'SHA-256',+ sha384: 'SHA-384',+ sha512: 'SHA-512',+};++/** Subresource-integrity string for `bytes` under the given lockfile prefix. */+const computeIntegrity = async (bytes: Uint8Array, prefix: string): Promise<string> => {+ const digest = await crypto.subtle.digest(+ SUBTLE_ALGORITHM_BY_PREFIX[prefix] as string,+ bytes as BufferSource+ );+ const base64 = btoa(String.fromCharCode(...new Uint8Array(digest)));+ return `${prefix}-${base64}`;+};++/**+ * Throws unless `bytes` hashes to `expectedIntegrity`.+ *+ * The algorithm is read from the expected value so a lockfile that moves off+ * sha512 keeps working, and an unrecognised one fails loudly here rather than+ * comparing two strings that could never match.+ */+export const assertBytesMatchIntegrity = async (+ bytes: Uint8Array,+ expectedIntegrity: string,+ label: string+): Promise<void> => {+ const prefix = expectedIntegrity.split('-')[0] ?? '';+ if (!(prefix in SUBTLE_ALGORITHM_BY_PREFIX)) {+ throw new TarballIntegrityError(+ `Unsupported integrity algorithm for ${label}: "${expectedIntegrity}".`+ );+ }++ const actualIntegrity = await computeIntegrity(bytes, prefix);++ if (actualIntegrity !== expectedIntegrity) {+ throw new TarballIntegrityError(+ `Integrity mismatch for ${label}.\n Expected: ${expectedIntegrity}\n Actual: ${actualIntegrity}`+ );+ }+};++/**+ * Nearest `pnpm-lock.yaml` at or above `startDir`.+ *+ * Walks up rather than resolving a fixed `../../..` so the script keeps working+ * from whichever directory a build or test invokes it in.+ */+export const findLockfilePath = async (startDir: string): Promise<string> => {+ let directory = path.resolve(startDir);++ for (;;) {+ const candidate = path.join(directory, LOCKFILE_NAME);+ const found = await stat(candidate)+ .then(stats => stats.isFile())+ .catch(() => false);++ if (found) {+ return candidate;+ }++ const parent = path.dirname(directory);+ if (parent === directory) {+ throw new TarballIntegrityError(+ `Could not find ${LOCKFILE_NAME} at or above ${startDir}; cannot verify downloaded tarballs.`+ );+ }+ directory = parent;+ }+};++/** Lockfile integrity map for the workspace containing `startDir`. */+export const loadLockfileIntegrity = async (+ startDir: string+): Promise<ReadonlyMap<string, string>> =>+ parseLockfileIntegrity(await readFile(await findLockfilePath(startDir), 'utf8'));diff --git a/ts/packages/cli/test/src/scripts/tarball-integrity.test.ts b/ts/packages/cli/test/src/scripts/tarball-integrity.test.tsnew file mode 100644index 000000000..a6a5be548--- /dev/null+++ b/ts/packages/cli/test/src/scripts/tarball-integrity.test.ts@@ -0,0 +1,206 @@+import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import * as path from 'node:path';+import { fileURLToPath } from 'node:url';+import { afterAll, describe, expect, it } from 'vitest';++import {+ assertBytesMatchIntegrity,+ expectedIntegrityFor,+ findLockfilePath,+ loadLockfileIntegrity,+ parseLockfileIntegrity,+ TarballIntegrityError,+} from '../../../scripts/_tarball-integrity';++/**+ * A pnpm v9 lockfile excerpt covering the shapes that matter: a quoted scoped+ * key, an unquoted bare key, an entry whose resolution carries no integrity,+ * and a `snapshots:` repeat of a key already seen in `packages:`.+ */+const LOCKFILE = `lockfileVersion: '9.0'++packages:++ '@zed-industries/[email protected]':+ resolution: {integrity: sha512-xs5zZBLpJuciEbZNx6ZSNL0qCa9h3i/zWpj40sp6QtF+L4Ow/7qzHdBzboGhHdcz1jrLedfZeRFDA2Elj8TLMA==}+ cpu: [x64]+ os: [linux]+ hasBin: true++ resolution: {integrity: sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==}+ engines: {node: '>= 10.17.0'}++ resolution: {type: git, repo: [email protected]:owner/repo.git, commit: abc123}++snapshots:++ dependencies:+ debug: 4.4.3+`;++const integrityOf = async (bytes: Uint8Array, prefix = 'sha512'): Promise<string> => {+ const algorithm = prefix === 'sha512' ? 'SHA-512' : 'SHA-256';+ const digest = await crypto.subtle.digest(algorithm, bytes as BufferSource);+ return `${prefix}-${btoa(String.fromCharCode(...new Uint8Array(digest)))}`;+};++const tempDirs: Array<string> = [];++const makeTempDir = async (): Promise<string> => {+ const dir = await mkdtemp(path.join(tmpdir(), 'tarball-integrity-'));+ tempDirs.push(dir);+ return dir;+};++afterAll(async () => {+ await Promise.all(tempDirs.map(dir => rm(dir, { force: true, recursive: true })));+});++describe('parseLockfileIntegrity', () => {+ it('reads quoted scoped keys and unquoted bare keys', () => {+ const parsed = parseLockfileIntegrity(LOCKFILE);++ expect(parsed.get('@zed-industries/[email protected]')).toBe(+ 'sha512-xs5zZBLpJuciEbZNx6ZSNL0qCa9h3i/zWpj40sp6QtF+L4Ow/7qzHdBzboGhHdcz1jrLedfZeRFDA2Elj8TLMA=='+ );+ expect(parsed.get('[email protected]')).toBe(+ 'sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg=='+ );+ });++ it('skips resolutions that carry no integrity', () => {+ expect(parseLockfileIntegrity(LOCKFILE).has('[email protected]')).toBe(false);+ });++ it('is empty for a lockfile with no packages', () => {+ expect(parseLockfileIntegrity("lockfileVersion: '9.0'\n").size).toBe(0);+ });+});++describe('expectedIntegrityFor', () => {+ const parsed = parseLockfileIntegrity(LOCKFILE);++ it('returns the pinned hash for a package the lockfile resolved', () => {+ expect(expectedIntegrityFor(parsed, 'extract-zip', '2.0.1')).toMatch(/^sha512-/);+ });++ it('refuses a package the lockfile never resolved', () => {+ expect(() => expectedIntegrityFor(parsed, 'extract-zip', '9.9.9')).toThrow(+ TarballIntegrityError+ );+ });++ it('refuses a version the lockfile pins under a different number', () => {+ expect(() =>+ expectedIntegrityFor(parsed, '@zed-industries/codex-acp-linux-x64', '0.17.0')+ ).toThrow(/no integrity hash/);+ });+});++describe('assertBytesMatchIntegrity', () => {+ const bytes = new TextEncoder().encode('codex-acp binary bytes');++ it('accepts bytes that hash to the expected value', async () => {+ await expect(+ assertBytesMatchIntegrity(bytes, await integrityOf(bytes), '[email protected]')+ ).resolves.toBeUndefined();+ });++ it('rejects bytes swapped after the hash was pinned', async () => {+ const tampered = new TextEncoder().encode('codex-acp binary bytes + backdoor');++ await expect(+ assertBytesMatchIntegrity(tampered, await integrityOf(bytes), '[email protected]')+ ).rejects.toThrow(TarballIntegrityError);+ });++ it('rejects a single flipped byte', async () => {+ const tampered = new Uint8Array(bytes);+ tampered[0] ^= 0xff;++ await expect(+ assertBytesMatchIntegrity(tampered, await integrityOf(bytes), '[email protected]')+ ).rejects.toThrow(TarballIntegrityError);+ });++ it('names the package and both hashes so a mismatch is diagnosable', async () => {+ const tampered = new TextEncoder().encode('other');++ await expect(+ assertBytesMatchIntegrity(tampered, await integrityOf(bytes), '[email protected]')+ ).rejects.toThrow(/pkg@1\.0\.0[\s\S]*Expected:[\s\S]*Actual:/);+ });++ it('rejects an integrity string whose algorithm it cannot compute', async () => {+ await expect(assertBytesMatchIntegrity(bytes, 'md6-abcdef', '[email protected]')).rejects.toThrow(+ /Unsupported integrity algorithm/+ );+ });++ it('verifies under whichever sha algorithm the lockfile used', async () => {+ await expect(+ assertBytesMatchIntegrity(bytes, await integrityOf(bytes, 'sha256'), '[email protected]')+ ).resolves.toBeUndefined();+ });+});++describe('findLockfilePath', () => {+ it('walks up from a nested directory to the workspace lockfile', async () => {+ const root = await makeTempDir();+ const nested = path.join(root, 'ts', 'packages', 'cli', 'scripts');+ await mkdir(nested, { recursive: true });+ await writeFile(path.join(root, 'pnpm-lock.yaml'), LOCKFILE, 'utf8');++ await expect(findLockfilePath(nested)).resolves.toBe(path.join(root, 'pnpm-lock.yaml'));+ });++ it('fails closed when no lockfile exists above the start directory', async () => {+ const root = await makeTempDir();++ await expect(findLockfilePath(root)).rejects.toThrow(TarballIntegrityError);+ });+});++describe('loadLockfileIntegrity', () => {+ it('parses the nearest lockfile found above the start directory', async () => {+ const root = await makeTempDir();+ const nested = path.join(root, 'scripts');+ await mkdir(nested, { recursive: true });+ await writeFile(path.join(root, 'pnpm-lock.yaml'), LOCKFILE, 'utf8');++ const parsed = await loadLockfileIntegrity(nested);++ expect(parsed.get('[email protected]')).toMatch(/^sha512-/);+ });+});++/**+ * Named literally rather than imported from `RUN_CODEX_ACP_BINARY_TARGETS`, for+ * the same reason `verify-archive-companions.sh` spells its list out: importing+ * that module pulls in `@composio/core`, and this file otherwise needs nothing+ * built. Adding a codex target there must extend this list.+ */+const CODEX_ACP_PACKAGES = [+ '@zed-industries/codex-acp-darwin-arm64',+ '@zed-industries/codex-acp-darwin-x64',+ '@zed-industries/codex-acp-linux-arm64',+ '@zed-industries/codex-acp-linux-x64',+];++describe('this repository', () => {+ it('pins an integrity hash for every codex-acp binary the release packages', async () => {+ const parsed = await loadLockfileIntegrity(path.dirname(fileURLToPath(import.meta.url)));+ const keys = [...parsed.keys()];++ for (const packageName of CODEX_ACP_PACKAGES) {+ expect(+ keys.find(key => key.startsWith(`${packageName}@`)),+ `${packageName} has no integrity hash in pnpm-lock.yaml`+ ).toBeDefined();+ }+ });+});