OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
refactor(package)!: reject sparse entries in release tarballs (#2456)
Michael D'Angeloce25486e8dff
diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.mdindex 7a60f2de7..fd1eabe7f 100644--- a/sdk/typescript/TESTING.md+++ b/sdk/typescript/TESTING.md@@ -45,6 +45,10 @@ pnpm run test:mcppnpm run test```+The release checker rejects tar archives with sparse entries. If you created a+candidate archive with a sparse-aware tar tool, regenerate it from the source+checkout with `pnpm pack` before signing or publishing it.+To collect JUnit and coverage while running the SDK suite, use `pnpm run test:ci` in place of `pnpm run test`. To test what an npm consumer receives, pack the archive and pass its exact path to the inspection command:```shdiff --git a/sdk/typescript/scripts/check-package.mjs b/sdk/typescript/scripts/check-package.mjsindex f02bdf0e9..071681810 100644--- a/sdk/typescript/scripts/check-package.mjs+++ b/sdk/typescript/scripts/check-package.mjs@@ -15,14 +15,12 @@ import { gunzipSync } from "node:zlib";import { packageDistFiles } from "./package-dist-files.mjs";import {assertPublicPackageContents,+ assertPublicText,MAX_EXPANDED_ASSET_BYTES,} from "./package-public-content.mjs";import { assertExpectedGitHead } from "./package-provenance.mjs";import { packageSmokeTimeouts } from "./package-smoke-timeouts.mjs";-import {- assertStoredSparseContents,- readTarArchive,-} from "./package-tar-entries.mjs";+import { readTarArchive } from "./package-tar-entries.mjs";import {assertTarListingSizes,regularTarListingLines,@@ -57,7 +55,7 @@ const rawEntries = storedArchive.entries;validatePackagePaths(rawEntries.map(({ path }) => path));assertPublicPackageContents(storedArchive.files,- Buffer.concat(storedArchive.metadata.filter(Buffer.isBuffer)),+ Buffer.concat(storedArchive.metadata),);const processEnvironment = { ...process.env };delete processEnvironment.TAR_OPTIONS;@@ -194,6 +192,8 @@ const listingSizes = assertTarListingSizes(listingLines,MAX_EXPANDED_ASSET_BYTES,);+if (rawEntries.some(({ size }, index) => size !== listingSizes[index]))+ invalidTarEntry();for (const [path, name] of [["package/bin/codex-security.mjs", "CLI"],["package/_bundled_plugin/scripts/launch_codex_security_mcp", "MCP"],@@ -215,20 +215,18 @@ function privateExtractionDirectories(directory) {}function extractedArchiveFiles() {- const logicalSizes = new Map();const expectedPaths = new Map();- for (const [index, { path }] of rawEntries.entries()) {+ for (const { path } of rawEntries) {const directory = path.endsWith("/");const extractedPath = directory ? path.slice(0, -1) : path;const type = directory ? "directory" : "file";const previousType = expectedPaths.get(extractedPath);if (previousType !== undefined && previousType !== type) invalidTarEntry();- if (!directory) logicalSizes.set(path, listingSizes[index]);expectedPaths.set(extractedPath, type);const parts = extractedPath.split("/");for (let index = 1; index < parts.length; index++) {const directory = parts.slice(0, index).join("/");- if (logicalSizes.has(directory)) invalidTarEntry();+ if (storedArchive.files.has(directory)) invalidTarEntry();expectedPaths.set(directory, "directory");}}@@ -239,7 +237,7 @@ function extractedArchiveFiles() {try {chmodSync(extractionRoot, 0o700);const fileList = join(extractionRoot, "members.list");- writeFileSync(fileList, [...logicalSizes.keys(), ""].join("\0"));+ writeFileSync(fileList, [...storedArchive.files.keys(), ""].join("\0"));try {tar(["-m",@@ -278,10 +276,11 @@ function extractedArchiveFiles() {continue;}+ const stored = storedArchive.files.get(path);if (!stats.isFile() ||stats.nlink !== 1 ||- stats.size !== logicalSizes.get(path) ||+ stats.size !== stored?.length ||stats.size > MAX_EXPANDED_ASSET_BYTES ||expandedBytes > MAX_EXPANDED_ASSET_BYTES - stats.size) {@@ -289,12 +288,17 @@ function extractedArchiveFiles() {}expandedBytes += stats.size;chmodSync(extractedPath, 0o600);- archiveFiles.set(path, readFileSync(extractedPath));+ const contents = readFileSync(extractedPath);+ if (!contents.equals(stored)) invalidTarEntry();+ archiveFiles.set(path, contents);}}visit(extractionRoot);- if (expectedPaths.size !== 0 || archiveFiles.size !== logicalSizes.size) {+ if (+ expectedPaths.size !== 0 ||+ archiveFiles.size !== storedArchive.files.size+ ) {invalidTarEntry();}return archiveFiles;@@ -319,7 +323,10 @@ assertExpectedGitHead();assertPublicPackageContents(archiveFiles);-assertStoredSparseContents(storedArchive, archiveFiles);+// Keep the separate UTF-8 decoding of headers and padding alongside the raw metadata scan.+assertPublicText(+ storedArchive.metadata.map((part) => part.toString("utf8")).join(""),+);for (const path of archiveFiles.keys()) {if (/\.(?:png|br(?:\.part-[0-9]+)?)$/iu.test(path) &&diff --git a/sdk/typescript/scripts/package-public-content.mjs b/sdk/typescript/scripts/package-public-content.mjsindex 2005dce34..7e6fb519d 100644--- a/sdk/typescript/scripts/package-public-content.mjs+++ b/sdk/typescript/scripts/package-public-content.mjs@@ -60,17 +60,7 @@ function assertPublicBrotli(bytes, path) {assertPublicText(result.buffer.toString("utf8"));}-export function assertPublicText(contents, binaryRanges = []) {- const pattern = new RegExp(internalMarker.source, "giu");- let match;- while ((match = pattern.exec(contents)) !== null) {- if (- !binaryRanges.some(- ({ start, end }) =>- match.index >= start && match.index + match[0].length <= end,- )- )- throw new Error("npm tarball contains an internal reference.");- pattern.lastIndex = match.index + 1;- }+export function assertPublicText(contents) {+ if (internalMarker.test(contents))+ throw new Error("npm tarball contains an internal reference.");}diff --git a/sdk/typescript/scripts/package-tar-entries.mjs b/sdk/typescript/scripts/package-tar-entries.mjsindex 5b6e81ade..d8a06a630 100644--- a/sdk/typescript/scripts/package-tar-entries.mjs+++ b/sdk/typescript/scripts/package-tar-entries.mjs@@ -5,6 +5,10 @@ function invalidTarEntry() {throw new Error("npm tarball contains an invalid tar entry.");}+function unsupportedSparseEntry() {+ throw new Error("npm tarball contains an unsupported sparse entry.");+}+function headerText(header, start, end) {return header.subarray(start, end).toString("utf8").split("\0", 1)[0];}@@ -31,8 +35,17 @@ function paxAttributes(contents) {contents[end - 1] !== 0x0a)invalidTarEntry();+ const key = contents.subarray(separator + 1, equals).toString("utf8");+ if (+ key === "GNU.sparse" ||+ key.startsWith("GNU.sparse.") ||+ key === "SCHILY.realsize" ||+ key === "SUN.holesdata"+ ) {+ unsupportedSparseEntry();+ }attributes.set(- contents.subarray(separator + 1, equals).toString("utf8"),+ key,contents.subarray(equals + 1, end - 1).toString("utf8"),);offset = end;@@ -40,103 +53,10 @@ function paxAttributes(contents) {return attributes;}-function sparseMap(contents, paxMap) {- if (paxMap) {- const values = paxMap.split(",").map(Number);- const extents = [];- for (let index = 0; index < values.length; index += 2)- extents.push({ offset: values[index], size: values[index + 1] });- return { contents, extents, dataOffset: 0 };- }- let offset = 0;- function number() {- while (offset < contents.length) {- const end = contents.indexOf(0x0a, offset);- if (end === -1) invalidTarEntry();- const line = contents.subarray(offset, end).toString("ascii");- offset = end + 1;- if (line.startsWith("#")) continue;- if (!/^[0-9]*$/u.test(line)) invalidTarEntry();- return Number(line);- }- invalidTarEntry();- }- const count = number();- const extents = [];- for (let index = 0; index < count; index++) {- extents.push({ offset: number(), size: number() });- }- const dataOffset = Math.ceil(offset / blockSize) * blockSize;- if (dataOffset > contents.length) invalidTarEntry();- return { contents, extents, dataOffset };-}--export function assertStoredSparseContents(archive, extractedFiles) {- const sparseMetadata = new Map();- for (const [path, { contents, extents, dataOffset }] of archive.sparseFiles) {- const extracted = extractedFiles.get(path);- let metadata;- // GNU tar pads stored extents; libarchive also accepts packed extents.- for (const padded of [false, true]) {- let offset = dataOffset;- const parts = [contents.subarray(0, dataOffset)];- const matches = extents.every((extent, index) => {- const end = offset + extent.size;- if (- !contents- .subarray(offset, end)- .equals(- extracted.subarray(extent.offset, extent.offset + extent.size),- )- )- return false;- const next =- padded && index < extents.length - 1- ? Math.ceil(end / blockSize) * blockSize- : end;- parts.push(- { binary: contents.subarray(offset, end) },- contents.subarray(end, next),- );- offset = next;- return true;- });- if (matches) {- parts.push(contents.subarray(offset));- metadata = parts;- break;- }- }- if (metadata === undefined) invalidTarEntry();- sparseMetadata.set(path, metadata);- }- const parts = archive.metadata.flatMap((part) =>- typeof part === "string" ? sparseMetadata.get(part) : part,- );- const binaryRanges = [];- let text = "";- for (const part of parts) {- const binary = !Buffer.isBuffer(part);- const decoded = (binary ? part.binary : part).toString("utf8");- if (binary) {- const previous = binaryRanges.at(-1);- if (previous?.end === text.length) previous.end += decoded.length;- else- binaryRanges.push({- start: text.length,- end: text.length + decoded.length,- });- }- text += decoded;- }- assertPublicText(text, binaryRanges);-}-export function readTarArchive(archiveBytes) {const entries = [];const archiveFiles = new Map();const archiveMetadata = [];- const sparseFiles = new Map();const npmFiles = new Map();let offset = 0;const globalAttributes = new Map();@@ -154,15 +74,19 @@ export function readTarArchive(archiveBytes) {const signature = header.subarray(257, 265).toString("latin1");const directory = header[156] === 0x35;- const oldSparse = header[156] === 0x53;+ // libarchive also reads the GNU sparse map, continuation flag and real+ // size from GNU member headers, even on an otherwise regular entry.+ if (+ header[156] === 0x53 ||+ (signature === "ustar \0" &&+ (header[386] !== 0 || header[482] !== 0 || header[483] !== 0))+ ) {+ unsupportedSparseEntry();+ }const longName = header[156] === 0x4c;const extended = header[156] === 0x78 || header[156] === 0x67 || longName;if (- (header[156] !== 0 &&- header[156] !== 0x30 &&- !directory &&- !extended &&- !oldSparse) ||+ (header[156] !== 0 && header[156] !== 0x30 && !directory && !extended) ||(signature !== "ustar\0" + "00" &&signature !== "ustar \0" &&signature !== "\0".repeat(8))@@ -179,11 +103,7 @@ export function readTarArchive(archiveBytes) {? nextAttributes.get(key) || undefined: globalAttributes.get(key);const headerPath = prefix === "" ? name : `${prefix}/${name}`;- const path =- attribute("GNU.sparse.name") ??- attribute("path") ??- nextName ??- headerPath;+ const path = attribute("path") ?? nextName ?? headerPath;if (!extended && (path === "" || path.endsWith("/") !== directory))invalidTarEntry();assertPublicText(path);@@ -192,30 +112,7 @@ export function readTarArchive(archiveBytes) {const paxSize = extended ? undefined : attribute("size");if (paxSize !== undefined && !/^[0-9]+$/u.test(paxSize)) invalidTarEntry();const size = paxSize === undefined ? headerSize : Number(paxSize);- let contentsStart = offset + blockSize;- const oldSparseExtents = [];- if (oldSparse) {- let map = header;- let start = 386;- let count = 4;- for (;;) {- let index = 0;- for (; index < count; index++) {- const field = start + index * 24;- if (map[field + 12] === 0) break;- const offset = octalNumber(map, field, field + 12);- const size = octalNumber(map, field + 12, field + 24);- if (offset !== 0 || size !== 0)- oldSparseExtents.push({ offset, size });- }- if (index < count || map[start + count * 24] === 0) break;- map = archiveBytes.subarray(contentsStart, contentsStart + blockSize);- if (map.length !== blockSize) invalidTarEntry();- contentsStart += blockSize;- start = 0;- count = 21;- }- }+ const contentsStart = offset + blockSize;const contentsEnd = contentsStart + size;const nextOffset = contentsStart + Math.ceil(size / blockSize) * blockSize;if (nextOffset > archiveBytes.byteLength || (directory && size !== 0)) {@@ -245,30 +142,9 @@ export function readTarArchive(archiveBytes) {archiveMetadata.push(archiveBytes.subarray(offset, nextOffset));else {const contents = archiveBytes.subarray(contentsStart, contentsEnd);- // npm ignores old-GNU sparse entries and GNU sparse name overrides.- const npmPath = oldSparse ? undefined : (nextNpmPath ?? headerPath);- if (npmPath !== undefined) npmFiles.set(npmPath, contents);+ npmFiles.set(nextNpmPath ?? headerPath, contents);+ archiveFiles.set(path, contents);archiveMetadata.push(archiveBytes.subarray(offset, contentsStart));- const paxSparseMap = nextAttributes.get("GNU.sparse.map");- if (- (oldSparse ||- paxSparseMap ||- Number.parseInt(nextAttributes.get("GNU.sparse.major"), 10) ===- 1) &&- /\.(?:png|br(?:\.part-[0-9]+)?)$/iu.test(path)- ) {- sparseFiles.set(- path,- oldSparse- ? { contents, extents: oldSparseExtents, dataOffset: 0 }- : sparseMap(contents, paxSparseMap),- );- // Retain sparse framing in order with the surrounding headers and padding.- archiveMetadata.push(path);- } else {- archiveFiles.set(path, contents);- if (oldSparse) archiveMetadata.push(contents);- }archiveMetadata.push(archiveBytes.subarray(contentsEnd, nextOffset));}entries.push({ path, size });@@ -281,19 +157,10 @@ export function readTarArchive(archiveBytes) {if (archiveBytes.subarray(offset).some((byte) => byte !== 0))invalidTarEntry();- const deferredStreams = new Set(- [...sparseFiles.keys()].map((path) => path.replace(/\.part-[0-9]+$/iu, "")),- );- for (const path of archiveFiles.keys()) {- if (deferredStreams.has(path.replace(/\.part-[0-9]+$/iu, ""))) {- archiveFiles.delete(path);- }- }return {entries,files: archiveFiles,metadata: archiveMetadata,- sparseFiles,npmFiles,};}diff --git a/sdk/typescript/tests-ts/package-tar-entries.test.ts b/sdk/typescript/tests-ts/package-tar-entries.test.tsindex f1f790b70..85e2b0748 100644--- a/sdk/typescript/tests-ts/package-tar-entries.test.ts+++ b/sdk/typescript/tests-ts/package-tar-entries.test.ts@@ -1,9 +1,7 @@-import { readFileSync } from "node:fs";import { describe, expect, test } from "bun:test";import {archive,blockSize,- cleanCompressedPayload,octal,paxRecords,tarRecord,@@ -18,15 +16,11 @@ type PackageTarEntries = {readTarArchive: (archiveBytes: Buffer) => {entries: PlainTarEntry[];files: Map<string, Buffer>;- metadata: (Buffer | string)[];+ metadata: Buffer[];};- assertStoredSparseContents: (- archive: ReturnType<PackageTarEntries["readTarArchive"]>,- files: Map<string, Buffer>,- ) => void;};-const { readTarArchive, assertStoredSparseContents } = (await import(+const { readTarArchive } = (await import(new URL("../scripts/package-tar-entries.mjs", import.meta.url).href)) as PackageTarEntries;const { assertPublicPackageContents } = (await import(@@ -40,16 +34,12 @@ const { assertPublicPackageContents } = (await import(function plainTarEntries(bytes: Buffer): PlainTarEntry[] {const archive = readTarArchive(bytes);- assertPublicPackageContents(- archive.files,- Buffer.concat(- archive.metadata.filter((part): part is Buffer => Buffer.isBuffer(part)),- ),- );+ assertPublicPackageContents(archive.files, Buffer.concat(archive.metadata));return archive.entries;}const invalidTarEntryError = "npm tarball contains an invalid tar entry.";+const sparseEntryError = "npm tarball contains an unsupported sparse entry.";const internalReferenceError = "npm tarball contains an internal reference.";describe("plain npm tar entries", () => {@@ -72,7 +62,8 @@ describe("plain npm tar entries", () => {test.each([0, 0x30])("accepts regular typeflag %i and prefix paths",(type) => {- const prefix = `package/${"nested/".repeat(13)}deep`;+ // POSIX prefix bytes can occupy all three old GNU sparse field offsets.+ const prefix = `package/${"nested/".repeat(20)}deep`;const longPath = `${prefix}/README.md`;expect(longPath.length).toBeGreaterThan(100);@@ -124,7 +115,10 @@ describe("plain npm tar entries", () => {});test.each([0x78, 0x67])("accepts POSIX pax metadata typeflag %i", (type) => {- const attributes = paxRecords({ ctime: "0.123456789" });+ const attributes = paxRecords({+ ctime: "0.123456789",+ "SCHILY.xattr.user.note": "sparse",+ });expect(plainTarEntries(archive(@@ -193,124 +187,85 @@ describe("plain npm tar entries", () => {).toThrow(internalReferenceError);});- test.each([false, true])(- "scans discarded sparse-map padding, marker=%j",- (marker) => {- const map = Buffer.alloc(blockSize);- map.write("1\n512\n512\n");- if (marker) map.write("go/example", 32);- const bytes = archive(- tarRecord(- paxRecords({- "GNU.sparse.major": "1",- "GNU.sparse.minor": "0",- "GNU.sparse.name": "package/README.md",- "GNU.sparse.realsize": "1024",- }),- { name: "package/PaxHeaders/README.md", type: 0x78 },- ),- tarRecord(Buffer.concat([map, Buffer.alloc(blockSize)]), {- name: "package/README.md",- }),- );- if (marker)- expect(() => plainTarEntries(bytes)).toThrow(internalReferenceError);- else- expect(plainTarEntries(bytes)).toEqual([- { path: "package/README.md", size: 1024 },- ]);- },- );-- for (const format of [- "gnu-01",- "gnu-nul",- "bsd-comments",- "bsd-solaris",- ] as const) {- test.each([false, true])(- `scans all stored sparse bytes for ${format}, marker=%j`,- (marker) => {- let attributes: Record<string, string>;- let contents: Buffer;- if (format === "gnu-01") {- attributes = {- "GNU.sparse.size": "1024",- "GNU.sparse.numblocks": "1",- "GNU.sparse.map": "512,512",- };- contents = Buffer.alloc(1024);- if (marker) contents.write("go/example", 512);- } else if (format === "bsd-solaris") {- attributes = { "SUN.holesdata": " 512 1024" };- contents = Buffer.alloc(1024);- if (marker) contents.write("go/example", 0);- } else {- attributes = {- "GNU.sparse.major": format === "gnu-nul" ? "1\0" : "1",- "GNU.sparse.minor": "0",- "GNU.sparse.name": "package/README.md",- "GNU.sparse.realsize": format === "bsd-comments" ? "1536" : "1024",- };- contents = Buffer.alloc(format === "bsd-comments" ? 1536 : 1024);- if (format === "bsd-comments") {- const comments = `${`# ${"x".repeat(60)}\n`.repeat(9)}# ${marker ? "go/example" : "public note"}\n1\n1024\n512\n`;- contents.write(comments);- } else {- contents.write("1\n512\n512\n");- if (marker) contents.write("go/example", 32);- }- }- const bytes = archive(- tarRecord(paxRecords(attributes), {- name: "package/PaxHeaders/README.md",- type: 0x78,- }),- tarRecord(contents, { name: "package/README.md" }),- );- if (marker)- expect(() => plainTarEntries(bytes)).toThrow(internalReferenceError);- else- expect(plainTarEntries(bytes)).toEqual([- { path: "package/README.md", size: contents.length },- ]);- },- );- }-- test("checks stored old GNU sparse bodies", () => {- const contents = Buffer.alloc(1024);- contents.write("go/example");- const record = tarRecord(contents, {+ test("rejects the old GNU sparse typeflag", () => {+ const member = tarRecord(Buffer.from("data"), {name: "package/README.md",+ type: 0x53,magic: "ustar ",version: " \0",});- for (const offset of [386, 410]) {- octal(0, 12).copy(record, offset);- octal(512, 12).copy(record, offset + 12);- }- octal(1024, 12).copy(record, 483);- expect(() => plainTarEntries(archive(record))).toThrow(- internalReferenceError,- );+ expect(() => readTarArchive(archive(member))).toThrow(sparseEntryError);});- test("checks stored bytes after an empty repeated sparse attribute", () => {- const contents = Buffer.alloc(1024);- contents.write("go/example", 512);+ test.each([+ [386, "00000000000"],+ [482, "\x01"],+ [483, "00000000004"],+ ] as const)(+ "rejects a regular GNU member with a sparse field at byte %i",+ (offset, field) => {+ const member = tarRecord(Buffer.from("data"), {+ name: "package/README.md",+ type: 0x30,+ magic: "ustar ",+ version: " \0",+ prefix: "\0".repeat(offset - 345) + field,+ });+ expect(() => readTarArchive(archive(member))).toThrow(sparseEntryError);+ },+ );++ test.each([0x78, 0x67])(+ "rejects sparse PAX attributes of type %i",+ (type) => {+ for (const [key, value] of [+ ["GNU.sparse", ""],+ ["GNU.sparse.map", "0,4"],+ ["GNU.sparse.major", "1\0"],+ ["GNU.sparse.name", "package/README.md"],+ ["SCHILY.realsize", "1024"],+ ["SUN.holesdata", " 512 1024"],+ ] as const) {+ const header = tarRecord(paxRecords({ [key]: value }), {+ name: "PaxHeaders/readme",+ type,+ });+ expect(() =>+ readTarArchive(+ archive(+ header,+ tarRecord(Buffer.from("data"), { name: "package/README.md" }),+ ),+ ),+ ).toThrow(sparseEntryError);+ }+ },+ );++ test("rejects trailing sparse metadata even when the attribute is reset", () => {const attributes = Buffer.concat([- paxRecords({ "GNU.sparse.map": "512,512" }),+ paxRecords({ "GNU.sparse.map": "0,4" }),paxRecords({ "GNU.sparse.map": "" }),]);expect(() =>- plainTarEntries(+ readTarArchive(archive(+ tarRecord(Buffer.from("data"), { name: "package/README.md" }),tarRecord(attributes, { name: "PaxHeaders/readme", type: 0x78 }),- tarRecord(contents, { name: "package/README.md" }),),),- ).toThrow(internalReferenceError);+ ).toThrow(sparseEntryError);+ });++ test("scans regular file contents under a GNU header", () => {+ const member = tarRecord(Buffer.from("go/example"), {+ name: "package/README.md",+ magic: "ustar ",+ version: " \0",+ });+ expect(() => plainTarEntries(archive(member))).toThrow(+ internalReferenceError,+ );});test("checks markers across concatenated archive metadata", () => {@@ -328,132 +283,6 @@ describe("plain npm tar entries", () => {).toThrow(internalReferenceError);});- test.each([false, true])(- "keeps public Brotli content semantics across retained sparse extents, split=%j",- (split) => {- const path = "package/runtime.mjs.br";- const size = cleanCompressedPayload.length;- const middle = cleanCompressedPayload.indexOf("Go/w") + 2;- expect(middle).toBeGreaterThan(2);- const bytes = archive(- tarRecord(- paxRecords({- "GNU.sparse.size": String(size),- "GNU.sparse.numblocks": split ? "2" : "1",- "GNU.sparse.map": split- ? `0,${middle},${middle},${size - middle}`- : `0,${size}`,- }),- { name: "PaxHeaders/runtime", type: 0x78 },- ),- tarRecord(cleanCompressedPayload, { name: path }),- );- expect(plainTarEntries(bytes)).toEqual([{ path, size }]);- expect(() =>- assertStoredSparseContents(- readTarArchive(bytes),- new Map([[path, cleanCompressedPayload]]),- ),- ).not.toThrow();- },- );-- test("keeps retained extents between separate sparse metadata segments", () => {- const path = "package/logo.png";- const logo = readFileSync(- new URL(- "../../../plugins/codex-security/assets/logo.png",- import.meta.url,- ),- );- const firstPadding = Buffer.alloc(512 - 250);- firstPadding.write(".openai.", firstPadding.length - 8);- const secondPadding = Buffer.alloc(512 - 250);- secondPadding.write("org");- const bytes = archive(- tarRecord(- paxRecords({- "GNU.sparse.size": String(logo.length),- "GNU.sparse.numblocks": "3",- "GNU.sparse.map": `0,250,250,250,500,${logo.length - 500}`,- }),- { name: "PaxHeaders/logo", type: 0x78 },- ),- tarRecord(- Buffer.concat([- logo.subarray(0, 250),- firstPadding,- logo.subarray(250, 500),- secondPadding,- logo.subarray(500),- ]),- { name: path },- ),- );- expect(() =>- assertStoredSparseContents(- readTarArchive(bytes),- new Map([[path, logo]]),- ),- ).not.toThrow();- });-- test("checks markers crossing discarded padding and retained sparse bytes", () => {- const path = "package/logo.png";- const logo = readFileSync(- new URL(- "../../../plugins/codex-security/assets/logo.png",- import.meta.url,- ),- );- expect(logo.subarray(387, 390).toString()).toBe("org");- const padding = Buffer.alloc(512 - 387);- padding.write(".openai.", padding.length - 8);- const bytes = archive(- tarRecord(- paxRecords({- "GNU.sparse.size": String(logo.length),- "GNU.sparse.numblocks": "2",- "GNU.sparse.map": `0,387,387,${logo.length - 387}`,- }),- { name: "PaxHeaders/logo", type: 0x78 },- ),- tarRecord(- Buffer.concat([logo.subarray(0, 387), padding, logo.subarray(387)]),- { name: path },- ),- );- expect(() =>- assertStoredSparseContents(- readTarArchive(bytes),- new Map([[path, logo]]),- ),- ).toThrow(internalReferenceError);- });-- test("uses the native GNU sparse name while retaining stored content", () => {- const path = "package/README.md";- const stored = Buffer.alloc(1024);- stored.write("1\n512\n512\n");- stored.fill(0x78, 512);- expect(- plainTarEntries(- archive(- tarRecord(- paxRecords({- "GNU.sparse.major": "1",- "GNU.sparse.minor": "0",- "GNU.sparse.name": path,- "GNU.sparse.realsize": "1024",- }),- { name: "PaxHeaders/readme", type: 0x78 },- ),- tarRecord(stored, { name: "package/GNUSparseFile.1/readme" }),- ),- ),- ).toEqual([{ path, size: stored.byteLength }]);- });-test("accepts an empty size field for an empty file", () => {expect(plainTarEntries(diff --git a/sdk/typescript/tests-ts/package-tar-fixtures.ts b/sdk/typescript/tests-ts/package-tar-fixtures.tsindex 550e1534b..d942ec89d 100644--- a/sdk/typescript/tests-ts/package-tar-fixtures.ts+++ b/sdk/typescript/tests-ts/package-tar-fixtures.ts@@ -66,62 +66,6 @@ export function tarRecord(]);}-export function oldGnuSparseRecord(- contents: Buffer,- name: string,- extents = [{ offset: 0, size: contents.length }],- metadataMarker?: "continuation" | "boundary" | "unused",-): Buffer {- const stored = Buffer.concat(- extents.flatMap(({ offset, size }, index) => [- contents.subarray(offset, offset + size),- Buffer.alloc(index < extents.length - 1 ? (512 - (size % 512)) % 512 : 0),- ]),- );- const record = tarRecord(stored, {- name,- type: 0x53,- magic: "ustar ",- version: " \0",- });- const header = record.subarray(0, blockSize);- const continuations = [];- let remaining = extents;- let map = header;- let start = 386;- let count = 4;- do {- for (const [index, extent] of remaining.slice(0, count).entries()) {- octal(extent.offset, 12).copy(map, start + index * 24);- octal(extent.size, 12).copy(map, start + index * 24 + 12);- }- remaining = remaining.slice(count);- map[start + count * 24] = remaining.length > 0 ? 1 : 0;- if (remaining.length > 0) {- map = Buffer.alloc(blockSize);- continuations.push(map);- start = 0;- count = 21;- }- } while (remaining.length > 0);- if (metadataMarker === "unused") {- const used =- extents.length <= 4 ? extents.length : ((extents.length - 5) % 21) + 1;- map.write("public", start + (used + 1) * 24);- }- octal(contents.length, 12).copy(header, 483);- if (metadataMarker === "continuation")- continuations.at(-1)!.write("go/test", 505);- if (metadataMarker === "boundary") header.write("go/", 509);- header.fill(0x20, 148, 156);- octal(- header.reduce((sum, byte) => sum + byte, 0),- 8,- "\0 ",- ).copy(header, 148);- return Buffer.concat([header, ...continuations, record.subarray(blockSize)]);-}-export function archive(...records: Buffer[]): Buffer {return Buffer.concat([...records, Buffer.alloc(blockSize * 2)]);}diff --git a/sdk/typescript/tests-ts/package-tar-listing.test.ts b/sdk/typescript/tests-ts/package-tar-listing.test.tsindex 9c8277005..48448f9f9 100644--- a/sdk/typescript/tests-ts/package-tar-listing.test.ts+++ b/sdk/typescript/tests-ts/package-tar-listing.test.ts@@ -15,13 +15,12 @@ import { tmpdir } from "node:os";import { delimiter, join } from "node:path";import { fileURLToPath } from "node:url";import { promisify } from "node:util";-import { brotliCompressSync, brotliDecompressSync, gzipSync } from "node:zlib";+import { brotliCompressSync, gzipSync } from "node:zlib";import { describe, expect, test } from "bun:test";import {archive,cleanCompressedPayload,octal,- oldGnuSparseRecord,paxRecords,tarRecord,} from "./package-tar-fixtures.js";@@ -44,6 +43,10 @@ const pluginContract = {shippedExact: ["scripts/launch_codex_security_mcp"],};+function paxExtension(attributes: Record<string, string>, type = 0x78): Buffer {+ return tarRecord(paxRecords(attributes), { name: "PaxHeaders/entry", type });+}+function packageTar({trailingZeroBytes = 0,sizeTerminator = " ",@@ -53,7 +56,6 @@ function packageTar({directoryMode = 0o755,readmeMode = 0o644,mtime = 0,- readmeSparse,readmeLongName = false,additionalRecords = [],overrides = new Map(),@@ -66,7 +68,6 @@ function packageTar({directoryMode?: number;readmeMode?: number;mtime?: number;- readmeSparse?: "0.0" | "1.0";readmeLongName?: boolean;additionalRecords?: Buffer[];overrides?: ReadonlyMap<string, Buffer>;@@ -97,37 +98,6 @@ function packageTar({const records = paths.map((path) => {const override = overrides.get(path);if (override !== undefined) return override;- if (path === "package/README.md" && readmeSparse !== undefined) {- const contents = Buffer.alloc(512, 0x78);- const map = Buffer.alloc(512);- map.write("1\n512\n512\n");- const attributes: Record<string, string> =- readmeSparse === "0.0"- ? {- "GNU.sparse.size": "1024",- "GNU.sparse.numblocks": "1",- "GNU.sparse.map": "512,512",- }- : {- "GNU.sparse.major": "1",- "GNU.sparse.minor": "0",- "GNU.sparse.name": path,- "GNU.sparse.realsize": "1024",- };- return Buffer.concat([- tarRecord(paxRecords(attributes), {- name: "PaxHeaders/readme",- type: 0x78,- }),- tarRecord(- readmeSparse === "0.0" ? contents : Buffer.concat([map, contents]),- {- name:- readmeSparse === "0.0" ? path : "package/GNUSparseFile.1/readme",- },- ),- ]);- }const contents =path === "package/package.json"? Buffer.from(@@ -330,7 +300,7 @@ describe("npm package tar listings", () => {test.each(["-rw-r--r-- 0/0 33554433 1970-01-01 00:00 package/README.md","-rw-r--r-- 0 0 0 33554433 Jan 1 1970 package/README.md",- ])("bounds native sparse logical sizes before extraction: %s", (line) => {+ ])("bounds native file sizes before extraction: %s", (line) => {expect(() => assertTarListingSizes([line], 32 * 1024 * 1024)).toThrow("npm tarball contains an invalid tar entry",);@@ -343,7 +313,7 @@ describe("npm package tar listings", () => {).toThrow();});- test("retains bounded logical sparse sizes in listing order", () => {+ test("retains bounded file sizes in listing order", () => {expect(assertTarListingSizes([@@ -428,8 +398,6 @@ describe("npm package tar listings", () => {const archives = [["default", gzipSync(tarBytes)],["level-0", gzipSync(tarBytes, { level: 0 })],- ["sparse-0.0", gzipSync(packageTar({ readmeSparse: "0.0" }))],- ["sparse-1.0", gzipSync(packageTar({ readmeSparse: "1.0" }))],["gnu-long-name",gzipSync(@@ -535,23 +503,55 @@ describe("npm package tar listings", () => {);test.each([- "pax-full",- "pax-map-prefix",- "pax-hole",- "oldgnu",- "sparse-name",+ [+ "old GNU",+ tarRecord(Buffer.alloc(0), {+ name: "package/README.md",+ type: 0x53,+ magic: "ustar ",+ version: " \0",+ }),+ ],+ ["GNU PAX map", paxExtension({ "GNU.sparse.map": "0,1" })],+ ["GNU global PAX", paxExtension({ "GNU.sparse.major": "1" }, 0x67)],+ ["Solaris hole map", paxExtension({ "SUN.holesdata": " 0 1" })],+ ["star logical size", paxExtension({ "SCHILY.realsize": "1" })],+ ] as const)(+ "rejects %s sparse entries before invoking native tar",+ async (_label, record) => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-sparse-"));+ try {+ const archivePath = join(root, "sparse.tgz");+ writeFileSync(+ archivePath,+ gzipSync(packageTar({ additionalRecords: [record] })),+ );+ const contractPath = join(root, "contract.json");+ writeFileSync(contractPath, JSON.stringify(pluginContract));+ const environment = { ...process.env, PATH: root };+ const checked = await checkPackage(+ root,+ archivePath,+ contractPath,+ environment,+ );+ expect(checked.status).toBe(1);+ expect(checked.stderr).toContain(+ "npm tarball contains an unsupported sparse entry.",+ );+ } finally {+ rmSync(root, { force: true, recursive: true });+ }+ },+ );++ test.each(["pax-path",- "global-path",- "pax-then-long-name",- "long-name-then-pax","pax-then-wrong-long-name",- "overwrite-text",- "overwrite-logo",- "ordinary-pax-then-wrong-long-name",- "ordinary-pax-then-correct-long-name",- "ordinary-long-name-then-pax",- ])("validates assets installed by npm: %s", async (representation) => {- const root = mkdtempSync(join(tmpdir(), "codex-package-npm-sparse-"));+ "pax-then-correct-long-name",+ "long-name-then-pax",+ ])("validates the asset path used by npm: %s", async (order) => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-npm-path-"));try {const logo = readFileSync(new URL(@@ -560,97 +560,28 @@ describe("npm package tar listings", () => {),);const path = "package/_bundled_plugin/logo.png";- const renamed =- representation === "sparse-name" || representation === "pax-path";- const wrongPath = "package/synthetic-storage.png";- const ordinary = representation.startsWith("ordinary-");- const pathOrder = ordinary- ? representation.slice("ordinary-".length)- : representation;- const wrongLongName = pathOrder === "pax-then-wrong-long-name";- const orderedPath = representation.includes("then");- const overwrite = representation.startsWith("overwrite");- let stored =- representation === "overwrite-text" ? Buffer.from("fixture\n") : logo;- const attributes: Record<string, string> = ordinary- ? {}- : {- "GNU.sparse.size": String(stored.length),- "GNU.sparse.numblocks": "1",- "GNU.sparse.map": `0,${stored.length}`,- };- if (representation === "pax-map-prefix") {- const map = Buffer.alloc(512);- map.write(`1\n0\n${logo.length}\n`);- stored = Buffer.concat([map, logo]);- for (const key of Object.keys(attributes)) delete attributes[key];- Object.assign(attributes, {- "GNU.sparse.major": "1",- "GNU.sparse.minor": "0",- "GNU.sparse.name": path,- "GNU.sparse.realsize": String(logo.length),- });- } else if (representation === "pax-hole") {- const hole = logo.findIndex(- (byte, index) => index > 0 && index % 512 === 0 && byte === 0,- );- expect(hole).toBeGreaterThan(0);- stored = Buffer.concat([- logo.subarray(0, hole),- logo.subarray(hole + 1),- ]);- attributes["GNU.sparse.numblocks"] = "2";- attributes["GNU.sparse.map"] =- `0,${hole},${hole + 1},${logo.length - hole - 1}`;- }- if (- !ordinary &&- (renamed || orderedPath || representation === "global-path")- )- attributes["GNU.sparse.name"] = path;- if (overwrite) attributes["GNU.sparse.name"] = "package/README.md";- if (representation === "pax-path" || orderedPath)- attributes["path"] =- representation === "pax-then-long-name" ? wrongPath : path;+ const storagePath = "package/synthetic-storage.png";+ const wrongLongName = order === "pax-then-wrong-long-name";+ const pax = paxExtension({ path });const longName = tarRecord(- Buffer.from(- `${representation === "pax-then-long-name" || representation === "ordinary-pax-then-correct-long-name" || pathOrder === "long-name-then-pax" ? path : wrongPath}\0`,- ),+ Buffer.from(`${wrongLongName ? storagePath : path}\0`),{ name: "././@LongLink", type: 0x4c },);- const record =- representation === "oldgnu"- ? oldGnuSparseRecord(logo, path)- : Buffer.concat([- ...(representation === "global-path"- ? [- tarRecord(paxRecords({ path: wrongPath }), {- name: "GlobalHead",- type: 0x67,- }),- ]- : []),- ...(pathOrder === "long-name-then-pax" ? [longName] : []),- tarRecord(paxRecords(attributes), {- name: "PaxHeaders/asset",- type: 0x78,- }),- ...(pathOrder.startsWith("pax-then") ? [longName] : []),- tarRecord(stored, {- name: renamed || ordinary ? wrongPath : path,- }),- ]);+ const headers =+ order === "pax-path"+ ? [pax]+ : order === "long-name-then-pax"+ ? [longName, pax]+ : [pax, longName];const archivePath = join(root, "package.tgz");writeFileSync(archivePath,gzipSync(packageTar({- overrides: overwrite- ? new Map([["package/README.md", Buffer.alloc(0)]])- : undefined,- additionalRecords: overwrite- ? [tarRecord(logo, { name: path }), record]- : [record],+ additionalRecords: [+ ...headers,+ tarRecord(logo, { name: storagePath }),+ ],}),),);@@ -674,182 +605,85 @@ describe("npm package tar listings", () => {contractPath,environment,);- // The wrong-long-name fixtures have no portable asset path to select.+ // The two readers can disagree on a final GNU long name. Extract everything in that case.const extraction = execFileAsync("tar",["-xzf", archivePath, "-C", native, ...(wrongLongName ? [] : [path])],- { encoding: "buffer", timeout: 30_000, windowsHide: true },+ {+ encoding: "buffer",+ timeout: 30_000,+ windowsHide: true,+ },);const installation = installPackage(root, archivePath, environment);- // All three read the same archive. Keep it until every reader exits, including on failure.await Promise.allSettled([extraction, installation, packageCheck]);- const extracted = await extraction;- expect(extracted.stderr.toString()).toBe("");+ expect((await extraction).stderr.toString()).toBe("");if (!wrongLongName)- expect(readFileSync(join(native, path)).equals(logo)).toBe(true);+ expect(readFileSync(join(native, path))).toEqual(logo);const installed = await installation;- const consumer = join(root, "consumer");- expect({- status: installed.status,- stderr: installed.status === 0 ? "" : installed.stderr,- }).toEqual({ status: 0, stderr: "" });+ expect(installed.status, installed.stderr).toBe(0);const installedPackage = join(- consumer,+ root,+ "consumer","node_modules","@openai","codex-security",);- const installedLogo = join(- installedPackage,- "_bundled_plugin",- "logo.png",- );- if (wrongLongName) {- expect(existsSync(installedLogo)).toBe(false);+ if (wrongLongName)expect(- readFileSync(- join(installedPackage, wrongPath.slice("package/".length)),- ).equals(logo),- ).toBe(true);- }- const compatible = [- "pax-full",- "pax-path",- "global-path",- "pax-then-long-name",- "long-name-then-pax",- "overwrite-logo",- "ordinary-pax-then-correct-long-name",- "ordinary-long-name-then-pax",- ].includes(representation);+ existsSync(join(installedPackage, "_bundled_plugin", "logo.png")),+ ).toBe(false);+ const installedPath = wrongLongName ? storagePath : path;expect(- existsSync(installedLogo) && readFileSync(installedLogo).equals(logo),- ).toBe(compatible);- if (overwrite)- expect(readFileSync(installedLogo).equals(stored)).toBe(true);+ readFileSync(+ join(installedPackage, installedPath.slice("package/".length)),+ ),+ ).toEqual(logo);const checked = await packageCheck;- expect({- status: checked.status,- stderr: compatible ? checked.stderr : "",- }).toEqual({ status: compatible ? 0 : 1, stderr: "" });+ expect(checked.status, checked.stderr).toBe(wrongLongName ? 1 : 0);+ if (!wrongLongName) expect(checked.stderr).toBe("");} finally {rmSync(root, { force: true, recursive: true });}});test.each([- "oldgnu-manifest",- "pax-map-prefix",- "overwrite-json",- "overwrite-metadata",- "overwrite-git-head",- "pax-full",- "pax-path",- "oldgnu-readme",- ])("validates the manifest npm installs: %s", async (representation) => {- const root = mkdtempSync(join(tmpdir(), "codex-package-npm-manifest-"));- try {- const path = "package/package.json";- const gitHead = "a".repeat(40);- const metadata = {- name: "@openai/codex-security",- version: "1.0.0",- license: "Apache-2.0",- gitHead,- };- const manifest = Buffer.from(JSON.stringify(metadata));- const overrides = new Map([[path, tarRecord(manifest, { name: path })]]);- if (representation.startsWith("oldgnu")) {- const oldPath =- representation === "oldgnu-manifest" ? path : "package/README.md";- overrides.set(- oldPath,- oldGnuSparseRecord(- oldPath === path ? manifest : Buffer.from("fixture\n"),- oldPath,- ),+ ["Apache-2.0", "a".repeat(40), undefined],+ ["MIT", "a".repeat(40), "expected public metadata"],+ ["Apache-2.0", "b".repeat(40), "gitHead must match release commit"],+ ] as const)(+ "validates metadata from the PAX manifest npm installs: %s, %s",+ async (license, gitHead, error) => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-npm-manifest-"));+ try {+ const path = "package/package.json";+ const manifest = Buffer.from(+ JSON.stringify({+ name: "@openai/codex-security",+ version: "1.0.0",+ license,+ gitHead,+ }),);- } else {- const overwrite = representation.startsWith("overwrite");- const logicalPath = overwrite ? "package/README.md" : path;- let stored = overwrite- ? Buffer.from(- representation === "overwrite-json"- ? "fixture\n"- : JSON.stringify({- ...metadata,- ...(representation === "overwrite-metadata"- ? { license: "MIT" }- : { gitHead: "b".repeat(40) }),- }),- )- : manifest;- const attributes: Record<string, string> = {- "GNU.sparse.size": String(stored.length),- "GNU.sparse.numblocks": "1",- "GNU.sparse.map": `0,${stored.length}`,- "GNU.sparse.name": logicalPath,- };- if (representation === "pax-map-prefix") {- const map = Buffer.alloc(512);- map.write(`1\n0\n${stored.length}\n`);- for (const key of Object.keys(attributes)) delete attributes[key];- Object.assign(attributes, {- "GNU.sparse.major": "1",- "GNU.sparse.minor": "0",- "GNU.sparse.name": path,- "GNU.sparse.realsize": String(stored.length),- });- stored = Buffer.concat([map, stored]);- }- if (representation === "pax-path") attributes["path"] = path;- overrides.set(- logicalPath,- Buffer.concat([- tarRecord(paxRecords(attributes), {- name: "PaxHeaders/manifest",- type: 0x78,- }),- tarRecord(stored, {- name:- representation === "pax-path"- ? "package/synthetic-storage.json"- : path,- }),- ]),+ const entry = Buffer.concat([+ paxExtension({ path }),+ tarRecord(manifest, { name: "package/synthetic-storage.json" }),+ ]);+ const archivePath = join(root, "package.tgz");+ writeFileSync(+ archivePath,+ gzipSync(packageTar({ overrides: new Map([[path, entry]]) })),);- }- const archivePath = join(root, "package.tgz");- writeFileSync(archivePath, gzipSync(packageTar({ overrides })));- const extracted = await execFileAsync(- "tar",- ["-xOzf", archivePath, path],- {- encoding: "buffer",- timeout: 30_000,- windowsHide: true,- },- );- expect(extracted.stderr.toString()).toBe("");- expect(extracted.stdout.equals(manifest)).toBe(true);- const environment: NodeJS.ProcessEnv = {- ...process.env,- npm_config_cache: join(root, "npm-cache"),- CODEX_SECURITY_EXPECTED_GIT_HEAD: gitHead,- };- const installed = await installPackage(root, archivePath, environment);- const invalidJson = [- "oldgnu-manifest",- "pax-map-prefix",- "overwrite-json",- ].includes(representation);- if (invalidJson) expect(installed.status).not.toBe(0);- else {- expect({- status: installed.status,- stderr: installed.status === 0 ? "" : installed.stderr,- }).toEqual({ status: 0, stderr: "" });- const actual = JSON.parse(+ const environment = {+ ...process.env,+ npm_config_cache: join(root, "npm-cache"),+ CODEX_SECURITY_EXPECTED_GIT_HEAD: "a".repeat(40),+ };+ const contractPath = join(root, "contract.json");+ writeFileSync(contractPath, JSON.stringify(pluginContract));+ const installed = await installPackage(root, archivePath, environment);+ expect(installed.status, installed.stderr).toBe(0);+ const installedManifest = JSON.parse(readFileSync(join(root,@@ -862,380 +696,52 @@ describe("npm package tar listings", () => {"utf8",),);- expect(actual.license).toBe(- representation === "overwrite-metadata" ? "MIT" : metadata.license,- );- expect(actual.gitHead).toBe(- representation === "overwrite-git-head" ? "b".repeat(40) : gitHead,+ expect(installedManifest).toMatchObject({ license, gitHead });+ const checked = await checkPackage(+ root,+ archivePath,+ contractPath,+ environment,);+ if (error !== undefined) {+ expect(checked.status).toBe(1);+ expect(checked.stderr).toContain(error);+ } else expect(checked).toEqual({ status: 0, stderr: "" });+ } finally {+ rmSync(root, { force: true, recursive: true });}- const compatible = ["pax-full", "pax-path", "oldgnu-readme"].includes(- representation,- );- const contractPath = join(root, "contract.json");- writeFileSync(contractPath, JSON.stringify(pluginContract));- const checked = await checkPackage(- root,- archivePath,- contractPath,- environment,- );- expect({- status: checked.status,- stderr: compatible ? checked.stderr : "",- }).toEqual({ status: compatible ? 0 : 1, stderr: "" });- if (representation === "overwrite-metadata")- expect(checked.stderr).toContain("expected public metadata");- if (representation === "overwrite-git-head")- expect(checked.stderr).toContain("gitHead must match release commit");- } finally {- rmSync(root, { force: true, recursive: true });- }- });+ },+ );- describe("reconstructed sparse binary assets and discarded metadata", () => {- const logo = readFileSync(- new URL(- "../../../plugins/codex-security/assets/logo.png",- import.meta.url,- ),- );- const oldGnuText = Buffer.alloc(32 * 1024);- const oldGnuExtents = Array.from({ length: 32 }, (_, index) => {- oldGnuText.write("Public contents.", index * 1024);- return { offset: index * 1024, size: 512 };- });- oldGnuExtents.push({ offset: oldGnuText.length, size: 0 });- const unicodeCompressedPayload = Buffer.from(cleanCompressedPayload);- Buffer.from("😀").copy(- unicodeCompressedPayload,- unicodeCompressedPayload.indexOf("Go/w") - 4,- );- const leadingMarker = cleanCompressedPayload.indexOf("Go/w");- const repositoryCompressedPayload = Buffer.from(cleanCompressedPayload);- const repositoryMarker = Buffer.from(- ["github.com", "openai", "openai"].join("/"),- );- repositoryMarker.copy(repositoryCompressedPayload, 300);- const repositoryMarkerEnd = 300 + repositoryMarker.length;- const middle = Math.floor(cleanCompressedPayload.length / 2);- const cases = [- {- name: "brotli",- files: [["runtime.mjs.br", cleanCompressedPayload, true]],- },- {- name: "unicode-brotli",- files: [["runtime.mjs.br", unicodeCompressedPayload, true]],- },- {- name: "sparse-brotli-leading-delimiter",- checkFixture: () => expect(leadingMarker).toBeGreaterThan(0),- files: [- [- "runtime.mjs.br.part-000",- cleanCompressedPayload.subarray(0, leadingMarker),- false,- ],- [- "runtime.mjs.br.part-001",- cleanCompressedPayload.subarray(leadingMarker),- true,- ],- ],- },- {- name: "sparse-brotli-trailing-delimiter",- checkFixture: () =>- expect(- brotliDecompressSync(repositoryCompressedPayload).toString("utf8"),- ).toMatch(/^[A-Za-z0-9]+$/u),- files: [- [- "runtime.mjs.br.part-000",- repositoryCompressedPayload.subarray(0, repositoryMarkerEnd),- true,- ],- [- "runtime.mjs.br.part-001",- repositoryCompressedPayload.subarray(repositoryMarkerEnd),- false,- ],- ],- },- {- name: "mixed-split-brotli",- files: [- [- "runtime.mjs.br.part-000",- cleanCompressedPayload.subarray(0, middle),- true,- ],- [- "runtime.mjs.br.part-001",- cleanCompressedPayload.subarray(middle),- false,- ],- ],- },- {- name: "mixed-sparse-split-tail",- files: [- [- "runtime.mjs.br.part-000",- cleanCompressedPayload.subarray(0, middle),- true,- ],- [- "runtime.mjs.br.part-001",- cleanCompressedPayload.subarray(middle),- "0.1-tail",- ],- ],- error: "npm tarball contains trailing Brotli data",- },- { name: "png", files: [["logo.png", logo, true]] },- {- name: "pax-sparse-zero-byte-hole-png",- files: [["logo.png", logo, "0.1-hole"]],- error: "npm tarball contains an unexpected PNG asset",- },- {- name: "pax-sparse-zero-byte-hole-tail",- files: [["logo.png", logo, "0.1-hole"]],- tailMarker: true,- error:- /npm tarball contains (?:an internal reference\.|an invalid tar entry)/u,- },- {- name: "oldgnu-brotli",- files: [["runtime.mjs.br", cleanCompressedPayload, "oldgnu"]],- error: "npm tarball contains an invalid tar entry",- },- {- name: "oldgnu-png",- files: [["logo.png", logo, "oldgnu"]],- error: "npm tarball contains an invalid tar entry",- },- {- name: "oldgnu-continuations",- files: [["helpers.mjs", oldGnuText, "oldgnu"]],- extents: oldGnuExtents,- },- {- name: "oldgnu-unused-main-slot",- files: [["runtime.mjs.br", cleanCompressedPayload, "oldgnu"]],- metadataMarker: "unused",- error: "npm tarball contains an invalid tar entry",- },- {- name: "oldgnu-unused-continuation-slot",- files: [["helpers.mjs", oldGnuText, "oldgnu"]],- extents: oldGnuExtents,- metadataMarker: "unused",- },- {- name: "oldgnu-continuation-marker",- files: [["helpers.mjs", oldGnuText, "oldgnu"]],- extents: oldGnuExtents,- metadataMarker: "continuation",- error: "npm tarball contains an internal reference.",- },- {- name: "oldgnu-metadata-boundary-marker",- files: [["helpers.mjs", oldGnuText, "oldgnu"]],- extents: oldGnuExtents,- metadataMarker: "boundary",- error: "npm tarball contains an internal reference.",- },- {- name: "oldgnu-text-body-boundary-marker",- files: [- ["helpers.mjs", Buffer.from("synthetic-reference\n"), "oldgnu"],- ],- metadataMarker: "boundary",- error: "npm tarball contains an internal reference.",- },- {- name: "global-sparse-keys",- files: [["runtime.mjs.br", cleanCompressedPayload, false]],- globalSparse: true,- },- {- name: "expanded-marker",- files: [- [- "runtime.mjs.br",- brotliCompressSync(Buffer.from("go/synthetic-reference")),- true,- ],- ],- error: "npm tarball contains an internal reference.",- },- {- name: "sparse-header-map-boundary-marker",- files: [["runtime.mjs.br", cleanCompressedPayload, "1.0"]],- headerMapMarker: true,- error: "npm tarball contains an internal reference.",- },- {- name: "sparse-tail-header-boundary-marker",- files: [- ["runtime.mjs.br", cleanCompressedPayload, "1.0"],- ["extra.mjs", Buffer.from("Public contents."), false],- ],- tailBoundaryMarker: true,- error:- /npm tarball contains (?:an internal reference\.|an invalid tar entry:)/u,- },- {- name: "map-padding-marker",- files: [["runtime.mjs.br", cleanCompressedPayload, "1.0"]],- mapMarker: true,- error: "npm tarball contains an internal reference.",- },- {- name: "discarded-tail-marker",- files: [["logo.png", logo, "1.0"]],- tailMarker: true,- error:- /npm tarball contains (?:an internal reference\.|an invalid tar entry:)/u,- },- ] satisfies {- name: string;- checkFixture?: () => void;- files: [- string,- Buffer,- boolean | "1.0" | "0.1-tail" | "0.1-hole" | "oldgnu",- ][];- extents?: { offset: number; size: number }[];- metadataMarker?: "continuation" | "boundary" | "unused";- globalSparse?: boolean;- mapMarker?: boolean;- headerMapMarker?: boolean;- tailMarker?: boolean;- tailBoundaryMarker?: boolean;- error?: string | RegExp;- }[];- test.each(cases)("$name", async (scenario) => {- scenario.checkFixture?.();- const root = mkdtempSync(join(tmpdir(), "codex-package-sparse-assets-"));+ test.each([+ ["clean Brotli", cleanCompressedPayload, undefined],+ [+ "expanded marker",+ brotliCompressSync(Buffer.from("go/synthetic-reference")),+ "internal reference",+ ],+ [+ "trailing bytes",+ Buffer.concat([cleanCompressedPayload, Buffer.from("extra")]),+ "trailing Brotli data",+ ],+ ] as const)(+ "checks regular compressed assets: %s",+ async (_label, contents, error) => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-assets-"));try {- const records = scenario.files.map(([name, contents, sparse]) => {- const path = `package/_bundled_plugin/${name}`;- if (!sparse) return tarRecord(contents, { name: path });- if (sparse === "oldgnu")- return oldGnuSparseRecord(- contents,- path,- scenario.extents,- scenario.metadataMarker,- );- if (sparse === "0.1-hole") {- const hole = contents.findIndex(- (byte, index) => index > 0 && index % 512 === 0 && byte === 0,- );- expect(hole).toBeGreaterThan(0);- const stored = Buffer.concat([- contents.subarray(0, hole),- contents.subarray(hole + 1),- scenario.tailMarker- ? Buffer.from("go/synthetic-reference")- : Buffer.alloc(0),- ]);- return Buffer.concat([- tarRecord(- paxRecords({- "GNU.sparse.size": String(contents.length),- "GNU.sparse.numblocks": "2",- "GNU.sparse.map": `0,${hole},${hole + 1},${contents.length - hole - 1}`,- }),- { name: "PaxHeaders/asset", type: 0x78 },- ),- tarRecord(stored, { name: path }),- ]);- }- if (sparse === true) {- return Buffer.concat([- tarRecord(- paxRecords({- "GNU.sparse.size": String(contents.length),- "GNU.sparse.numblocks": "1",- "GNU.sparse.map": `0,${contents.length}`,- }),- { name: "PaxHeaders/asset", type: 0x78 },- ),- tarRecord(contents, { name: path }),- ]);- }- if (sparse === "0.1-tail") {- const storedContents = Buffer.concat([- contents,- Buffer.from("go/synthetic-reference"),- ]);- return Buffer.concat([- tarRecord(- paxRecords({- "GNU.sparse.size": String(storedContents.length),- "GNU.sparse.numblocks": "1",- "GNU.sparse.map": `0,${storedContents.length}`,- }),- { name: "PaxHeaders/asset", type: 0x78 },- ),- tarRecord(storedContents, { name: path }),- ]);- }- const map = Buffer.alloc(512);- map.write(`1\n0\n${contents.length}\n`);- if (scenario.mapMarker) map.write("go/synthetic-reference", 128);- const tail = scenario.tailBoundaryMarker- ? Buffer.alloc((512 - (contents.length % 512)) % 512 || 512)- : scenario.tailMarker- ? Buffer.from("go/synthetic-reference")- : Buffer.alloc(0);- if (scenario.tailBoundaryMarker) tail.write("go/", tail.length - 3);- return Buffer.concat([- tarRecord(- paxRecords({- "GNU.sparse.major": "1",- "GNU.sparse.minor": "0",- "GNU.sparse.name": path,- "GNU.sparse.realsize": String(contents.length),- }),- { name: "PaxHeaders/asset", type: 0x78 },- ),- tarRecord(Buffer.concat([map, contents, tail]), {- name: "package/GNUSparseFile.1/asset",- ...(scenario.headerMapMarker- ? {- reserved: Buffer.concat([- Buffer.alloc(9),- Buffer.from("go/"),- ]),- }- : {}),- }),- ]);- });- const archivePath = join(root, `${scenario.name}.tgz`);+ const name = "runtime.mjs.br";+ const archivePath = join(root, "package.tgz");writeFileSync(archivePath,gzipSync(- Buffer.concat([- ...(scenario.globalSparse- ? [- tarRecord(- paxRecords({- "GNU.sparse.major": "1",- "GNU.sparse.minor": "0",- }),- { name: "GlobalHead", type: 0x67 },- ),- ]- : []),- packageTar({ additionalRecords: records }),- ]),+ packageTar({+ additionalRecords: [+ tarRecord(contents, {+ name: `package/_bundled_plugin/${name}`,+ }),+ ],+ }),),);const contractPath = join(root, "contract.json");@@ -1243,33 +749,26 @@ describe("npm package tar listings", () => {contractPath,JSON.stringify({...pluginContract,- shippedExact: [- ...pluginContract.shippedExact,- ...scenario.files.map(([name]) => name),- ],+ shippedExact: [...pluginContract.shippedExact, name],}),);const environment = { ...process.env };delete environment["CODEX_SECURITY_EXPECTED_GIT_HEAD"];- const result = await checkPackage(+ const checked = await checkPackage(root,archivePath,contractPath,environment,);- if (scenario.error) {- expect(result.status, result.stderr).toBe(1);- if (typeof scenario.error === "string")- expect(result.stderr).toContain(scenario.error);- else expect(result.stderr).toMatch(scenario.error);- } else {- expect(result).toEqual({ status: 0, stderr: "" });- }+ if (error) {+ expect(checked.status).toBe(1);+ expect(checked.stderr).toContain(error);+ } else expect(checked).toEqual({ status: 0, stderr: "" });} finally {rmSync(root, { force: true, recursive: true });}- });- });+ },+ );test.each([0x78, 0x67])("preserves inherited character locale for Unicode pax owners, type=%i",@@ -1383,7 +882,7 @@ process.exit(99);const proxySource = `#!/usr/bin/env nodeconst { spawnSync } = require("node:child_process");const { createHash } = require("node:crypto");-const { appendFileSync, readFileSync } = require("node:fs");+const { appendFileSync, chmodSync, readFileSync, writeFileSync } = require("node:fs");const input = readFileSync(0);appendFileSync(process.env.TAR_PROXY_LOG,@@ -1394,13 +893,25 @@ appendFileSync(inputSha256: createHash("sha256").update(input).digest("hex"),}) + "\\n",);+const mismatch = process.env.TAR_PROXY_MISMATCH;+const alterListing = mismatch === "size" && process.argv.includes("-tvzf");const result = spawnSync(process.env.REAL_TAR, process.argv.slice(2), {env: process.env,input,- stdio: ["pipe", "inherit", "inherit"],+ stdio: ["pipe", alterListing ? "pipe" : "inherit", "inherit"],windowsHide: true,});if (result.error !== undefined) throw result.error;+if (alterListing) {+ const lines = result.stdout.toString("utf8").split("\\n");+ process.stdout.write(lines.map(line => line.endsWith(" package/README.md")+ ? "-rw------- 0/0 1 1970-01-01 00:00 package/README.md" : line).join("\\n"));+}+if (mismatch === "contents" && process.argv.includes("-xzf") && result.status === 0) {+ const path = require("node:path").join(process.argv[process.argv.indexOf("-C") + 1], "package", "README.md");+ chmodSync(path, 0o600);+ writeFileSync(path, "changed\\n"); // Same length as fixture\\n.+}if (result.status === 0 && process.argv.includes("-xzf") && process.env.TAR_PROXY_FAIL_EXTRACT) {process.stderr.write("synthetic extraction failure\\n");process.exit(23);@@ -1472,6 +983,22 @@ process.exit(result.status ?? 1);"-C",]);+ for (const mismatch of ["size", "contents"]) {+ writeFileSync(logPath, "");+ const rejected = await checkPackage(root, archivePath, contractPath, {+ ...environment,+ TAR_PROXY_MISMATCH: mismatch,+ });+ expect(rejected.status).toBe(1);+ expect(rejected.stderr).toContain(+ "npm tarball contains an invalid tar entry.",+ );+ expect(readFileSync(logPath, "utf8").includes('"-xzf"')).toBe(+ mismatch === "contents",+ );+ expect(readdirSync(extractionDirectory)).toEqual([]);+ }+environment["TAR_PROXY_FAIL_EXTRACT"] = "true";const failed = await checkPackage(root,