Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(package)!: reject sparse entries in release tarballs (#2456)

Michael D'Angeloce25486e8dff
diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md
index 7a60f2de7..fd1eabe7f 100644
--- a/sdk/typescript/TESTING.md
+++ b/sdk/typescript/TESTING.md
@@ -45,6 +45,10 @@ pnpm run test:mcp
pnpm run test
```
+The release checker rejects tar archives with sparse entries. If you created a
+candidate archive with a sparse-aware tar tool, regenerate it from the source
+checkout with `pnpm pack` before signing or publishing it.
+
To collect JUnit and coverage while running the SDK suite, use `pnpm run test:ci` in place of `pnpm run test`. To test what an npm consumer receives, pack the archive and pass its exact path to the inspection command:
```sh
diff --git a/sdk/typescript/scripts/check-package.mjs b/sdk/typescript/scripts/check-package.mjs
index f02bdf0e9..071681810 100644
--- a/sdk/typescript/scripts/check-package.mjs
+++ b/sdk/typescript/scripts/check-package.mjs
@@ -15,14 +15,12 @@ import { gunzipSync } from "node:zlib";
import { packageDistFiles } from "./package-dist-files.mjs";
import {
assertPublicPackageContents,
+ assertPublicText,
MAX_EXPANDED_ASSET_BYTES,
} from "./package-public-content.mjs";
import { assertExpectedGitHead } from "./package-provenance.mjs";
import { packageSmokeTimeouts } from "./package-smoke-timeouts.mjs";
-import {
- assertStoredSparseContents,
- readTarArchive,
-} from "./package-tar-entries.mjs";
+import { readTarArchive } from "./package-tar-entries.mjs";
import {
assertTarListingSizes,
regularTarListingLines,
@@ -57,7 +55,7 @@ const rawEntries = storedArchive.entries;
validatePackagePaths(rawEntries.map(({ path }) => path));
assertPublicPackageContents(
storedArchive.files,
- Buffer.concat(storedArchive.metadata.filter(Buffer.isBuffer)),
+ Buffer.concat(storedArchive.metadata),
);
const processEnvironment = { ...process.env };
delete processEnvironment.TAR_OPTIONS;
@@ -194,6 +192,8 @@ const listingSizes = assertTarListingSizes(
listingLines,
MAX_EXPANDED_ASSET_BYTES,
);
+if (rawEntries.some(({ size }, index) => size !== listingSizes[index]))
+ invalidTarEntry();
for (const [path, name] of [
["package/bin/codex-security.mjs", "CLI"],
["package/_bundled_plugin/scripts/launch_codex_security_mcp", "MCP"],
@@ -215,20 +215,18 @@ function privateExtractionDirectories(directory) {
}
function extractedArchiveFiles() {
- const logicalSizes = new Map();
const expectedPaths = new Map();
- for (const [index, { path }] of rawEntries.entries()) {
+ for (const { path } of rawEntries) {
const directory = path.endsWith("/");
const extractedPath = directory ? path.slice(0, -1) : path;
const type = directory ? "directory" : "file";
const previousType = expectedPaths.get(extractedPath);
if (previousType !== undefined && previousType !== type) invalidTarEntry();
- if (!directory) logicalSizes.set(path, listingSizes[index]);
expectedPaths.set(extractedPath, type);
const parts = extractedPath.split("/");
for (let index = 1; index < parts.length; index++) {
const directory = parts.slice(0, index).join("/");
- if (logicalSizes.has(directory)) invalidTarEntry();
+ if (storedArchive.files.has(directory)) invalidTarEntry();
expectedPaths.set(directory, "directory");
}
}
@@ -239,7 +237,7 @@ function extractedArchiveFiles() {
try {
chmodSync(extractionRoot, 0o700);
const fileList = join(extractionRoot, "members.list");
- writeFileSync(fileList, [...logicalSizes.keys(), ""].join("\0"));
+ writeFileSync(fileList, [...storedArchive.files.keys(), ""].join("\0"));
try {
tar([
"-m",
@@ -278,10 +276,11 @@ function extractedArchiveFiles() {
continue;
}
+ const stored = storedArchive.files.get(path);
if (
!stats.isFile() ||
stats.nlink !== 1 ||
- stats.size !== logicalSizes.get(path) ||
+ stats.size !== stored?.length ||
stats.size > MAX_EXPANDED_ASSET_BYTES ||
expandedBytes > MAX_EXPANDED_ASSET_BYTES - stats.size
) {
@@ -289,12 +288,17 @@ function extractedArchiveFiles() {
}
expandedBytes += stats.size;
chmodSync(extractedPath, 0o600);
- archiveFiles.set(path, readFileSync(extractedPath));
+ const contents = readFileSync(extractedPath);
+ if (!contents.equals(stored)) invalidTarEntry();
+ archiveFiles.set(path, contents);
}
}
visit(extractionRoot);
- if (expectedPaths.size !== 0 || archiveFiles.size !== logicalSizes.size) {
+ if (
+ expectedPaths.size !== 0 ||
+ archiveFiles.size !== storedArchive.files.size
+ ) {
invalidTarEntry();
}
return archiveFiles;
@@ -319,7 +323,10 @@ assertExpectedGitHead(
);
assertPublicPackageContents(archiveFiles);
-assertStoredSparseContents(storedArchive, archiveFiles);
+// Keep the separate UTF-8 decoding of headers and padding alongside the raw metadata scan.
+assertPublicText(
+ storedArchive.metadata.map((part) => part.toString("utf8")).join(""),
+);
for (const path of archiveFiles.keys()) {
if (
/\.(?:png|br(?:\.part-[0-9]+)?)$/iu.test(path) &&
diff --git a/sdk/typescript/scripts/package-public-content.mjs b/sdk/typescript/scripts/package-public-content.mjs
index 2005dce34..7e6fb519d 100644
--- a/sdk/typescript/scripts/package-public-content.mjs
+++ b/sdk/typescript/scripts/package-public-content.mjs
@@ -60,17 +60,7 @@ function assertPublicBrotli(bytes, path) {
assertPublicText(result.buffer.toString("utf8"));
}
-export function assertPublicText(contents, binaryRanges = []) {
- const pattern = new RegExp(internalMarker.source, "giu");
- let match;
- while ((match = pattern.exec(contents)) !== null) {
- if (
- !binaryRanges.some(
- ({ start, end }) =>
- match.index >= start && match.index + match[0].length <= end,
- )
- )
- throw new Error("npm tarball contains an internal reference.");
- pattern.lastIndex = match.index + 1;
- }
+export function assertPublicText(contents) {
+ if (internalMarker.test(contents))
+ throw new Error("npm tarball contains an internal reference.");
}
diff --git a/sdk/typescript/scripts/package-tar-entries.mjs b/sdk/typescript/scripts/package-tar-entries.mjs
index 5b6e81ade..d8a06a630 100644
--- a/sdk/typescript/scripts/package-tar-entries.mjs
+++ b/sdk/typescript/scripts/package-tar-entries.mjs
@@ -5,6 +5,10 @@ function invalidTarEntry() {
throw new Error("npm tarball contains an invalid tar entry.");
}
+function unsupportedSparseEntry() {
+ throw new Error("npm tarball contains an unsupported sparse entry.");
+}
+
function headerText(header, start, end) {
return header.subarray(start, end).toString("utf8").split("\0", 1)[0];
}
@@ -31,8 +35,17 @@ function paxAttributes(contents) {
contents[end - 1] !== 0x0a
)
invalidTarEntry();
+ const key = contents.subarray(separator + 1, equals).toString("utf8");
+ if (
+ key === "GNU.sparse" ||
+ key.startsWith("GNU.sparse.") ||
+ key === "SCHILY.realsize" ||
+ key === "SUN.holesdata"
+ ) {
+ unsupportedSparseEntry();
+ }
attributes.set(
- contents.subarray(separator + 1, equals).toString("utf8"),
+ key,
contents.subarray(equals + 1, end - 1).toString("utf8"),
);
offset = end;
@@ -40,103 +53,10 @@ function paxAttributes(contents) {
return attributes;
}
-function sparseMap(contents, paxMap) {
- if (paxMap) {
- const values = paxMap.split(",").map(Number);
- const extents = [];
- for (let index = 0; index < values.length; index += 2)
- extents.push({ offset: values[index], size: values[index + 1] });
- return { contents, extents, dataOffset: 0 };
- }
- let offset = 0;
- function number() {
- while (offset < contents.length) {
- const end = contents.indexOf(0x0a, offset);
- if (end === -1) invalidTarEntry();
- const line = contents.subarray(offset, end).toString("ascii");
- offset = end + 1;
- if (line.startsWith("#")) continue;
- if (!/^[0-9]*$/u.test(line)) invalidTarEntry();
- return Number(line);
- }
- invalidTarEntry();
- }
- const count = number();
- const extents = [];
- for (let index = 0; index < count; index++) {
- extents.push({ offset: number(), size: number() });
- }
- const dataOffset = Math.ceil(offset / blockSize) * blockSize;
- if (dataOffset > contents.length) invalidTarEntry();
- return { contents, extents, dataOffset };
-}
-
-export function assertStoredSparseContents(archive, extractedFiles) {
- const sparseMetadata = new Map();
- for (const [path, { contents, extents, dataOffset }] of archive.sparseFiles) {
- const extracted = extractedFiles.get(path);
- let metadata;
- // GNU tar pads stored extents; libarchive also accepts packed extents.
- for (const padded of [false, true]) {
- let offset = dataOffset;
- const parts = [contents.subarray(0, dataOffset)];
- const matches = extents.every((extent, index) => {
- const end = offset + extent.size;
- if (
- !contents
- .subarray(offset, end)
- .equals(
- extracted.subarray(extent.offset, extent.offset + extent.size),
- )
- )
- return false;
- const next =
- padded && index < extents.length - 1
- ? Math.ceil(end / blockSize) * blockSize
- : end;
- parts.push(
- { binary: contents.subarray(offset, end) },
- contents.subarray(end, next),
- );
- offset = next;
- return true;
- });
- if (matches) {
- parts.push(contents.subarray(offset));
- metadata = parts;
- break;
- }
- }
- if (metadata === undefined) invalidTarEntry();
- sparseMetadata.set(path, metadata);
- }
- const parts = archive.metadata.flatMap((part) =>
- typeof part === "string" ? sparseMetadata.get(part) : part,
- );
- const binaryRanges = [];
- let text = "";
- for (const part of parts) {
- const binary = !Buffer.isBuffer(part);
- const decoded = (binary ? part.binary : part).toString("utf8");
- if (binary) {
- const previous = binaryRanges.at(-1);
- if (previous?.end === text.length) previous.end += decoded.length;
- else
- binaryRanges.push({
- start: text.length,
- end: text.length + decoded.length,
- });
- }
- text += decoded;
- }
- assertPublicText(text, binaryRanges);
-}
-
export function readTarArchive(archiveBytes) {
const entries = [];
const archiveFiles = new Map();
const archiveMetadata = [];
- const sparseFiles = new Map();
const npmFiles = new Map();
let offset = 0;
const globalAttributes = new Map();
@@ -154,15 +74,19 @@ export function readTarArchive(archiveBytes) {
const signature = header.subarray(257, 265).toString("latin1");
const directory = header[156] === 0x35;
- const oldSparse = header[156] === 0x53;
+ // libarchive also reads the GNU sparse map, continuation flag and real
+ // size from GNU member headers, even on an otherwise regular entry.
+ if (
+ header[156] === 0x53 ||
+ (signature === "ustar \0" &&
+ (header[386] !== 0 || header[482] !== 0 || header[483] !== 0))
+ ) {
+ unsupportedSparseEntry();
+ }
const longName = header[156] === 0x4c;
const extended = header[156] === 0x78 || header[156] === 0x67 || longName;
if (
- (header[156] !== 0 &&
- header[156] !== 0x30 &&
- !directory &&
- !extended &&
- !oldSparse) ||
+ (header[156] !== 0 && header[156] !== 0x30 && !directory && !extended) ||
(signature !== "ustar\0" + "00" &&
signature !== "ustar \0" &&
signature !== "\0".repeat(8))
@@ -179,11 +103,7 @@ export function readTarArchive(archiveBytes) {
? nextAttributes.get(key) || undefined
: globalAttributes.get(key);
const headerPath = prefix === "" ? name : `${prefix}/${name}`;
- const path =
- attribute("GNU.sparse.name") ??
- attribute("path") ??
- nextName ??
- headerPath;
+ const path = attribute("path") ?? nextName ?? headerPath;
if (!extended && (path === "" || path.endsWith("/") !== directory))
invalidTarEntry();
assertPublicText(path);
@@ -192,30 +112,7 @@ export function readTarArchive(archiveBytes) {
const paxSize = extended ? undefined : attribute("size");
if (paxSize !== undefined && !/^[0-9]+$/u.test(paxSize)) invalidTarEntry();
const size = paxSize === undefined ? headerSize : Number(paxSize);
- let contentsStart = offset + blockSize;
- const oldSparseExtents = [];
- if (oldSparse) {
- let map = header;
- let start = 386;
- let count = 4;
- for (;;) {
- let index = 0;
- for (; index < count; index++) {
- const field = start + index * 24;
- if (map[field + 12] === 0) break;
- const offset = octalNumber(map, field, field + 12);
- const size = octalNumber(map, field + 12, field + 24);
- if (offset !== 0 || size !== 0)
- oldSparseExtents.push({ offset, size });
- }
- if (index < count || map[start + count * 24] === 0) break;
- map = archiveBytes.subarray(contentsStart, contentsStart + blockSize);
- if (map.length !== blockSize) invalidTarEntry();
- contentsStart += blockSize;
- start = 0;
- count = 21;
- }
- }
+ const contentsStart = offset + blockSize;
const contentsEnd = contentsStart + size;
const nextOffset = contentsStart + Math.ceil(size / blockSize) * blockSize;
if (nextOffset > archiveBytes.byteLength || (directory && size !== 0)) {
@@ -245,30 +142,9 @@ export function readTarArchive(archiveBytes) {
archiveMetadata.push(archiveBytes.subarray(offset, nextOffset));
else {
const contents = archiveBytes.subarray(contentsStart, contentsEnd);
- // npm ignores old-GNU sparse entries and GNU sparse name overrides.
- const npmPath = oldSparse ? undefined : (nextNpmPath ?? headerPath);
- if (npmPath !== undefined) npmFiles.set(npmPath, contents);
+ npmFiles.set(nextNpmPath ?? headerPath, contents);
+ archiveFiles.set(path, contents);
archiveMetadata.push(archiveBytes.subarray(offset, contentsStart));
- const paxSparseMap = nextAttributes.get("GNU.sparse.map");
- if (
- (oldSparse ||
- paxSparseMap ||
- Number.parseInt(nextAttributes.get("GNU.sparse.major"), 10) ===
- 1) &&
- /\.(?:png|br(?:\.part-[0-9]+)?)$/iu.test(path)
- ) {
- sparseFiles.set(
- path,
- oldSparse
- ? { contents, extents: oldSparseExtents, dataOffset: 0 }
- : sparseMap(contents, paxSparseMap),
- );
- // Retain sparse framing in order with the surrounding headers and padding.
- archiveMetadata.push(path);
- } else {
- archiveFiles.set(path, contents);
- if (oldSparse) archiveMetadata.push(contents);
- }
archiveMetadata.push(archiveBytes.subarray(contentsEnd, nextOffset));
}
entries.push({ path, size });
@@ -281,19 +157,10 @@ export function readTarArchive(archiveBytes) {
if (archiveBytes.subarray(offset).some((byte) => byte !== 0))
invalidTarEntry();
- const deferredStreams = new Set(
- [...sparseFiles.keys()].map((path) => path.replace(/\.part-[0-9]+$/iu, "")),
- );
- for (const path of archiveFiles.keys()) {
- if (deferredStreams.has(path.replace(/\.part-[0-9]+$/iu, ""))) {
- archiveFiles.delete(path);
- }
- }
return {
entries,
files: archiveFiles,
metadata: archiveMetadata,
- sparseFiles,
npmFiles,
};
}
diff --git a/sdk/typescript/tests-ts/package-tar-entries.test.ts b/sdk/typescript/tests-ts/package-tar-entries.test.ts
index f1f790b70..85e2b0748 100644
--- a/sdk/typescript/tests-ts/package-tar-entries.test.ts
+++ b/sdk/typescript/tests-ts/package-tar-entries.test.ts
@@ -1,9 +1,7 @@
-import { readFileSync } from "node:fs";
import { describe, expect, test } from "bun:test";
import {
archive,
blockSize,
- cleanCompressedPayload,
octal,
paxRecords,
tarRecord,
@@ -18,15 +16,11 @@ type PackageTarEntries = {
readTarArchive: (archiveBytes: Buffer) => {
entries: PlainTarEntry[];
files: Map<string, Buffer>;
- metadata: (Buffer | string)[];
+ metadata: Buffer[];
};
- assertStoredSparseContents: (
- archive: ReturnType<PackageTarEntries["readTarArchive"]>,
- files: Map<string, Buffer>,
- ) => void;
};
-const { readTarArchive, assertStoredSparseContents } = (await import(
+const { readTarArchive } = (await import(
new URL("../scripts/package-tar-entries.mjs", import.meta.url).href
)) as PackageTarEntries;
const { assertPublicPackageContents } = (await import(
@@ -40,16 +34,12 @@ const { assertPublicPackageContents } = (await import(
function plainTarEntries(bytes: Buffer): PlainTarEntry[] {
const archive = readTarArchive(bytes);
- assertPublicPackageContents(
- archive.files,
- Buffer.concat(
- archive.metadata.filter((part): part is Buffer => Buffer.isBuffer(part)),
- ),
- );
+ assertPublicPackageContents(archive.files, Buffer.concat(archive.metadata));
return archive.entries;
}
const invalidTarEntryError = "npm tarball contains an invalid tar entry.";
+const sparseEntryError = "npm tarball contains an unsupported sparse entry.";
const internalReferenceError = "npm tarball contains an internal reference.";
describe("plain npm tar entries", () => {
@@ -72,7 +62,8 @@ describe("plain npm tar entries", () => {
test.each([0, 0x30])(
"accepts regular typeflag %i and prefix paths",
(type) => {
- const prefix = `package/${"nested/".repeat(13)}deep`;
+ // POSIX prefix bytes can occupy all three old GNU sparse field offsets.
+ const prefix = `package/${"nested/".repeat(20)}deep`;
const longPath = `${prefix}/README.md`;
expect(longPath.length).toBeGreaterThan(100);
@@ -124,7 +115,10 @@ describe("plain npm tar entries", () => {
});
test.each([0x78, 0x67])("accepts POSIX pax metadata typeflag %i", (type) => {
- const attributes = paxRecords({ ctime: "0.123456789" });
+ const attributes = paxRecords({
+ ctime: "0.123456789",
+ "SCHILY.xattr.user.note": "sparse",
+ });
expect(
plainTarEntries(
archive(
@@ -193,124 +187,85 @@ describe("plain npm tar entries", () => {
).toThrow(internalReferenceError);
});
- test.each([false, true])(
- "scans discarded sparse-map padding, marker=%j",
- (marker) => {
- const map = Buffer.alloc(blockSize);
- map.write("1\n512\n512\n");
- if (marker) map.write("go/example", 32);
- const bytes = archive(
- tarRecord(
- paxRecords({
- "GNU.sparse.major": "1",
- "GNU.sparse.minor": "0",
- "GNU.sparse.name": "package/README.md",
- "GNU.sparse.realsize": "1024",
- }),
- { name: "package/PaxHeaders/README.md", type: 0x78 },
- ),
- tarRecord(Buffer.concat([map, Buffer.alloc(blockSize)]), {
- name: "package/README.md",
- }),
- );
- if (marker)
- expect(() => plainTarEntries(bytes)).toThrow(internalReferenceError);
- else
- expect(plainTarEntries(bytes)).toEqual([
- { path: "package/README.md", size: 1024 },
- ]);
- },
- );
-
- for (const format of [
- "gnu-01",
- "gnu-nul",
- "bsd-comments",
- "bsd-solaris",
- ] as const) {
- test.each([false, true])(
- `scans all stored sparse bytes for ${format}, marker=%j`,
- (marker) => {
- let attributes: Record<string, string>;
- let contents: Buffer;
- if (format === "gnu-01") {
- attributes = {
- "GNU.sparse.size": "1024",
- "GNU.sparse.numblocks": "1",
- "GNU.sparse.map": "512,512",
- };
- contents = Buffer.alloc(1024);
- if (marker) contents.write("go/example", 512);
- } else if (format === "bsd-solaris") {
- attributes = { "SUN.holesdata": " 512 1024" };
- contents = Buffer.alloc(1024);
- if (marker) contents.write("go/example", 0);
- } else {
- attributes = {
- "GNU.sparse.major": format === "gnu-nul" ? "1\0" : "1",
- "GNU.sparse.minor": "0",
- "GNU.sparse.name": "package/README.md",
- "GNU.sparse.realsize": format === "bsd-comments" ? "1536" : "1024",
- };
- contents = Buffer.alloc(format === "bsd-comments" ? 1536 : 1024);
- if (format === "bsd-comments") {
- const comments = `${`# ${"x".repeat(60)}\n`.repeat(9)}# ${marker ? "go/example" : "public note"}\n1\n1024\n512\n`;
- contents.write(comments);
- } else {
- contents.write("1\n512\n512\n");
- if (marker) contents.write("go/example", 32);
- }
- }
- const bytes = archive(
- tarRecord(paxRecords(attributes), {
- name: "package/PaxHeaders/README.md",
- type: 0x78,
- }),
- tarRecord(contents, { name: "package/README.md" }),
- );
- if (marker)
- expect(() => plainTarEntries(bytes)).toThrow(internalReferenceError);
- else
- expect(plainTarEntries(bytes)).toEqual([
- { path: "package/README.md", size: contents.length },
- ]);
- },
- );
- }
-
- test("checks stored old GNU sparse bodies", () => {
- const contents = Buffer.alloc(1024);
- contents.write("go/example");
- const record = tarRecord(contents, {
+ test("rejects the old GNU sparse typeflag", () => {
+ const member = tarRecord(Buffer.from("data"), {
name: "package/README.md",
+ type: 0x53,
magic: "ustar ",
version: " \0",
});
- for (const offset of [386, 410]) {
- octal(0, 12).copy(record, offset);
- octal(512, 12).copy(record, offset + 12);
- }
- octal(1024, 12).copy(record, 483);
- expect(() => plainTarEntries(archive(record))).toThrow(
- internalReferenceError,
- );
+ expect(() => readTarArchive(archive(member))).toThrow(sparseEntryError);
});
- test("checks stored bytes after an empty repeated sparse attribute", () => {
- const contents = Buffer.alloc(1024);
- contents.write("go/example", 512);
+ test.each([
+ [386, "00000000000"],
+ [482, "\x01"],
+ [483, "00000000004"],
+ ] as const)(
+ "rejects a regular GNU member with a sparse field at byte %i",
+ (offset, field) => {
+ const member = tarRecord(Buffer.from("data"), {
+ name: "package/README.md",
+ type: 0x30,
+ magic: "ustar ",
+ version: " \0",
+ prefix: "\0".repeat(offset - 345) + field,
+ });
+ expect(() => readTarArchive(archive(member))).toThrow(sparseEntryError);
+ },
+ );
+
+ test.each([0x78, 0x67])(
+ "rejects sparse PAX attributes of type %i",
+ (type) => {
+ for (const [key, value] of [
+ ["GNU.sparse", ""],
+ ["GNU.sparse.map", "0,4"],
+ ["GNU.sparse.major", "1\0"],
+ ["GNU.sparse.name", "package/README.md"],
+ ["SCHILY.realsize", "1024"],
+ ["SUN.holesdata", " 512 1024"],
+ ] as const) {
+ const header = tarRecord(paxRecords({ [key]: value }), {
+ name: "PaxHeaders/readme",
+ type,
+ });
+ expect(() =>
+ readTarArchive(
+ archive(
+ header,
+ tarRecord(Buffer.from("data"), { name: "package/README.md" }),
+ ),
+ ),
+ ).toThrow(sparseEntryError);
+ }
+ },
+ );
+
+ test("rejects trailing sparse metadata even when the attribute is reset", () => {
const attributes = Buffer.concat([
- paxRecords({ "GNU.sparse.map": "512,512" }),
+ paxRecords({ "GNU.sparse.map": "0,4" }),
paxRecords({ "GNU.sparse.map": "" }),
]);
expect(() =>
- plainTarEntries(
+ readTarArchive(
archive(
+ tarRecord(Buffer.from("data"), { name: "package/README.md" }),
tarRecord(attributes, { name: "PaxHeaders/readme", type: 0x78 }),
- tarRecord(contents, { name: "package/README.md" }),
),
),
- ).toThrow(internalReferenceError);
+ ).toThrow(sparseEntryError);
+ });
+
+ test("scans regular file contents under a GNU header", () => {
+ const member = tarRecord(Buffer.from("go/example"), {
+ name: "package/README.md",
+ magic: "ustar ",
+ version: " \0",
+ });
+ expect(() => plainTarEntries(archive(member))).toThrow(
+ internalReferenceError,
+ );
});
test("checks markers across concatenated archive metadata", () => {
@@ -328,132 +283,6 @@ describe("plain npm tar entries", () => {
).toThrow(internalReferenceError);
});
- test.each([false, true])(
- "keeps public Brotli content semantics across retained sparse extents, split=%j",
- (split) => {
- const path = "package/runtime.mjs.br";
- const size = cleanCompressedPayload.length;
- const middle = cleanCompressedPayload.indexOf("Go/w") + 2;
- expect(middle).toBeGreaterThan(2);
- const bytes = archive(
- tarRecord(
- paxRecords({
- "GNU.sparse.size": String(size),
- "GNU.sparse.numblocks": split ? "2" : "1",
- "GNU.sparse.map": split
- ? `0,${middle},${middle},${size - middle}`
- : `0,${size}`,
- }),
- { name: "PaxHeaders/runtime", type: 0x78 },
- ),
- tarRecord(cleanCompressedPayload, { name: path }),
- );
- expect(plainTarEntries(bytes)).toEqual([{ path, size }]);
- expect(() =>
- assertStoredSparseContents(
- readTarArchive(bytes),
- new Map([[path, cleanCompressedPayload]]),
- ),
- ).not.toThrow();
- },
- );
-
- test("keeps retained extents between separate sparse metadata segments", () => {
- const path = "package/logo.png";
- const logo = readFileSync(
- new URL(
- "../../../plugins/codex-security/assets/logo.png",
- import.meta.url,
- ),
- );
- const firstPadding = Buffer.alloc(512 - 250);
- firstPadding.write(".openai.", firstPadding.length - 8);
- const secondPadding = Buffer.alloc(512 - 250);
- secondPadding.write("org");
- const bytes = archive(
- tarRecord(
- paxRecords({
- "GNU.sparse.size": String(logo.length),
- "GNU.sparse.numblocks": "3",
- "GNU.sparse.map": `0,250,250,250,500,${logo.length - 500}`,
- }),
- { name: "PaxHeaders/logo", type: 0x78 },
- ),
- tarRecord(
- Buffer.concat([
- logo.subarray(0, 250),
- firstPadding,
- logo.subarray(250, 500),
- secondPadding,
- logo.subarray(500),
- ]),
- { name: path },
- ),
- );
- expect(() =>
- assertStoredSparseContents(
- readTarArchive(bytes),
- new Map([[path, logo]]),
- ),
- ).not.toThrow();
- });
-
- test("checks markers crossing discarded padding and retained sparse bytes", () => {
- const path = "package/logo.png";
- const logo = readFileSync(
- new URL(
- "../../../plugins/codex-security/assets/logo.png",
- import.meta.url,
- ),
- );
- expect(logo.subarray(387, 390).toString()).toBe("org");
- const padding = Buffer.alloc(512 - 387);
- padding.write(".openai.", padding.length - 8);
- const bytes = archive(
- tarRecord(
- paxRecords({
- "GNU.sparse.size": String(logo.length),
- "GNU.sparse.numblocks": "2",
- "GNU.sparse.map": `0,387,387,${logo.length - 387}`,
- }),
- { name: "PaxHeaders/logo", type: 0x78 },
- ),
- tarRecord(
- Buffer.concat([logo.subarray(0, 387), padding, logo.subarray(387)]),
- { name: path },
- ),
- );
- expect(() =>
- assertStoredSparseContents(
- readTarArchive(bytes),
- new Map([[path, logo]]),
- ),
- ).toThrow(internalReferenceError);
- });
-
- test("uses the native GNU sparse name while retaining stored content", () => {
- const path = "package/README.md";
- const stored = Buffer.alloc(1024);
- stored.write("1\n512\n512\n");
- stored.fill(0x78, 512);
- expect(
- plainTarEntries(
- archive(
- tarRecord(
- paxRecords({
- "GNU.sparse.major": "1",
- "GNU.sparse.minor": "0",
- "GNU.sparse.name": path,
- "GNU.sparse.realsize": "1024",
- }),
- { name: "PaxHeaders/readme", type: 0x78 },
- ),
- tarRecord(stored, { name: "package/GNUSparseFile.1/readme" }),
- ),
- ),
- ).toEqual([{ path, size: stored.byteLength }]);
- });
-
test("accepts an empty size field for an empty file", () => {
expect(
plainTarEntries(
diff --git a/sdk/typescript/tests-ts/package-tar-fixtures.ts b/sdk/typescript/tests-ts/package-tar-fixtures.ts
index 550e1534b..d942ec89d 100644
--- a/sdk/typescript/tests-ts/package-tar-fixtures.ts
+++ b/sdk/typescript/tests-ts/package-tar-fixtures.ts
@@ -66,62 +66,6 @@ export function tarRecord(
]);
}
-export function oldGnuSparseRecord(
- contents: Buffer,
- name: string,
- extents = [{ offset: 0, size: contents.length }],
- metadataMarker?: "continuation" | "boundary" | "unused",
-): Buffer {
- const stored = Buffer.concat(
- extents.flatMap(({ offset, size }, index) => [
- contents.subarray(offset, offset + size),
- Buffer.alloc(index < extents.length - 1 ? (512 - (size % 512)) % 512 : 0),
- ]),
- );
- const record = tarRecord(stored, {
- name,
- type: 0x53,
- magic: "ustar ",
- version: " \0",
- });
- const header = record.subarray(0, blockSize);
- const continuations = [];
- let remaining = extents;
- let map = header;
- let start = 386;
- let count = 4;
- do {
- for (const [index, extent] of remaining.slice(0, count).entries()) {
- octal(extent.offset, 12).copy(map, start + index * 24);
- octal(extent.size, 12).copy(map, start + index * 24 + 12);
- }
- remaining = remaining.slice(count);
- map[start + count * 24] = remaining.length > 0 ? 1 : 0;
- if (remaining.length > 0) {
- map = Buffer.alloc(blockSize);
- continuations.push(map);
- start = 0;
- count = 21;
- }
- } while (remaining.length > 0);
- if (metadataMarker === "unused") {
- const used =
- extents.length <= 4 ? extents.length : ((extents.length - 5) % 21) + 1;
- map.write("public", start + (used + 1) * 24);
- }
- octal(contents.length, 12).copy(header, 483);
- if (metadataMarker === "continuation")
- continuations.at(-1)!.write("go/test", 505);
- if (metadataMarker === "boundary") header.write("go/", 509);
- header.fill(0x20, 148, 156);
- octal(
- header.reduce((sum, byte) => sum + byte, 0),
- 8,
- "\0 ",
- ).copy(header, 148);
- return Buffer.concat([header, ...continuations, record.subarray(blockSize)]);
-}
-
export function archive(...records: Buffer[]): Buffer {
return Buffer.concat([...records, Buffer.alloc(blockSize * 2)]);
}
diff --git a/sdk/typescript/tests-ts/package-tar-listing.test.ts b/sdk/typescript/tests-ts/package-tar-listing.test.ts
index 9c8277005..48448f9f9 100644
--- a/sdk/typescript/tests-ts/package-tar-listing.test.ts
+++ b/sdk/typescript/tests-ts/package-tar-listing.test.ts
@@ -15,13 +15,12 @@ import { tmpdir } from "node:os";
import { delimiter, join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
-import { brotliCompressSync, brotliDecompressSync, gzipSync } from "node:zlib";
+import { brotliCompressSync, gzipSync } from "node:zlib";
import { describe, expect, test } from "bun:test";
import {
archive,
cleanCompressedPayload,
octal,
- oldGnuSparseRecord,
paxRecords,
tarRecord,
} from "./package-tar-fixtures.js";
@@ -44,6 +43,10 @@ const pluginContract = {
shippedExact: ["scripts/launch_codex_security_mcp"],
};
+function paxExtension(attributes: Record<string, string>, type = 0x78): Buffer {
+ return tarRecord(paxRecords(attributes), { name: "PaxHeaders/entry", type });
+}
+
function packageTar({
trailingZeroBytes = 0,
sizeTerminator = " ",
@@ -53,7 +56,6 @@ function packageTar({
directoryMode = 0o755,
readmeMode = 0o644,
mtime = 0,
- readmeSparse,
readmeLongName = false,
additionalRecords = [],
overrides = new Map(),
@@ -66,7 +68,6 @@ function packageTar({
directoryMode?: number;
readmeMode?: number;
mtime?: number;
- readmeSparse?: "0.0" | "1.0";
readmeLongName?: boolean;
additionalRecords?: Buffer[];
overrides?: ReadonlyMap<string, Buffer>;
@@ -97,37 +98,6 @@ function packageTar({
const records = paths.map((path) => {
const override = overrides.get(path);
if (override !== undefined) return override;
- if (path === "package/README.md" && readmeSparse !== undefined) {
- const contents = Buffer.alloc(512, 0x78);
- const map = Buffer.alloc(512);
- map.write("1\n512\n512\n");
- const attributes: Record<string, string> =
- readmeSparse === "0.0"
- ? {
- "GNU.sparse.size": "1024",
- "GNU.sparse.numblocks": "1",
- "GNU.sparse.map": "512,512",
- }
- : {
- "GNU.sparse.major": "1",
- "GNU.sparse.minor": "0",
- "GNU.sparse.name": path,
- "GNU.sparse.realsize": "1024",
- };
- return Buffer.concat([
- tarRecord(paxRecords(attributes), {
- name: "PaxHeaders/readme",
- type: 0x78,
- }),
- tarRecord(
- readmeSparse === "0.0" ? contents : Buffer.concat([map, contents]),
- {
- name:
- readmeSparse === "0.0" ? path : "package/GNUSparseFile.1/readme",
- },
- ),
- ]);
- }
const contents =
path === "package/package.json"
? Buffer.from(
@@ -330,7 +300,7 @@ describe("npm package tar listings", () => {
test.each([
"-rw-r--r-- 0/0 33554433 1970-01-01 00:00 package/README.md",
"-rw-r--r-- 0 0 0 33554433 Jan 1 1970 package/README.md",
- ])("bounds native sparse logical sizes before extraction: %s", (line) => {
+ ])("bounds native file sizes before extraction: %s", (line) => {
expect(() => assertTarListingSizes([line], 32 * 1024 * 1024)).toThrow(
"npm tarball contains an invalid tar entry",
);
@@ -343,7 +313,7 @@ describe("npm package tar listings", () => {
).toThrow();
});
- test("retains bounded logical sparse sizes in listing order", () => {
+ test("retains bounded file sizes in listing order", () => {
expect(
assertTarListingSizes(
[
@@ -428,8 +398,6 @@ describe("npm package tar listings", () => {
const archives = [
["default", gzipSync(tarBytes)],
["level-0", gzipSync(tarBytes, { level: 0 })],
- ["sparse-0.0", gzipSync(packageTar({ readmeSparse: "0.0" }))],
- ["sparse-1.0", gzipSync(packageTar({ readmeSparse: "1.0" }))],
[
"gnu-long-name",
gzipSync(
@@ -535,23 +503,55 @@ describe("npm package tar listings", () => {
);
test.each([
- "pax-full",
- "pax-map-prefix",
- "pax-hole",
- "oldgnu",
- "sparse-name",
+ [
+ "old GNU",
+ tarRecord(Buffer.alloc(0), {
+ name: "package/README.md",
+ type: 0x53,
+ magic: "ustar ",
+ version: " \0",
+ }),
+ ],
+ ["GNU PAX map", paxExtension({ "GNU.sparse.map": "0,1" })],
+ ["GNU global PAX", paxExtension({ "GNU.sparse.major": "1" }, 0x67)],
+ ["Solaris hole map", paxExtension({ "SUN.holesdata": " 0 1" })],
+ ["star logical size", paxExtension({ "SCHILY.realsize": "1" })],
+ ] as const)(
+ "rejects %s sparse entries before invoking native tar",
+ async (_label, record) => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-sparse-"));
+ try {
+ const archivePath = join(root, "sparse.tgz");
+ writeFileSync(
+ archivePath,
+ gzipSync(packageTar({ additionalRecords: [record] })),
+ );
+ const contractPath = join(root, "contract.json");
+ writeFileSync(contractPath, JSON.stringify(pluginContract));
+ const environment = { ...process.env, PATH: root };
+ const checked = await checkPackage(
+ root,
+ archivePath,
+ contractPath,
+ environment,
+ );
+ expect(checked.status).toBe(1);
+ expect(checked.stderr).toContain(
+ "npm tarball contains an unsupported sparse entry.",
+ );
+ } finally {
+ rmSync(root, { force: true, recursive: true });
+ }
+ },
+ );
+
+ test.each([
"pax-path",
- "global-path",
- "pax-then-long-name",
- "long-name-then-pax",
"pax-then-wrong-long-name",
- "overwrite-text",
- "overwrite-logo",
- "ordinary-pax-then-wrong-long-name",
- "ordinary-pax-then-correct-long-name",
- "ordinary-long-name-then-pax",
- ])("validates assets installed by npm: %s", async (representation) => {
- const root = mkdtempSync(join(tmpdir(), "codex-package-npm-sparse-"));
+ "pax-then-correct-long-name",
+ "long-name-then-pax",
+ ])("validates the asset path used by npm: %s", async (order) => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-npm-path-"));
try {
const logo = readFileSync(
new URL(
@@ -560,97 +560,28 @@ describe("npm package tar listings", () => {
),
);
const path = "package/_bundled_plugin/logo.png";
- const renamed =
- representation === "sparse-name" || representation === "pax-path";
- const wrongPath = "package/synthetic-storage.png";
- const ordinary = representation.startsWith("ordinary-");
- const pathOrder = ordinary
- ? representation.slice("ordinary-".length)
- : representation;
- const wrongLongName = pathOrder === "pax-then-wrong-long-name";
- const orderedPath = representation.includes("then");
- const overwrite = representation.startsWith("overwrite");
- let stored =
- representation === "overwrite-text" ? Buffer.from("fixture\n") : logo;
- const attributes: Record<string, string> = ordinary
- ? {}
- : {
- "GNU.sparse.size": String(stored.length),
- "GNU.sparse.numblocks": "1",
- "GNU.sparse.map": `0,${stored.length}`,
- };
- if (representation === "pax-map-prefix") {
- const map = Buffer.alloc(512);
- map.write(`1\n0\n${logo.length}\n`);
- stored = Buffer.concat([map, logo]);
- for (const key of Object.keys(attributes)) delete attributes[key];
- Object.assign(attributes, {
- "GNU.sparse.major": "1",
- "GNU.sparse.minor": "0",
- "GNU.sparse.name": path,
- "GNU.sparse.realsize": String(logo.length),
- });
- } else if (representation === "pax-hole") {
- const hole = logo.findIndex(
- (byte, index) => index > 0 && index % 512 === 0 && byte === 0,
- );
- expect(hole).toBeGreaterThan(0);
- stored = Buffer.concat([
- logo.subarray(0, hole),
- logo.subarray(hole + 1),
- ]);
- attributes["GNU.sparse.numblocks"] = "2";
- attributes["GNU.sparse.map"] =
- `0,${hole},${hole + 1},${logo.length - hole - 1}`;
- }
- if (
- !ordinary &&
- (renamed || orderedPath || representation === "global-path")
- )
- attributes["GNU.sparse.name"] = path;
- if (overwrite) attributes["GNU.sparse.name"] = "package/README.md";
- if (representation === "pax-path" || orderedPath)
- attributes["path"] =
- representation === "pax-then-long-name" ? wrongPath : path;
+ const storagePath = "package/synthetic-storage.png";
+ const wrongLongName = order === "pax-then-wrong-long-name";
+ const pax = paxExtension({ path });
const longName = tarRecord(
- Buffer.from(
- `${representation === "pax-then-long-name" || representation === "ordinary-pax-then-correct-long-name" || pathOrder === "long-name-then-pax" ? path : wrongPath}\0`,
- ),
+ Buffer.from(`${wrongLongName ? storagePath : path}\0`),
{ name: "././@LongLink", type: 0x4c },
);
- const record =
- representation === "oldgnu"
- ? oldGnuSparseRecord(logo, path)
- : Buffer.concat([
- ...(representation === "global-path"
- ? [
- tarRecord(paxRecords({ path: wrongPath }), {
- name: "GlobalHead",
- type: 0x67,
- }),
- ]
- : []),
- ...(pathOrder === "long-name-then-pax" ? [longName] : []),
- tarRecord(paxRecords(attributes), {
- name: "PaxHeaders/asset",
- type: 0x78,
- }),
- ...(pathOrder.startsWith("pax-then") ? [longName] : []),
- tarRecord(stored, {
- name: renamed || ordinary ? wrongPath : path,
- }),
- ]);
+ const headers =
+ order === "pax-path"
+ ? [pax]
+ : order === "long-name-then-pax"
+ ? [longName, pax]
+ : [pax, longName];
const archivePath = join(root, "package.tgz");
writeFileSync(
archivePath,
gzipSync(
packageTar({
- overrides: overwrite
- ? new Map([["package/README.md", Buffer.alloc(0)]])
- : undefined,
- additionalRecords: overwrite
- ? [tarRecord(logo, { name: path }), record]
- : [record],
+ additionalRecords: [
+ ...headers,
+ tarRecord(logo, { name: storagePath }),
+ ],
}),
),
);
@@ -674,182 +605,85 @@ describe("npm package tar listings", () => {
contractPath,
environment,
);
- // The wrong-long-name fixtures have no portable asset path to select.
+ // The two readers can disagree on a final GNU long name. Extract everything in that case.
const extraction = execFileAsync(
"tar",
["-xzf", archivePath, "-C", native, ...(wrongLongName ? [] : [path])],
- { encoding: "buffer", timeout: 30_000, windowsHide: true },
+ {
+ encoding: "buffer",
+ timeout: 30_000,
+ windowsHide: true,
+ },
);
const installation = installPackage(root, archivePath, environment);
- // All three read the same archive. Keep it until every reader exits, including on failure.
await Promise.allSettled([extraction, installation, packageCheck]);
- const extracted = await extraction;
- expect(extracted.stderr.toString()).toBe("");
+ expect((await extraction).stderr.toString()).toBe("");
if (!wrongLongName)
- expect(readFileSync(join(native, path)).equals(logo)).toBe(true);
+ expect(readFileSync(join(native, path))).toEqual(logo);
const installed = await installation;
- const consumer = join(root, "consumer");
- expect({
- status: installed.status,
- stderr: installed.status === 0 ? "" : installed.stderr,
- }).toEqual({ status: 0, stderr: "" });
+ expect(installed.status, installed.stderr).toBe(0);
const installedPackage = join(
- consumer,
+ root,
+ "consumer",
"node_modules",
"@openai",
"codex-security",
);
- const installedLogo = join(
- installedPackage,
- "_bundled_plugin",
- "logo.png",
- );
- if (wrongLongName) {
- expect(existsSync(installedLogo)).toBe(false);
+ if (wrongLongName)
expect(
- readFileSync(
- join(installedPackage, wrongPath.slice("package/".length)),
- ).equals(logo),
- ).toBe(true);
- }
- const compatible = [
- "pax-full",
- "pax-path",
- "global-path",
- "pax-then-long-name",
- "long-name-then-pax",
- "overwrite-logo",
- "ordinary-pax-then-correct-long-name",
- "ordinary-long-name-then-pax",
- ].includes(representation);
+ existsSync(join(installedPackage, "_bundled_plugin", "logo.png")),
+ ).toBe(false);
+ const installedPath = wrongLongName ? storagePath : path;
expect(
- existsSync(installedLogo) && readFileSync(installedLogo).equals(logo),
- ).toBe(compatible);
- if (overwrite)
- expect(readFileSync(installedLogo).equals(stored)).toBe(true);
+ readFileSync(
+ join(installedPackage, installedPath.slice("package/".length)),
+ ),
+ ).toEqual(logo);
const checked = await packageCheck;
- expect({
- status: checked.status,
- stderr: compatible ? checked.stderr : "",
- }).toEqual({ status: compatible ? 0 : 1, stderr: "" });
+ expect(checked.status, checked.stderr).toBe(wrongLongName ? 1 : 0);
+ if (!wrongLongName) expect(checked.stderr).toBe("");
} finally {
rmSync(root, { force: true, recursive: true });
}
});
test.each([
- "oldgnu-manifest",
- "pax-map-prefix",
- "overwrite-json",
- "overwrite-metadata",
- "overwrite-git-head",
- "pax-full",
- "pax-path",
- "oldgnu-readme",
- ])("validates the manifest npm installs: %s", async (representation) => {
- const root = mkdtempSync(join(tmpdir(), "codex-package-npm-manifest-"));
- try {
- const path = "package/package.json";
- const gitHead = "a".repeat(40);
- const metadata = {
- name: "@openai/codex-security",
- version: "1.0.0",
- license: "Apache-2.0",
- gitHead,
- };
- const manifest = Buffer.from(JSON.stringify(metadata));
- const overrides = new Map([[path, tarRecord(manifest, { name: path })]]);
- if (representation.startsWith("oldgnu")) {
- const oldPath =
- representation === "oldgnu-manifest" ? path : "package/README.md";
- overrides.set(
- oldPath,
- oldGnuSparseRecord(
- oldPath === path ? manifest : Buffer.from("fixture\n"),
- oldPath,
- ),
+ ["Apache-2.0", "a".repeat(40), undefined],
+ ["MIT", "a".repeat(40), "expected public metadata"],
+ ["Apache-2.0", "b".repeat(40), "gitHead must match release commit"],
+ ] as const)(
+ "validates metadata from the PAX manifest npm installs: %s, %s",
+ async (license, gitHead, error) => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-npm-manifest-"));
+ try {
+ const path = "package/package.json";
+ const manifest = Buffer.from(
+ JSON.stringify({
+ name: "@openai/codex-security",
+ version: "1.0.0",
+ license,
+ gitHead,
+ }),
);
- } else {
- const overwrite = representation.startsWith("overwrite");
- const logicalPath = overwrite ? "package/README.md" : path;
- let stored = overwrite
- ? Buffer.from(
- representation === "overwrite-json"
- ? "fixture\n"
- : JSON.stringify({
- ...metadata,
- ...(representation === "overwrite-metadata"
- ? { license: "MIT" }
- : { gitHead: "b".repeat(40) }),
- }),
- )
- : manifest;
- const attributes: Record<string, string> = {
- "GNU.sparse.size": String(stored.length),
- "GNU.sparse.numblocks": "1",
- "GNU.sparse.map": `0,${stored.length}`,
- "GNU.sparse.name": logicalPath,
- };
- if (representation === "pax-map-prefix") {
- const map = Buffer.alloc(512);
- map.write(`1\n0\n${stored.length}\n`);
- for (const key of Object.keys(attributes)) delete attributes[key];
- Object.assign(attributes, {
- "GNU.sparse.major": "1",
- "GNU.sparse.minor": "0",
- "GNU.sparse.name": path,
- "GNU.sparse.realsize": String(stored.length),
- });
- stored = Buffer.concat([map, stored]);
- }
- if (representation === "pax-path") attributes["path"] = path;
- overrides.set(
- logicalPath,
- Buffer.concat([
- tarRecord(paxRecords(attributes), {
- name: "PaxHeaders/manifest",
- type: 0x78,
- }),
- tarRecord(stored, {
- name:
- representation === "pax-path"
- ? "package/synthetic-storage.json"
- : path,
- }),
- ]),
+ const entry = Buffer.concat([
+ paxExtension({ path }),
+ tarRecord(manifest, { name: "package/synthetic-storage.json" }),
+ ]);
+ const archivePath = join(root, "package.tgz");
+ writeFileSync(
+ archivePath,
+ gzipSync(packageTar({ overrides: new Map([[path, entry]]) })),
);
- }
- const archivePath = join(root, "package.tgz");
- writeFileSync(archivePath, gzipSync(packageTar({ overrides })));
- const extracted = await execFileAsync(
- "tar",
- ["-xOzf", archivePath, path],
- {
- encoding: "buffer",
- timeout: 30_000,
- windowsHide: true,
- },
- );
- expect(extracted.stderr.toString()).toBe("");
- expect(extracted.stdout.equals(manifest)).toBe(true);
- const environment: NodeJS.ProcessEnv = {
- ...process.env,
- npm_config_cache: join(root, "npm-cache"),
- CODEX_SECURITY_EXPECTED_GIT_HEAD: gitHead,
- };
- const installed = await installPackage(root, archivePath, environment);
- const invalidJson = [
- "oldgnu-manifest",
- "pax-map-prefix",
- "overwrite-json",
- ].includes(representation);
- if (invalidJson) expect(installed.status).not.toBe(0);
- else {
- expect({
- status: installed.status,
- stderr: installed.status === 0 ? "" : installed.stderr,
- }).toEqual({ status: 0, stderr: "" });
- const actual = JSON.parse(
+ const environment = {
+ ...process.env,
+ npm_config_cache: join(root, "npm-cache"),
+ CODEX_SECURITY_EXPECTED_GIT_HEAD: "a".repeat(40),
+ };
+ const contractPath = join(root, "contract.json");
+ writeFileSync(contractPath, JSON.stringify(pluginContract));
+ const installed = await installPackage(root, archivePath, environment);
+ expect(installed.status, installed.stderr).toBe(0);
+ const installedManifest = JSON.parse(
readFileSync(
join(
root,
@@ -862,380 +696,52 @@ describe("npm package tar listings", () => {
"utf8",
),
);
- expect(actual.license).toBe(
- representation === "overwrite-metadata" ? "MIT" : metadata.license,
- );
- expect(actual.gitHead).toBe(
- representation === "overwrite-git-head" ? "b".repeat(40) : gitHead,
+ expect(installedManifest).toMatchObject({ license, gitHead });
+ const checked = await checkPackage(
+ root,
+ archivePath,
+ contractPath,
+ environment,
);
+ if (error !== undefined) {
+ expect(checked.status).toBe(1);
+ expect(checked.stderr).toContain(error);
+ } else expect(checked).toEqual({ status: 0, stderr: "" });
+ } finally {
+ rmSync(root, { force: true, recursive: true });
}
- const compatible = ["pax-full", "pax-path", "oldgnu-readme"].includes(
- representation,
- );
- const contractPath = join(root, "contract.json");
- writeFileSync(contractPath, JSON.stringify(pluginContract));
- const checked = await checkPackage(
- root,
- archivePath,
- contractPath,
- environment,
- );
- expect({
- status: checked.status,
- stderr: compatible ? checked.stderr : "",
- }).toEqual({ status: compatible ? 0 : 1, stderr: "" });
- if (representation === "overwrite-metadata")
- expect(checked.stderr).toContain("expected public metadata");
- if (representation === "overwrite-git-head")
- expect(checked.stderr).toContain("gitHead must match release commit");
- } finally {
- rmSync(root, { force: true, recursive: true });
- }
- });
+ },
+ );
- describe("reconstructed sparse binary assets and discarded metadata", () => {
- const logo = readFileSync(
- new URL(
- "../../../plugins/codex-security/assets/logo.png",
- import.meta.url,
- ),
- );
- const oldGnuText = Buffer.alloc(32 * 1024);
- const oldGnuExtents = Array.from({ length: 32 }, (_, index) => {
- oldGnuText.write("Public contents.", index * 1024);
- return { offset: index * 1024, size: 512 };
- });
- oldGnuExtents.push({ offset: oldGnuText.length, size: 0 });
- const unicodeCompressedPayload = Buffer.from(cleanCompressedPayload);
- Buffer.from("😀").copy(
- unicodeCompressedPayload,
- unicodeCompressedPayload.indexOf("Go/w") - 4,
- );
- const leadingMarker = cleanCompressedPayload.indexOf("Go/w");
- const repositoryCompressedPayload = Buffer.from(cleanCompressedPayload);
- const repositoryMarker = Buffer.from(
- ["github.com", "openai", "openai"].join("/"),
- );
- repositoryMarker.copy(repositoryCompressedPayload, 300);
- const repositoryMarkerEnd = 300 + repositoryMarker.length;
- const middle = Math.floor(cleanCompressedPayload.length / 2);
- const cases = [
- {
- name: "brotli",
- files: [["runtime.mjs.br", cleanCompressedPayload, true]],
- },
- {
- name: "unicode-brotli",
- files: [["runtime.mjs.br", unicodeCompressedPayload, true]],
- },
- {
- name: "sparse-brotli-leading-delimiter",
- checkFixture: () => expect(leadingMarker).toBeGreaterThan(0),
- files: [
- [
- "runtime.mjs.br.part-000",
- cleanCompressedPayload.subarray(0, leadingMarker),
- false,
- ],
- [
- "runtime.mjs.br.part-001",
- cleanCompressedPayload.subarray(leadingMarker),
- true,
- ],
- ],
- },
- {
- name: "sparse-brotli-trailing-delimiter",
- checkFixture: () =>
- expect(
- brotliDecompressSync(repositoryCompressedPayload).toString("utf8"),
- ).toMatch(/^[A-Za-z0-9]+$/u),
- files: [
- [
- "runtime.mjs.br.part-000",
- repositoryCompressedPayload.subarray(0, repositoryMarkerEnd),
- true,
- ],
- [
- "runtime.mjs.br.part-001",
- repositoryCompressedPayload.subarray(repositoryMarkerEnd),
- false,
- ],
- ],
- },
- {
- name: "mixed-split-brotli",
- files: [
- [
- "runtime.mjs.br.part-000",
- cleanCompressedPayload.subarray(0, middle),
- true,
- ],
- [
- "runtime.mjs.br.part-001",
- cleanCompressedPayload.subarray(middle),
- false,
- ],
- ],
- },
- {
- name: "mixed-sparse-split-tail",
- files: [
- [
- "runtime.mjs.br.part-000",
- cleanCompressedPayload.subarray(0, middle),
- true,
- ],
- [
- "runtime.mjs.br.part-001",
- cleanCompressedPayload.subarray(middle),
- "0.1-tail",
- ],
- ],
- error: "npm tarball contains trailing Brotli data",
- },
- { name: "png", files: [["logo.png", logo, true]] },
- {
- name: "pax-sparse-zero-byte-hole-png",
- files: [["logo.png", logo, "0.1-hole"]],
- error: "npm tarball contains an unexpected PNG asset",
- },
- {
- name: "pax-sparse-zero-byte-hole-tail",
- files: [["logo.png", logo, "0.1-hole"]],
- tailMarker: true,
- error:
- /npm tarball contains (?:an internal reference\.|an invalid tar entry)/u,
- },
- {
- name: "oldgnu-brotli",
- files: [["runtime.mjs.br", cleanCompressedPayload, "oldgnu"]],
- error: "npm tarball contains an invalid tar entry",
- },
- {
- name: "oldgnu-png",
- files: [["logo.png", logo, "oldgnu"]],
- error: "npm tarball contains an invalid tar entry",
- },
- {
- name: "oldgnu-continuations",
- files: [["helpers.mjs", oldGnuText, "oldgnu"]],
- extents: oldGnuExtents,
- },
- {
- name: "oldgnu-unused-main-slot",
- files: [["runtime.mjs.br", cleanCompressedPayload, "oldgnu"]],
- metadataMarker: "unused",
- error: "npm tarball contains an invalid tar entry",
- },
- {
- name: "oldgnu-unused-continuation-slot",
- files: [["helpers.mjs", oldGnuText, "oldgnu"]],
- extents: oldGnuExtents,
- metadataMarker: "unused",
- },
- {
- name: "oldgnu-continuation-marker",
- files: [["helpers.mjs", oldGnuText, "oldgnu"]],
- extents: oldGnuExtents,
- metadataMarker: "continuation",
- error: "npm tarball contains an internal reference.",
- },
- {
- name: "oldgnu-metadata-boundary-marker",
- files: [["helpers.mjs", oldGnuText, "oldgnu"]],
- extents: oldGnuExtents,
- metadataMarker: "boundary",
- error: "npm tarball contains an internal reference.",
- },
- {
- name: "oldgnu-text-body-boundary-marker",
- files: [
- ["helpers.mjs", Buffer.from("synthetic-reference\n"), "oldgnu"],
- ],
- metadataMarker: "boundary",
- error: "npm tarball contains an internal reference.",
- },
- {
- name: "global-sparse-keys",
- files: [["runtime.mjs.br", cleanCompressedPayload, false]],
- globalSparse: true,
- },
- {
- name: "expanded-marker",
- files: [
- [
- "runtime.mjs.br",
- brotliCompressSync(Buffer.from("go/synthetic-reference")),
- true,
- ],
- ],
- error: "npm tarball contains an internal reference.",
- },
- {
- name: "sparse-header-map-boundary-marker",
- files: [["runtime.mjs.br", cleanCompressedPayload, "1.0"]],
- headerMapMarker: true,
- error: "npm tarball contains an internal reference.",
- },
- {
- name: "sparse-tail-header-boundary-marker",
- files: [
- ["runtime.mjs.br", cleanCompressedPayload, "1.0"],
- ["extra.mjs", Buffer.from("Public contents."), false],
- ],
- tailBoundaryMarker: true,
- error:
- /npm tarball contains (?:an internal reference\.|an invalid tar entry:)/u,
- },
- {
- name: "map-padding-marker",
- files: [["runtime.mjs.br", cleanCompressedPayload, "1.0"]],
- mapMarker: true,
- error: "npm tarball contains an internal reference.",
- },
- {
- name: "discarded-tail-marker",
- files: [["logo.png", logo, "1.0"]],
- tailMarker: true,
- error:
- /npm tarball contains (?:an internal reference\.|an invalid tar entry:)/u,
- },
- ] satisfies {
- name: string;
- checkFixture?: () => void;
- files: [
- string,
- Buffer,
- boolean | "1.0" | "0.1-tail" | "0.1-hole" | "oldgnu",
- ][];
- extents?: { offset: number; size: number }[];
- metadataMarker?: "continuation" | "boundary" | "unused";
- globalSparse?: boolean;
- mapMarker?: boolean;
- headerMapMarker?: boolean;
- tailMarker?: boolean;
- tailBoundaryMarker?: boolean;
- error?: string | RegExp;
- }[];
- test.each(cases)("$name", async (scenario) => {
- scenario.checkFixture?.();
- const root = mkdtempSync(join(tmpdir(), "codex-package-sparse-assets-"));
+ test.each([
+ ["clean Brotli", cleanCompressedPayload, undefined],
+ [
+ "expanded marker",
+ brotliCompressSync(Buffer.from("go/synthetic-reference")),
+ "internal reference",
+ ],
+ [
+ "trailing bytes",
+ Buffer.concat([cleanCompressedPayload, Buffer.from("extra")]),
+ "trailing Brotli data",
+ ],
+ ] as const)(
+ "checks regular compressed assets: %s",
+ async (_label, contents, error) => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-assets-"));
try {
- const records = scenario.files.map(([name, contents, sparse]) => {
- const path = `package/_bundled_plugin/${name}`;
- if (!sparse) return tarRecord(contents, { name: path });
- if (sparse === "oldgnu")
- return oldGnuSparseRecord(
- contents,
- path,
- scenario.extents,
- scenario.metadataMarker,
- );
- if (sparse === "0.1-hole") {
- const hole = contents.findIndex(
- (byte, index) => index > 0 && index % 512 === 0 && byte === 0,
- );
- expect(hole).toBeGreaterThan(0);
- const stored = Buffer.concat([
- contents.subarray(0, hole),
- contents.subarray(hole + 1),
- scenario.tailMarker
- ? Buffer.from("go/synthetic-reference")
- : Buffer.alloc(0),
- ]);
- return Buffer.concat([
- tarRecord(
- paxRecords({
- "GNU.sparse.size": String(contents.length),
- "GNU.sparse.numblocks": "2",
- "GNU.sparse.map": `0,${hole},${hole + 1},${contents.length - hole - 1}`,
- }),
- { name: "PaxHeaders/asset", type: 0x78 },
- ),
- tarRecord(stored, { name: path }),
- ]);
- }
- if (sparse === true) {
- return Buffer.concat([
- tarRecord(
- paxRecords({
- "GNU.sparse.size": String(contents.length),
- "GNU.sparse.numblocks": "1",
- "GNU.sparse.map": `0,${contents.length}`,
- }),
- { name: "PaxHeaders/asset", type: 0x78 },
- ),
- tarRecord(contents, { name: path }),
- ]);
- }
- if (sparse === "0.1-tail") {
- const storedContents = Buffer.concat([
- contents,
- Buffer.from("go/synthetic-reference"),
- ]);
- return Buffer.concat([
- tarRecord(
- paxRecords({
- "GNU.sparse.size": String(storedContents.length),
- "GNU.sparse.numblocks": "1",
- "GNU.sparse.map": `0,${storedContents.length}`,
- }),
- { name: "PaxHeaders/asset", type: 0x78 },
- ),
- tarRecord(storedContents, { name: path }),
- ]);
- }
- const map = Buffer.alloc(512);
- map.write(`1\n0\n${contents.length}\n`);
- if (scenario.mapMarker) map.write("go/synthetic-reference", 128);
- const tail = scenario.tailBoundaryMarker
- ? Buffer.alloc((512 - (contents.length % 512)) % 512 || 512)
- : scenario.tailMarker
- ? Buffer.from("go/synthetic-reference")
- : Buffer.alloc(0);
- if (scenario.tailBoundaryMarker) tail.write("go/", tail.length - 3);
- return Buffer.concat([
- tarRecord(
- paxRecords({
- "GNU.sparse.major": "1",
- "GNU.sparse.minor": "0",
- "GNU.sparse.name": path,
- "GNU.sparse.realsize": String(contents.length),
- }),
- { name: "PaxHeaders/asset", type: 0x78 },
- ),
- tarRecord(Buffer.concat([map, contents, tail]), {
- name: "package/GNUSparseFile.1/asset",
- ...(scenario.headerMapMarker
- ? {
- reserved: Buffer.concat([
- Buffer.alloc(9),
- Buffer.from("go/"),
- ]),
- }
- : {}),
- }),
- ]);
- });
- const archivePath = join(root, `${scenario.name}.tgz`);
+ const name = "runtime.mjs.br";
+ const archivePath = join(root, "package.tgz");
writeFileSync(
archivePath,
gzipSync(
- Buffer.concat([
- ...(scenario.globalSparse
- ? [
- tarRecord(
- paxRecords({
- "GNU.sparse.major": "1",
- "GNU.sparse.minor": "0",
- }),
- { name: "GlobalHead", type: 0x67 },
- ),
- ]
- : []),
- packageTar({ additionalRecords: records }),
- ]),
+ packageTar({
+ additionalRecords: [
+ tarRecord(contents, {
+ name: `package/_bundled_plugin/${name}`,
+ }),
+ ],
+ }),
),
);
const contractPath = join(root, "contract.json");
@@ -1243,33 +749,26 @@ describe("npm package tar listings", () => {
contractPath,
JSON.stringify({
...pluginContract,
- shippedExact: [
- ...pluginContract.shippedExact,
- ...scenario.files.map(([name]) => name),
- ],
+ shippedExact: [...pluginContract.shippedExact, name],
}),
);
const environment = { ...process.env };
delete environment["CODEX_SECURITY_EXPECTED_GIT_HEAD"];
- const result = await checkPackage(
+ const checked = await checkPackage(
root,
archivePath,
contractPath,
environment,
);
- if (scenario.error) {
- expect(result.status, result.stderr).toBe(1);
- if (typeof scenario.error === "string")
- expect(result.stderr).toContain(scenario.error);
- else expect(result.stderr).toMatch(scenario.error);
- } else {
- expect(result).toEqual({ status: 0, stderr: "" });
- }
+ if (error) {
+ expect(checked.status).toBe(1);
+ expect(checked.stderr).toContain(error);
+ } else expect(checked).toEqual({ status: 0, stderr: "" });
} finally {
rmSync(root, { force: true, recursive: true });
}
- });
- });
+ },
+ );
test.each([0x78, 0x67])(
"preserves inherited character locale for Unicode pax owners, type=%i",
@@ -1383,7 +882,7 @@ process.exit(99);
const proxySource = `#!/usr/bin/env node
const { spawnSync } = require("node:child_process");
const { createHash } = require("node:crypto");
-const { appendFileSync, readFileSync } = require("node:fs");
+const { appendFileSync, chmodSync, readFileSync, writeFileSync } = require("node:fs");
const input = readFileSync(0);
appendFileSync(
process.env.TAR_PROXY_LOG,
@@ -1394,13 +893,25 @@ appendFileSync(
inputSha256: createHash("sha256").update(input).digest("hex"),
}) + "\\n",
);
+const mismatch = process.env.TAR_PROXY_MISMATCH;
+const alterListing = mismatch === "size" && process.argv.includes("-tvzf");
const result = spawnSync(process.env.REAL_TAR, process.argv.slice(2), {
env: process.env,
input,
- stdio: ["pipe", "inherit", "inherit"],
+ stdio: ["pipe", alterListing ? "pipe" : "inherit", "inherit"],
windowsHide: true,
});
if (result.error !== undefined) throw result.error;
+if (alterListing) {
+ const lines = result.stdout.toString("utf8").split("\\n");
+ process.stdout.write(lines.map(line => line.endsWith(" package/README.md")
+ ? "-rw------- 0/0 1 1970-01-01 00:00 package/README.md" : line).join("\\n"));
+}
+if (mismatch === "contents" && process.argv.includes("-xzf") && result.status === 0) {
+ const path = require("node:path").join(process.argv[process.argv.indexOf("-C") + 1], "package", "README.md");
+ chmodSync(path, 0o600);
+ writeFileSync(path, "changed\\n"); // Same length as fixture\\n.
+}
if (result.status === 0 && process.argv.includes("-xzf") && process.env.TAR_PROXY_FAIL_EXTRACT) {
process.stderr.write("synthetic extraction failure\\n");
process.exit(23);
@@ -1472,6 +983,22 @@ process.exit(result.status ?? 1);
"-C",
]);
+ for (const mismatch of ["size", "contents"]) {
+ writeFileSync(logPath, "");
+ const rejected = await checkPackage(root, archivePath, contractPath, {
+ ...environment,
+ TAR_PROXY_MISMATCH: mismatch,
+ });
+ expect(rejected.status).toBe(1);
+ expect(rejected.stderr).toContain(
+ "npm tarball contains an invalid tar entry.",
+ );
+ expect(readFileSync(logPath, "utf8").includes('"-xzf"')).toBe(
+ mismatch === "contents",
+ );
+ expect(readdirSync(extractionDirectory)).toEqual([]);
+ }
+
environment["TAR_PROXY_FAIL_EXTRACT"] = "true";
const failed = await checkPackage(
root,