Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(package): inspect unpacked files for internal references (#618)

Michael D'Angelo5efc3d941613
diff --git a/sdk/typescript/scripts/check-package.mjs b/sdk/typescript/scripts/check-package.mjs
index 6d2d01090..d0a28a973 100644
--- a/sdk/typescript/scripts/check-package.mjs
+++ b/sdk/typescript/scripts/check-package.mjs
@@ -1,15 +1,31 @@
import { spawnSync } from "node:child_process";
-import { createHash } from "node:crypto";
-import { readFileSync } from "node:fs";
+import {
+ chmodSync,
+ lstatSync,
+ mkdtempSync,
+ readFileSync,
+ readdirSync,
+ rmSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { gunzipSync } from "node:zlib";
+import { packageDistFiles } from "./package-dist-files.mjs";
import {
assertPublicPackageContents,
MAX_EXPANDED_ASSET_BYTES,
} from "./package-public-content.mjs";
import { assertExpectedGitHead } from "./package-provenance.mjs";
import { packageSmokeTimeouts } from "./package-smoke-timeouts.mjs";
-import { regularTarListingLines } from "./package-tar-listing.mjs";
+import {
+ assertStoredSparseContents,
+ readTarArchive,
+} from "./package-tar-entries.mjs";
+import {
+ assertTarListingSizes,
+ regularTarListingLines,
+} from "./package-tar-listing.mjs";
import { pluginContractFiles } from "./plugin-contract.mjs";
const PACKAGE_SMOKE_PROCESS_TIMEOUT_MS =
@@ -30,12 +46,33 @@ if (archive === undefined || args.length > 2) {
);
}
-const archiveBytes = gunzipSync(readFileSync(archive), {
+const archivePath = resolve(archive);
+const compressedArchive = readFileSync(archivePath);
+const archiveBytes = gunzipSync(compressedArchive, {
maxOutputLength: MAX_EXPANDED_ASSET_BYTES,
});
-const PUBLIC_LOGO_SHA256 =
- "9b9c2b09b2fa064611fb62307d321d5c2ea70cf0789f7ce34cdb0fc0d9190b3a";
-const tarOptions = { maxBuffer: archiveBytes.byteLength + 1024 };
+const storedArchive = readTarArchive(archiveBytes);
+const rawEntries = storedArchive.entries;
+validatePackagePaths(rawEntries.map(({ path }) => path));
+assertPublicPackageContents(
+ storedArchive.files,
+ Buffer.concat(storedArchive.metadata.filter(Buffer.isBuffer)),
+);
+const processEnvironment = { ...process.env };
+delete processEnvironment.TAR_OPTIONS;
+const characterLocale =
+ processEnvironment.LC_ALL || processEnvironment.LC_CTYPE;
+delete processEnvironment.LC_ALL;
+const tarOptions = {
+ env: {
+ ...processEnvironment,
+ LC_CTYPE: characterLocale,
+ LC_MESSAGES: "C",
+ LC_NUMERIC: "C",
+ },
+ input: compressedArchive,
+ maxBuffer: archiveBytes.byteLength + 1024,
+};
function tar(args, encoding = "buffer") {
const result = spawnSync("tar", ["--ignore-zeros", ...args], {
...tarOptions,
@@ -51,216 +88,91 @@ function tar(args, encoding = "buffer") {
return result.stdout;
}
-let offset = 0;
-const archiveFiles = new Map();
-const archiveMetadata = [];
-for (; offset + 512 <= archiveBytes.byteLength;) {
- const header = archiveBytes.subarray(offset, offset + 512);
- if (header.every((byte) => byte === 0)) {
- archiveMetadata.push(header);
- offset += 512;
- continue;
+function invalidTarEntry() {
+ throw new Error("npm tarball contains an invalid tar entry.");
+}
+
+function validatePackagePaths(entries) {
+ const files = new Set(entries);
+ if (files.size !== entries.length)
+ throw new Error("npm tarball contains duplicate paths.");
+ const required = [
+ "package/package.json",
+ "package/README.md",
+ "package/docs/cli.md",
+ "package/docs/findings-service.md",
+ "package/docs/dedupe-records.md",
+ "package/LICENSE",
+ "package/bin/codex-security.mjs",
+ "package/dist/index.js",
+ "package/dist/index.d.ts",
+ "package/dist/cli.js",
+ "package/schemas/project-config.schema.json",
+ "package/_bundled_plugin/.codex-plugin/plugin.json",
+ ];
+
+ for (const file of required) {
+ if (!files.has(file)) throw new Error(`npm tarball is missing ${file}.`);
}
- const name = header.subarray(0, 100).toString("utf8").split("\0", 1)[0];
- const prefix = header.subarray(345, 500).toString("utf8").split("\0", 1)[0];
- const path = prefix === "" ? name : `${prefix}/${name}`;
- const sizeField = header
- .subarray(124, 136)
- .toString("ascii")
- .split("\0", 1)[0]
- .trim();
- if (!/^[0-7]*$/u.test(sizeField)) {
- throw new Error("npm tarball contains an invalid tar entry.");
+
+ const contract = JSON.parse(readFileSync(contractPath, "utf8"));
+ const pluginPaths = pluginContractFiles(contract);
+ if (new Set(pluginPaths).size !== pluginPaths.length) {
+ throw new Error("Plugin projection contract contains duplicate paths.");
}
- if (path.endsWith("/") && header[156] !== 0x35) {
- throw new Error("npm tarball contains an invalid tar entry.");
+
+ const pluginEntries = new Set();
+ for (const file of pluginPaths) {
+ const pluginArchivePath = `package/_bundled_plugin/${file}`;
+ pluginEntries.add(pluginArchivePath);
+ if (!files.has(pluginArchivePath)) {
+ throw new Error(`npm tarball is missing ${pluginArchivePath}.`);
+ }
}
- const size = Number.parseInt(sizeField || "0", 8);
- const contentsStart = offset + 512;
- const nextOffset = contentsStart + Math.ceil(size / 512) * 512;
- if (nextOffset > archiveBytes.byteLength) {
- throw new Error("npm tarball contains an invalid tar entry.");
+
+ const distFiles = new Set(packageDistFiles);
+ for (const file of distFiles) {
+ if (!files.has(file)) throw new Error(`npm tarball is missing ${file}.`);
}
- if (header[156] === 0 || header[156] === 0x30) {
- archiveFiles.set(
- path,
- archiveBytes.subarray(contentsStart, contentsStart + size),
- );
- archiveMetadata.push(
- header,
- archiveBytes.subarray(contentsStart + size, nextOffset),
- );
- } else {
- archiveMetadata.push(archiveBytes.subarray(offset, nextOffset));
+ const allowedFiles = new Set([...required, ...distFiles, ...pluginEntries]);
+ for (const file of [...allowedFiles]) {
+ const parts = file.split("/");
+ for (let index = 1; index < parts.length; index++) {
+ allowedFiles.add(`${parts.slice(0, index).join("/")}/`);
+ }
}
- offset = nextOffset;
-}
-if (archiveBytes.subarray(offset).some((byte) => byte !== 0)) {
- throw new Error("npm tarball contains trailing tar data.");
-}
-
-function archiveFile(path) {
- const contents = archiveFiles.get(path);
- if (contents === undefined) {
- throw new Error("npm tarball contains an invalid tar entry: " + path + ".");
+ const unsafePath = /(?:^|\/)\.{1,2}(?:\/|$)/u;
+ for (const file of files) {
+ if (
+ !allowedFiles.has(file) ||
+ unsafePath.test(file) ||
+ file.includes("\\")
+ ) {
+ const displayPath = file.replace(
+ /[\u0000-\u001f\u007f-\u009f]/gu,
+ (character) =>
+ `\\x${character.charCodeAt(0).toString(16).padStart(2, "0")}`,
+ );
+ throw new Error(
+ `npm tarball contains an unexpected file: ${displayPath}.`,
+ );
+ }
}
- return contents;
}
-const entries = tar(["-tzf", archive], "utf8").split(/\r?\n/u).filter(Boolean);
+const entries = tar(["-tzf", "-"], "utf8").split(/\r?\n/u).filter(Boolean);
const files = new Set(entries);
if (files.size !== entries.length) {
throw new Error("npm tarball contains duplicate paths.");
}
-const required = [
- "package/package.json",
- "package/README.md",
- "package/docs/cli.md",
- "package/docs/findings-service.md",
- "package/docs/dedupe-records.md",
- "package/LICENSE",
- "package/bin/codex-security.mjs",
- "package/dist/index.js",
- "package/dist/index.d.ts",
- "package/dist/cli.js",
- "package/schemas/project-config.schema.json",
- "package/_bundled_plugin/.codex-plugin/plugin.json",
-];
-
-for (const file of required) {
- if (!files.has(file)) throw new Error(`npm tarball is missing ${file}.`);
-}
-
-const contract = JSON.parse(readFileSync(contractPath, "utf8"));
-const pluginPaths = pluginContractFiles(contract);
-if (new Set(pluginPaths).size !== pluginPaths.length) {
- throw new Error("Plugin projection contract contains duplicate paths.");
-}
-
-const allowedFiles = new Set([
- ...required,
- ...pluginPaths.map((file) => `package/_bundled_plugin/${file}`),
- ...[
- "api",
- "artifact-export",
- "auth",
- "bulk-scan-discovery",
- "cli",
- "cli-help",
- "cli-scan-logs-json",
- "cli-signals",
- "classify-severity",
- "classify-scan-severity",
- "severity-store",
- "cloud-publish",
- "codex-prompt",
- "codex-sdk-environment",
- "component-plan",
- "component-scan",
- "config",
- "config-path",
- "contract",
- "cost",
- "cost-model",
- "custom-validation",
- "custom-validation-prompt",
- "custom-publish",
- "deep-progress",
- "deep-scan",
- "deep-config",
- "deep-scan-defaults",
- "project-config",
- "project-config-schema",
- "prompt-files",
- "provider-profile",
- "scan-modes",
- "scan-settings",
- "errors",
- "feedback",
- "finding-catalogue",
- "findings-import",
- "github",
- "index",
- "import-scan",
- "knowledge-base",
- "linear",
- "models",
- "multiscan",
- "mock-scan",
- "owner-evidence",
- "patch-tui",
- "publication",
- "publication-events",
- "publication-store",
- "publish",
- "result",
- "record",
- "request-metadata",
- "runtime",
- "scan-activity",
- "scan-comparison",
- "scan-dashboard",
- "scan-history-renderer",
- "scan-inputs",
- "scan-logs",
- "security-policy",
- "security-policy-cli",
- "suggest-owners",
- "scan-sessions",
- "server/api",
- "deduplication/codex-review",
- "deduplication/checkpointed-review",
- "deduplication/refusal",
- "deduplication/retry",
- "deduplication/deduplication",
- "finding-retrieval",
- "finding-workflow",
- "findings-client",
- "findings-errors",
- "finding-dedupe-groups",
- "deduplication/deduplication-prompts",
- "deduplication/deduplication-reviewer",
- "deduplication/diagnostics",
- "deduplication/scan",
- "deduplication/local",
- "deduplication/finding-schema",
- "deduplication/records",
- "deduplication/records-protocol",
- "deduplication/review",
- "saved-scan",
- "saved-scan-bootstrap",
- "server/embeddings",
- "server/errors",
- "server/sqlite-store",
- "server/storage",
- "targets",
- "thread-source",
- "trusted-executable",
- "value",
- "version",
- "windows-path",
- "worker-progress",
- ].flatMap((module) =>
- ["js", "js.map", "d.ts", "d.ts.map"].map(
- (extension) => `package/dist/${module}.${extension}`,
- ),
- ),
-]);
-for (const file of [...allowedFiles]) {
- if (!files.has(file)) throw new Error(`npm tarball is missing ${file}.`);
- const parts = file.split("/");
- for (let index = 1; index < parts.length; index++) {
- allowedFiles.add(`${parts.slice(0, index).join("/")}/`);
- }
-}
-const unsafePath = /(?:^|\/)\.{1,2}(?:\/|$)/u;
-for (const file of files) {
- if (!allowedFiles.has(file) || unsafePath.test(file) || file.includes("\\")) {
- throw new Error(`npm tarball contains an unexpected file: ${file}.`);
- }
+if (
+ rawEntries.length !== entries.length ||
+ rawEntries.some(({ path }, index) => path !== entries[index])
+) {
+ invalidTarEntry();
}
-const listing = tar(["-tvzf", archive], "utf8");
+const listing = tar(["--numeric-owner", "-tvzf", "-"], "utf8");
const listingLines = regularTarListingLines(listing);
if (
listingLines.length !== entries.length ||
@@ -268,8 +180,12 @@ if (
(line, index) => line.startsWith("d") !== entries[index].endsWith("/"),
)
) {
- throw new Error("npm tarball contains an invalid tar entry.");
+ invalidTarEntry();
}
+const listingSizes = assertTarListingSizes(
+ listingLines,
+ MAX_EXPANDED_ASSET_BYTES,
+);
for (const [path, name] of [
["package/bin/codex-security.mjs", "CLI"],
["package/_bundled_plugin/scripts/launch_codex_security_mcp", "MCP"],
@@ -280,9 +196,104 @@ for (const [path, name] of [
throw new Error(`npm package ${name} launcher is not executable.`);
}
}
-const packageJson = JSON.parse(
- archiveFile("package/package.json").toString("utf8"),
-);
+
+function privateExtractionDirectories(directory) {
+ chmodSync(directory, 0o700);
+ for (const entry of readdirSync(directory, { withFileTypes: true })) {
+ if (entry.isDirectory()) {
+ privateExtractionDirectories(join(directory, entry.name));
+ }
+ }
+}
+
+function extractedArchiveFiles() {
+ const logicalSizes = new Map();
+ const expectedPaths = new Map();
+ for (const [index, { path }] of rawEntries.entries()) {
+ const directory = path.endsWith("/");
+ const extractedPath = directory ? path.slice(0, -1) : path;
+ const type = directory ? "directory" : "file";
+ const previousType = expectedPaths.get(extractedPath);
+ if (previousType !== undefined && previousType !== type) invalidTarEntry();
+ if (!directory) logicalSizes.set(path, listingSizes[index]);
+ expectedPaths.set(extractedPath, type);
+ const parts = extractedPath.split("/");
+ for (let index = 1; index < parts.length; index++) {
+ const directory = parts.slice(0, index).join("/");
+ if (logicalSizes.has(directory)) invalidTarEntry();
+ expectedPaths.set(directory, "directory");
+ }
+ }
+
+ const extractionRoot = mkdtempSync(
+ join(tmpdir(), "codex-security-package-check-"),
+ );
+ try {
+ chmodSync(extractionRoot, 0o700);
+ try {
+ tar([
+ "-m",
+ "--keep-old-files",
+ "--no-same-owner",
+ "--no-same-permissions",
+ "--no-acls",
+ "--no-xattrs",
+ "-xzf",
+ "-",
+ "-C",
+ extractionRoot,
+ ...logicalSizes.keys(),
+ ]);
+ } finally {
+ privateExtractionDirectories(extractionRoot);
+ }
+
+ const archiveFiles = new Map();
+ let expandedBytes = 0;
+ function visit(directory, relative = "") {
+ for (const name of readdirSync(directory)) {
+ const path = relative === "" ? name : `${relative}/${name}`;
+ const expectedType = expectedPaths.get(path);
+ if (expectedType === undefined) invalidTarEntry();
+ const extractedPath = join(extractionRoot, path);
+ const stats = lstatSync(extractedPath);
+ expectedPaths.delete(path);
+
+ if (expectedType === "directory") {
+ if (!stats.isDirectory()) invalidTarEntry();
+ visit(extractedPath, path);
+ continue;
+ }
+
+ if (
+ !stats.isFile() ||
+ stats.nlink !== 1 ||
+ stats.size !== logicalSizes.get(path) ||
+ stats.size > MAX_EXPANDED_ASSET_BYTES ||
+ expandedBytes > MAX_EXPANDED_ASSET_BYTES - stats.size
+ ) {
+ invalidTarEntry();
+ }
+ expandedBytes += stats.size;
+ chmodSync(extractedPath, 0o600);
+ archiveFiles.set(path, readFileSync(extractedPath));
+ }
+ }
+ visit(extractionRoot);
+
+ if (expectedPaths.size !== 0 || archiveFiles.size !== logicalSizes.size) {
+ invalidTarEntry();
+ }
+ return archiveFiles;
+ } finally {
+ rmSync(extractionRoot, { force: true, recursive: true });
+ }
+}
+
+const archiveFiles = extractedArchiveFiles();
+const npmManifest = storedArchive.npmFiles.get("package/package.json");
+if (npmManifest === undefined) invalidTarEntry();
+const packageJson = JSON.parse(npmManifest.toString("utf8"));
if (
packageJson.name !== "@openai/codex-security" ||
packageJson.license !== "Apache-2.0"
@@ -294,21 +305,24 @@ assertExpectedGitHead(
process.env.CODEX_SECURITY_EXPECTED_GIT_HEAD,
);
-for (const file of files) {
- if (/\.png$/iu.test(file)) {
- const digest = createHash("sha256").update(archiveFile(file)).digest("hex");
- if (digest !== PUBLIC_LOGO_SHA256) {
- throw new Error(`npm tarball contains an unexpected PNG asset: ${file}.`);
- }
- }
+assertPublicPackageContents(archiveFiles);
+assertStoredSparseContents(storedArchive, archiveFiles);
+for (const path of archiveFiles.keys()) {
+ if (
+ /\.(?:png|br(?:\.part-[0-9]+)?)$/iu.test(path) &&
+ !storedArchive.npmFiles.has(path)
+ )
+ invalidTarEntry();
}
-
-assertPublicPackageContents(archiveFiles, Buffer.concat(archiveMetadata));
+assertPublicPackageContents(storedArchive.npmFiles);
if (args.length === 1) {
const smoke = spawnSync(
process.execPath,
- [fileURLToPath(new URL("./smoke-package.mjs", import.meta.url)), archive],
+ [
+ fileURLToPath(new URL("./smoke-package.mjs", import.meta.url)),
+ archivePath,
+ ],
{
stdio: "inherit",
timeout: PACKAGE_SMOKE_PROCESS_TIMEOUT_MS,
diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs
new file mode 100644
index 000000000..ec33e1c6d
--- /dev/null
+++ b/sdk/typescript/scripts/package-dist-files.mjs
@@ -0,0 +1,107 @@
+const distModules = [
+ "api",
+ "artifact-export",
+ "auth",
+ "bulk-scan-discovery",
+ "cli",
+ "cli-help",
+ "cli-scan-logs-json",
+ "cli-signals",
+ "classify-severity",
+ "classify-scan-severity",
+ "severity-store",
+ "cloud-publish",
+ "codex-prompt",
+ "codex-sdk-environment",
+ "component-plan",
+ "component-scan",
+ "config",
+ "config-path",
+ "contract",
+ "cost",
+ "cost-model",
+ "custom-validation",
+ "custom-validation-prompt",
+ "custom-publish",
+ "deep-progress",
+ "deep-scan",
+ "deep-config",
+ "deep-scan-defaults",
+ "project-config",
+ "project-config-schema",
+ "prompt-files",
+ "provider-profile",
+ "scan-modes",
+ "scan-settings",
+ "errors",
+ "feedback",
+ "finding-catalogue",
+ "findings-import",
+ "github",
+ "index",
+ "import-scan",
+ "knowledge-base",
+ "linear",
+ "models",
+ "multiscan",
+ "mock-scan",
+ "owner-evidence",
+ "patch-tui",
+ "publication",
+ "publication-events",
+ "publication-store",
+ "publish",
+ "result",
+ "record",
+ "request-metadata",
+ "runtime",
+ "scan-activity",
+ "scan-comparison",
+ "scan-dashboard",
+ "scan-history-renderer",
+ "scan-inputs",
+ "scan-logs",
+ "security-policy",
+ "security-policy-cli",
+ "suggest-owners",
+ "scan-sessions",
+ "server/api",
+ "deduplication/codex-review",
+ "deduplication/checkpointed-review",
+ "deduplication/refusal",
+ "deduplication/retry",
+ "deduplication/deduplication",
+ "finding-retrieval",
+ "finding-workflow",
+ "findings-client",
+ "findings-errors",
+ "finding-dedupe-groups",
+ "deduplication/deduplication-prompts",
+ "deduplication/deduplication-reviewer",
+ "deduplication/diagnostics",
+ "deduplication/scan",
+ "deduplication/local",
+ "deduplication/finding-schema",
+ "deduplication/records",
+ "deduplication/records-protocol",
+ "deduplication/review",
+ "saved-scan",
+ "saved-scan-bootstrap",
+ "server/embeddings",
+ "server/errors",
+ "server/sqlite-store",
+ "server/storage",
+ "targets",
+ "thread-source",
+ "trusted-executable",
+ "value",
+ "version",
+ "windows-path",
+ "worker-progress",
+];
+
+export const packageDistFiles = distModules.flatMap((module) =>
+ ["js", "js.map", "d.ts", "d.ts.map"].map(
+ (extension) => `package/dist/${module}.${extension}`,
+ ),
+);
diff --git a/sdk/typescript/scripts/package-public-content.mjs b/sdk/typescript/scripts/package-public-content.mjs
index 2d8458b1e..2005dce34 100644
--- a/sdk/typescript/scripts/package-public-content.mjs
+++ b/sdk/typescript/scripts/package-public-content.mjs
@@ -1,9 +1,13 @@
+import { createHash } from "node:crypto";
import { brotliDecompressSync } from "node:zlib";
export const MAX_EXPANDED_ASSET_BYTES = 32 * 1024 * 1024;
+const PUBLIC_LOGO_SHA256 =
+ "9b9c2b09b2fa064611fb62307d321d5c2ea70cf0789f7ce34cdb0fc0d9190b3a";
+
const internalMarker =
- /(?:internal\.api\.openai\.org|gateway\.[a-z0-9.-]*internal|\.openai\.org|openai\.firewall\.socket\.dev|socket\x2dfirewall\x2dregistry|openai\.(?:enterprise\.)?slack\.com|app\.slack\.com\/client|(?:app\.notion\.com\/p|notion\.so)\/openai|linear\.app\/openai|(?:github\.com[:/]|api\.github\.com\/repos\/|raw\.githubusercontent\.com\/)openai\/openai(?:\.git)?(?:[^a-z0-9_-]|$)|LicenseRef\x2dProprietary|\/Users\/|\/home\/dev-user|flow\.apps\.openai\.org|(?:^|[^a-z0-9_-])go\/[a-z0-9_-]+)/iu;
+ /(?:internal\.api\.openai\.org|gateway\.[a-z0-9.-]*internal|\.openai\.org|openai\.firewall\.socket\.dev|socket\x2dfirewall\x2dregistry|openai\.(?:enterprise\.)?slack\.com|app\.slack\.com\/client|(?:app\.notion\.com\/p|notion\.so)\/openai|linear\.app\/openai|(?:github\.com[:/]|api\.github\.com\/repos\/|raw\.githubusercontent\.com\/)openai\/openai(?:\.git)?(?![a-z0-9_-])|LicenseRef\x2dProprietary|\/Users\/|\/home\/dev-user|flow\.apps\.openai\.org|(?<![a-z0-9_-])go\/[a-z0-9_-]+)/iu;
/**
* @param {Map<string, Buffer>} archiveFiles
@@ -25,6 +29,13 @@ export function assertPublicPackageContents(
compressedParts.set(name, parts);
} else if (/\.br$/iu.test(path)) {
assertPublicBrotli(bytes, path);
+ } else if (/\.png$/iu.test(path)) {
+ if (
+ createHash("sha256").update(bytes).digest("hex") !== PUBLIC_LOGO_SHA256
+ )
+ throw new Error(
+ `npm tarball contains an unexpected PNG asset: ${path}.`,
+ );
} else {
assertPublicText(bytes.toString("utf8"));
}
@@ -49,8 +60,17 @@ function assertPublicBrotli(bytes, path) {
assertPublicText(result.buffer.toString("utf8"));
}
-function assertPublicText(contents) {
- if (internalMarker.test(contents)) {
- throw new Error("npm tarball contains an internal reference.");
+export function assertPublicText(contents, binaryRanges = []) {
+ const pattern = new RegExp(internalMarker.source, "giu");
+ let match;
+ while ((match = pattern.exec(contents)) !== null) {
+ if (
+ !binaryRanges.some(
+ ({ start, end }) =>
+ match.index >= start && match.index + match[0].length <= end,
+ )
+ )
+ throw new Error("npm tarball contains an internal reference.");
+ pattern.lastIndex = match.index + 1;
}
}
diff --git a/sdk/typescript/scripts/package-tar-entries.mjs b/sdk/typescript/scripts/package-tar-entries.mjs
new file mode 100644
index 000000000..5b6e81ade
--- /dev/null
+++ b/sdk/typescript/scripts/package-tar-entries.mjs
@@ -0,0 +1,299 @@
+import { assertPublicText } from "./package-public-content.mjs";
+
+const blockSize = 512;
+function invalidTarEntry() {
+ throw new Error("npm tarball contains an invalid tar entry.");
+}
+
+function headerText(header, start, end) {
+ return header.subarray(start, end).toString("utf8").split("\0", 1)[0];
+}
+
+function octalNumber(header, start, end) {
+ const field = headerText(header, start, end).trim();
+ if (!/^[0-7]*$/u.test(field)) invalidTarEntry();
+ return Number.parseInt(field || "0", 8);
+}
+
+function paxAttributes(contents) {
+ const attributes = new Map();
+ let offset = 0;
+ while (offset < contents.byteLength) {
+ const separator = contents.indexOf(0x20, offset);
+ const lengthField = contents.subarray(offset, separator).toString("ascii");
+ if (separator === -1 || !/^[0-9]+$/u.test(lengthField)) invalidTarEntry();
+ const end = offset + Number(lengthField);
+ const equals = contents.indexOf(0x3d, separator + 1);
+ if (
+ end > contents.byteLength ||
+ equals <= separator + 1 ||
+ equals >= end - 1 ||
+ contents[end - 1] !== 0x0a
+ )
+ invalidTarEntry();
+ attributes.set(
+ contents.subarray(separator + 1, equals).toString("utf8"),
+ contents.subarray(equals + 1, end - 1).toString("utf8"),
+ );
+ offset = end;
+ }
+ return attributes;
+}
+
+function sparseMap(contents, paxMap) {
+ if (paxMap) {
+ const values = paxMap.split(",").map(Number);
+ const extents = [];
+ for (let index = 0; index < values.length; index += 2)
+ extents.push({ offset: values[index], size: values[index + 1] });
+ return { contents, extents, dataOffset: 0 };
+ }
+ let offset = 0;
+ function number() {
+ while (offset < contents.length) {
+ const end = contents.indexOf(0x0a, offset);
+ if (end === -1) invalidTarEntry();
+ const line = contents.subarray(offset, end).toString("ascii");
+ offset = end + 1;
+ if (line.startsWith("#")) continue;
+ if (!/^[0-9]*$/u.test(line)) invalidTarEntry();
+ return Number(line);
+ }
+ invalidTarEntry();
+ }
+ const count = number();
+ const extents = [];
+ for (let index = 0; index < count; index++) {
+ extents.push({ offset: number(), size: number() });
+ }
+ const dataOffset = Math.ceil(offset / blockSize) * blockSize;
+ if (dataOffset > contents.length) invalidTarEntry();
+ return { contents, extents, dataOffset };
+}
+
+export function assertStoredSparseContents(archive, extractedFiles) {
+ const sparseMetadata = new Map();
+ for (const [path, { contents, extents, dataOffset }] of archive.sparseFiles) {
+ const extracted = extractedFiles.get(path);
+ let metadata;
+ // GNU tar pads stored extents; libarchive also accepts packed extents.
+ for (const padded of [false, true]) {
+ let offset = dataOffset;
+ const parts = [contents.subarray(0, dataOffset)];
+ const matches = extents.every((extent, index) => {
+ const end = offset + extent.size;
+ if (
+ !contents
+ .subarray(offset, end)
+ .equals(
+ extracted.subarray(extent.offset, extent.offset + extent.size),
+ )
+ )
+ return false;
+ const next =
+ padded && index < extents.length - 1
+ ? Math.ceil(end / blockSize) * blockSize
+ : end;
+ parts.push(
+ { binary: contents.subarray(offset, end) },
+ contents.subarray(end, next),
+ );
+ offset = next;
+ return true;
+ });
+ if (matches) {
+ parts.push(contents.subarray(offset));
+ metadata = parts;
+ break;
+ }
+ }
+ if (metadata === undefined) invalidTarEntry();
+ sparseMetadata.set(path, metadata);
+ }
+ const parts = archive.metadata.flatMap((part) =>
+ typeof part === "string" ? sparseMetadata.get(part) : part,
+ );
+ const binaryRanges = [];
+ let text = "";
+ for (const part of parts) {
+ const binary = !Buffer.isBuffer(part);
+ const decoded = (binary ? part.binary : part).toString("utf8");
+ if (binary) {
+ const previous = binaryRanges.at(-1);
+ if (previous?.end === text.length) previous.end += decoded.length;
+ else
+ binaryRanges.push({
+ start: text.length,
+ end: text.length + decoded.length,
+ });
+ }
+ text += decoded;
+ }
+ assertPublicText(text, binaryRanges);
+}
+
+export function readTarArchive(archiveBytes) {
+ const entries = [];
+ const archiveFiles = new Map();
+ const archiveMetadata = [];
+ const sparseFiles = new Map();
+ const npmFiles = new Map();
+ let offset = 0;
+ const globalAttributes = new Map();
+ const nextAttributes = new Map();
+ let nextName;
+ let nextNpmPath;
+
+ while (offset + blockSize <= archiveBytes.byteLength) {
+ const header = archiveBytes.subarray(offset, offset + blockSize);
+ if (header.every((byte) => byte === 0)) {
+ archiveMetadata.push(header);
+ offset += blockSize;
+ continue;
+ }
+
+ const signature = header.subarray(257, 265).toString("latin1");
+ const directory = header[156] === 0x35;
+ const oldSparse = header[156] === 0x53;
+ const longName = header[156] === 0x4c;
+ const extended = header[156] === 0x78 || header[156] === 0x67 || longName;
+ if (
+ (header[156] !== 0 &&
+ header[156] !== 0x30 &&
+ !directory &&
+ !extended &&
+ !oldSparse) ||
+ (signature !== "ustar\0" + "00" &&
+ signature !== "ustar \0" &&
+ signature !== "\0".repeat(8))
+ ) {
+ invalidTarEntry();
+ }
+
+ const name = headerText(header, 0, 100);
+ // GNU headers use this area for timestamps and sparse-file metadata.
+ const prefix =
+ signature === "ustar\0" + "00" ? headerText(header, 345, 500) : "";
+ const attribute = (key) =>
+ nextAttributes.has(key)
+ ? nextAttributes.get(key) || undefined
+ : globalAttributes.get(key);
+ const headerPath = prefix === "" ? name : `${prefix}/${name}`;
+ const path =
+ attribute("GNU.sparse.name") ??
+ attribute("path") ??
+ nextName ??
+ headerPath;
+ if (!extended && (path === "" || path.endsWith("/") !== directory))
+ invalidTarEntry();
+ assertPublicText(path);
+
+ const headerSize = octalNumber(header, 124, 136);
+ const paxSize = extended ? undefined : attribute("size");
+ if (paxSize !== undefined && !/^[0-9]+$/u.test(paxSize)) invalidTarEntry();
+ const size = paxSize === undefined ? headerSize : Number(paxSize);
+ let contentsStart = offset + blockSize;
+ const oldSparseExtents = [];
+ if (oldSparse) {
+ let map = header;
+ let start = 386;
+ let count = 4;
+ for (;;) {
+ let index = 0;
+ for (; index < count; index++) {
+ const field = start + index * 24;
+ if (map[field + 12] === 0) break;
+ const offset = octalNumber(map, field, field + 12);
+ const size = octalNumber(map, field + 12, field + 24);
+ if (offset !== 0 || size !== 0)
+ oldSparseExtents.push({ offset, size });
+ }
+ if (index < count || map[start + count * 24] === 0) break;
+ map = archiveBytes.subarray(contentsStart, contentsStart + blockSize);
+ if (map.length !== blockSize) invalidTarEntry();
+ contentsStart += blockSize;
+ start = 0;
+ count = 21;
+ }
+ }
+ const contentsEnd = contentsStart + size;
+ const nextOffset = contentsStart + Math.ceil(size / blockSize) * blockSize;
+ if (nextOffset > archiveBytes.byteLength || (directory && size !== 0)) {
+ invalidTarEntry();
+ }
+
+ if (extended) {
+ const contents = archiveBytes.subarray(contentsStart, contentsEnd);
+ archiveMetadata.push(archiveBytes.subarray(offset, nextOffset));
+ if (longName) {
+ nextName = headerText(contents, 0, contents.length);
+ nextNpmPath = nextName;
+ } else {
+ const destination =
+ header[156] === 0x67 ? globalAttributes : nextAttributes;
+ for (const [key, value] of paxAttributes(contents)) {
+ // npm merges local PAX paths and GNU long names in record order.
+ if (destination === nextAttributes && key === "path")
+ nextNpmPath = value || undefined;
+ if (destination === globalAttributes && value === "")
+ destination.delete(key);
+ else destination.set(key, value);
+ }
+ }
+ } else {
+ if (directory)
+ archiveMetadata.push(archiveBytes.subarray(offset, nextOffset));
+ else {
+ const contents = archiveBytes.subarray(contentsStart, contentsEnd);
+ // npm ignores old-GNU sparse entries and GNU sparse name overrides.
+ const npmPath = oldSparse ? undefined : (nextNpmPath ?? headerPath);
+ if (npmPath !== undefined) npmFiles.set(npmPath, contents);
+ archiveMetadata.push(archiveBytes.subarray(offset, contentsStart));
+ const paxSparseMap = nextAttributes.get("GNU.sparse.map");
+ if (
+ (oldSparse ||
+ paxSparseMap ||
+ Number.parseInt(nextAttributes.get("GNU.sparse.major"), 10) ===
+ 1) &&
+ /\.(?:png|br(?:\.part-[0-9]+)?)$/iu.test(path)
+ ) {
+ sparseFiles.set(
+ path,
+ oldSparse
+ ? { contents, extents: oldSparseExtents, dataOffset: 0 }
+ : sparseMap(contents, paxSparseMap),
+ );
+ // Retain sparse framing in order with the surrounding headers and padding.
+ archiveMetadata.push(path);
+ } else {
+ archiveFiles.set(path, contents);
+ if (oldSparse) archiveMetadata.push(contents);
+ }
+ archiveMetadata.push(archiveBytes.subarray(contentsEnd, nextOffset));
+ }
+ entries.push({ path, size });
+ nextAttributes.clear();
+ nextName = undefined;
+ nextNpmPath = undefined;
+ }
+ offset = nextOffset;
+ }
+
+ if (archiveBytes.subarray(offset).some((byte) => byte !== 0))
+ invalidTarEntry();
+ const deferredStreams = new Set(
+ [...sparseFiles.keys()].map((path) => path.replace(/\.part-[0-9]+$/iu, "")),
+ );
+ for (const path of archiveFiles.keys()) {
+ if (deferredStreams.has(path.replace(/\.part-[0-9]+$/iu, ""))) {
+ archiveFiles.delete(path);
+ }
+ }
+ return {
+ entries,
+ files: archiveFiles,
+ metadata: archiveMetadata,
+ sparseFiles,
+ npmFiles,
+ };
+}
diff --git a/sdk/typescript/scripts/package-tar-listing.mjs b/sdk/typescript/scripts/package-tar-listing.mjs
index d30d31782..6a0cac2b9 100644
--- a/sdk/typescript/scripts/package-tar-listing.mjs
+++ b/sdk/typescript/scripts/package-tar-listing.mjs
@@ -8,3 +8,24 @@ export function regularTarListingLines(listing) {
}
return lines;
}
+
+export function assertTarListingSizes(lines, maximum) {
+ let expandedBytes = 0;
+ const sizes = [];
+ for (const line of lines) {
+ if (line.startsWith("d")) {
+ sizes.push(0);
+ continue;
+ }
+ // GNU combines numeric owners; BSD lists the link count, UID and GID.
+ const match = /^\S+\s+(?:\d+\/\d+\s+|\d+\s+\d+\s+\d+\s+)(\d+)\s/u.exec(
+ line,
+ );
+ const size = match === null ? NaN : Number(match[1]);
+ if (!Number.isSafeInteger(size) || size > maximum - expandedBytes)
+ throw new Error("npm tarball contains an invalid tar entry.");
+ expandedBytes += size;
+ sizes.push(size);
+ }
+ return sizes;
+}
diff --git a/sdk/typescript/tests-ts/package-public-content.test.ts b/sdk/typescript/tests-ts/package-public-content.test.ts
index 8100d34bc..44733c02e 100644
--- a/sdk/typescript/tests-ts/package-public-content.test.ts
+++ b/sdk/typescript/tests-ts/package-public-content.test.ts
@@ -1,13 +1,10 @@
+import { readFileSync } from "node:fs";
import { brotliCompressSync, brotliDecompressSync } from "node:zlib";
import { describe, expect, test } from "bun:test";
import { assertPublicPackageContents } from "../scripts/package-public-content.mjs";
-// Synthetic alphanumeric text whose compressed bytes happen to contain " Go/w".
-const cleanCompressedPayload = Buffer.from(
- "G/8DAGRwm7EiGn34sGFNRM5DuwB8/2zECGTx7hk1/DY6puqEJWJumAz3HfjBg7bx5FxwfFCafaN2of0Oi2uTSSE+wAYOJS4Ii53urIa5BOIkTbR5t2YJ4/UM6dWzXcEvk92TZNz/Y7hNVxjrsXKemVnN4i1pUOdQiCYQNLU7Dbh77pXyrGIWSl+X+L7DUfFR+4Rz3GF85xCbj1NuwlDfIWj42+ueic5NDX4bPXmp46fE8bTdpAUmMvt48x99pm2wlp+oKNvvcb64GT4i7F9zCJMIdfmc06UCn7Go4jQ1WI2P+e05ZbpsWoTkPXnAFULsyUtefeZ1d5nbvE2CDXV3eF7Hxc2KgtQQv8j/CIEd+9vG3REfGcJVIVMqjo1SKzOfADpWQYh3p18YFMyF6R4qOYzPB+zjBy/ZQiCaNm1LpVYdJ7XL0wXPjAVOzJU4zeSUvNFqQ0TeXFGat6ikPEii43FRFSx3aQi2HnWI3rBd1tts0LlTjIWUq8+agHu/mwmJ+jWxAF74Yvij4++fUTHxm8PvqXOK+3QpR82rAPZ+pyRgKX1Z9k9XHlVwNXzJCd5laKwKOWnAOF6vKnrqhZz4aw1VqEJ+1x/4+AM++Pw58PdlRKBjKoLl7DKgEBCGKsd5FF8XPQz9pC3MWj1UbdVa/jLl5M1iIfl9P44zgKV1TaG1LEjuS09t7e0UFSrgXI0BWvBuyNfEras635PJBiZuI8mX6s5yt5NkocDkATyMNP1BnW+wKyTzduIFbd2yV45t/t6ttd5y5FeAAKnT0Jjsmk3zbYdl12fpKKDUvs363EVzTjWEhNaoTC3ntUZKNlYzX74UMDogR28vd8vvJniJvfamegrjMcf3fviFr9caYVQH+bq/TpxK078VtU0Z5iah0E2+ttjzWXHi//ftzs9e+xArq7zohSgVrMpgsL2kjfl/zaltHiNmTgogtQI6gRiI1ICa3kimcK2a/gj0cTkCVwKywUt6xu1N2x2lPi7dk/pnx8/iSkdyphyFA4OQov+d8maitV0XwU9/NKTW048C",
- "base64",
-);
+import { cleanCompressedPayload } from "./package-tar-fixtures.js";
function compressedFiles(bytes: Buffer, split: boolean): Map<string, Buffer> {
if (!split) return new Map([["package/runtime.mjs.br", bytes]]);
@@ -59,6 +56,31 @@ describe("npm package public contents", () => {
},
);
+ test("checks the approved PNG digest before exempting binary contents", () => {
+ const approved = readFileSync(
+ new URL(
+ "../../../plugins/codex-security/assets/logo.png",
+ import.meta.url,
+ ),
+ );
+ expect(() =>
+ assertPublicPackageContents(new Map([["package/logo.png", approved]])),
+ ).not.toThrow();
+ const stored = Buffer.from(approved);
+ expect(stored.subarray(171, 178)).toEqual(Buffer.alloc(7));
+ stored.set(Buffer.from("\0go/x\0\0"), 171);
+ expect(() =>
+ assertPublicPackageContents(new Map([["package/logo.png", stored]])),
+ ).toThrow("npm tarball contains an unexpected PNG asset");
+ });
+
+ test("keeps the expanded Brotli size bound", () => {
+ const bytes = brotliCompressSync(Buffer.alloc(32 * 1024 * 1024 + 1));
+ expect(() =>
+ assertPublicPackageContents(compressedFiles(bytes, false)),
+ ).toThrow();
+ });
+
test("checks tar metadata", () => {
expect(() =>
assertPublicPackageContents(new Map(), Buffer.from("go/synthetic-owner")),
diff --git a/sdk/typescript/tests-ts/package-tar-entries.test.ts b/sdk/typescript/tests-ts/package-tar-entries.test.ts
new file mode 100644
index 000000000..f1f790b70
--- /dev/null
+++ b/sdk/typescript/tests-ts/package-tar-entries.test.ts
@@ -0,0 +1,632 @@
+import { readFileSync } from "node:fs";
+import { describe, expect, test } from "bun:test";
+import {
+ archive,
+ blockSize,
+ cleanCompressedPayload,
+ octal,
+ paxRecords,
+ tarRecord,
+} from "./package-tar-fixtures.js";
+
+type PlainTarEntry = {
+ path: string;
+ size: number;
+};
+
+type PackageTarEntries = {
+ readTarArchive: (archiveBytes: Buffer) => {
+ entries: PlainTarEntry[];
+ files: Map<string, Buffer>;
+ metadata: (Buffer | string)[];
+ };
+ assertStoredSparseContents: (
+ archive: ReturnType<PackageTarEntries["readTarArchive"]>,
+ files: Map<string, Buffer>,
+ ) => void;
+};
+
+const { readTarArchive, assertStoredSparseContents } = (await import(
+ new URL("../scripts/package-tar-entries.mjs", import.meta.url).href
+)) as PackageTarEntries;
+const { assertPublicPackageContents } = (await import(
+ new URL("../scripts/package-public-content.mjs", import.meta.url).href
+)) as {
+ assertPublicPackageContents: (
+ files: Map<string, Buffer>,
+ metadata: Buffer,
+ ) => void;
+};
+
+function plainTarEntries(bytes: Buffer): PlainTarEntry[] {
+ const archive = readTarArchive(bytes);
+ assertPublicPackageContents(
+ archive.files,
+ Buffer.concat(
+ archive.metadata.filter((part): part is Buffer => Buffer.isBuffer(part)),
+ ),
+ );
+ return archive.entries;
+}
+
+const invalidTarEntryError = "npm tarball contains an invalid tar entry.";
+const internalReferenceError = "npm tarball contains an internal reference.";
+
+describe("plain npm tar entries", () => {
+ test.each([" ", " \0", "\0"])(
+ "accepts package size fields ending in %j",
+ (terminator) => {
+ expect(
+ plainTarEntries(
+ archive(
+ tarRecord(Buffer.from("readme"), {
+ name: "package/README.md",
+ sizeField: octal(6, 12, terminator),
+ }),
+ ),
+ ),
+ ).toEqual([{ path: "package/README.md", size: 6 }]);
+ },
+ );
+
+ test.each([0, 0x30])(
+ "accepts regular typeflag %i and prefix paths",
+ (type) => {
+ const prefix = `package/${"nested/".repeat(13)}deep`;
+ const longPath = `${prefix}/README.md`;
+ expect(longPath.length).toBeGreaterThan(100);
+
+ const bytes = archive(
+ tarRecord(Buffer.from("license"), { name: "package/LICENSE", type }),
+ tarRecord(Buffer.from("readme"), { name: "README.md", prefix, type }),
+ );
+
+ expect(plainTarEntries(bytes)).toEqual([
+ { path: "package/LICENSE", size: 7 },
+ { path: longPath, size: 6 },
+ ]);
+ },
+ );
+
+ test("rejects unsupported entry types and malformed extended headers", () => {
+ for (const type of [0x31, 0x32, 0x33, 0x34, 0x36, 0x44, 0x4b, 0x67, 0x78]) {
+ expect(() =>
+ plainTarEntries(
+ archive(
+ tarRecord(Buffer.from("x"), {
+ name: "package/README.md",
+ type,
+ }),
+ ),
+ ),
+ ).toThrow(invalidTarEntryError);
+ }
+ });
+
+ test("uses GNU long names for one member and scans their complete payload", () => {
+ const path = `package/${"nested/".repeat(16)}README.md`;
+ const records = (tail: string) =>
+ archive(
+ tarRecord(Buffer.from(`${path}\0${tail}`), {
+ name: "././@LongLink",
+ type: 0x4c,
+ }),
+ tarRecord(Buffer.from("readme"), { name: "placeholder" }),
+ tarRecord(Buffer.from("license"), { name: "package/LICENSE" }),
+ );
+ expect(plainTarEntries(records(""))).toEqual([
+ { path, size: 6 },
+ { path: "package/LICENSE", size: 7 },
+ ]);
+ expect(() => plainTarEntries(records("go/example"))).toThrow(
+ internalReferenceError,
+ );
+ });
+
+ test.each([0x78, 0x67])("accepts POSIX pax metadata typeflag %i", (type) => {
+ const attributes = paxRecords({ ctime: "0.123456789" });
+ expect(
+ plainTarEntries(
+ archive(
+ tarRecord(attributes, { name: "package/PaxHeaders/README.md", type }),
+ tarRecord(Buffer.from("readme"), { name: "package/README.md" }),
+ ),
+ ),
+ ).toEqual([{ path: "package/README.md", size: 6 }]);
+ });
+
+ test("uses local pax paths and sizes and clears them after their member", () => {
+ const path = `package/${"nested/".repeat(16)}README.md`;
+ expect(
+ plainTarEntries(
+ archive(
+ tarRecord(paxRecords({ path, size: "6" }), {
+ name: "package/PaxHeaders/README.md",
+ type: 0x78,
+ }),
+ tarRecord(Buffer.from("readme"), {
+ name: "placeholder",
+ sizeField: octal(0, 12),
+ }),
+ tarRecord(Buffer.from("license"), { name: "package/LICENSE" }),
+ ),
+ ),
+ ).toEqual([
+ { path, size: 6 },
+ { path: "package/LICENSE", size: 7 },
+ ]);
+ });
+
+ test("retains global pax attributes and honors local overrides", () => {
+ expect(
+ plainTarEntries(
+ archive(
+ tarRecord(paxRecords({ path: "package/README.md" }), {
+ name: "GlobalHead",
+ type: 0x67,
+ }),
+ tarRecord(Buffer.from("readme"), { name: "placeholder" }),
+ tarRecord(paxRecords({ path: "package/LICENSE" }), {
+ name: "package/PaxHeaders/LICENSE",
+ type: 0x78,
+ }),
+ tarRecord(Buffer.from("license"), { name: "placeholder" }),
+ ),
+ ),
+ ).toEqual([
+ { path: "package/README.md", size: 6 },
+ { path: "package/LICENSE", size: 7 },
+ ]);
+ });
+
+ test("scans the complete pax metadata payload", () => {
+ expect(() =>
+ plainTarEntries(
+ archive(
+ tarRecord(paxRecords({ comment: "go/example" }), {
+ name: "package/PaxHeaders/README.md",
+ type: 0x78,
+ }),
+ tarRecord(Buffer.from("readme"), { name: "package/README.md" }),
+ ),
+ ),
+ ).toThrow(internalReferenceError);
+ });
+
+ test.each([false, true])(
+ "scans discarded sparse-map padding, marker=%j",
+ (marker) => {
+ const map = Buffer.alloc(blockSize);
+ map.write("1\n512\n512\n");
+ if (marker) map.write("go/example", 32);
+ const bytes = archive(
+ tarRecord(
+ paxRecords({
+ "GNU.sparse.major": "1",
+ "GNU.sparse.minor": "0",
+ "GNU.sparse.name": "package/README.md",
+ "GNU.sparse.realsize": "1024",
+ }),
+ { name: "package/PaxHeaders/README.md", type: 0x78 },
+ ),
+ tarRecord(Buffer.concat([map, Buffer.alloc(blockSize)]), {
+ name: "package/README.md",
+ }),
+ );
+ if (marker)
+ expect(() => plainTarEntries(bytes)).toThrow(internalReferenceError);
+ else
+ expect(plainTarEntries(bytes)).toEqual([
+ { path: "package/README.md", size: 1024 },
+ ]);
+ },
+ );
+
+ for (const format of [
+ "gnu-01",
+ "gnu-nul",
+ "bsd-comments",
+ "bsd-solaris",
+ ] as const) {
+ test.each([false, true])(
+ `scans all stored sparse bytes for ${format}, marker=%j`,
+ (marker) => {
+ let attributes: Record<string, string>;
+ let contents: Buffer;
+ if (format === "gnu-01") {
+ attributes = {
+ "GNU.sparse.size": "1024",
+ "GNU.sparse.numblocks": "1",
+ "GNU.sparse.map": "512,512",
+ };
+ contents = Buffer.alloc(1024);
+ if (marker) contents.write("go/example", 512);
+ } else if (format === "bsd-solaris") {
+ attributes = { "SUN.holesdata": " 512 1024" };
+ contents = Buffer.alloc(1024);
+ if (marker) contents.write("go/example", 0);
+ } else {
+ attributes = {
+ "GNU.sparse.major": format === "gnu-nul" ? "1\0" : "1",
+ "GNU.sparse.minor": "0",
+ "GNU.sparse.name": "package/README.md",
+ "GNU.sparse.realsize": format === "bsd-comments" ? "1536" : "1024",
+ };
+ contents = Buffer.alloc(format === "bsd-comments" ? 1536 : 1024);
+ if (format === "bsd-comments") {
+ const comments = `${`# ${"x".repeat(60)}\n`.repeat(9)}# ${marker ? "go/example" : "public note"}\n1\n1024\n512\n`;
+ contents.write(comments);
+ } else {
+ contents.write("1\n512\n512\n");
+ if (marker) contents.write("go/example", 32);
+ }
+ }
+ const bytes = archive(
+ tarRecord(paxRecords(attributes), {
+ name: "package/PaxHeaders/README.md",
+ type: 0x78,
+ }),
+ tarRecord(contents, { name: "package/README.md" }),
+ );
+ if (marker)
+ expect(() => plainTarEntries(bytes)).toThrow(internalReferenceError);
+ else
+ expect(plainTarEntries(bytes)).toEqual([
+ { path: "package/README.md", size: contents.length },
+ ]);
+ },
+ );
+ }
+
+ test("checks stored old GNU sparse bodies", () => {
+ const contents = Buffer.alloc(1024);
+ contents.write("go/example");
+ const record = tarRecord(contents, {
+ name: "package/README.md",
+ magic: "ustar ",
+ version: " \0",
+ });
+ for (const offset of [386, 410]) {
+ octal(0, 12).copy(record, offset);
+ octal(512, 12).copy(record, offset + 12);
+ }
+ octal(1024, 12).copy(record, 483);
+ expect(() => plainTarEntries(archive(record))).toThrow(
+ internalReferenceError,
+ );
+ });
+
+ test("checks stored bytes after an empty repeated sparse attribute", () => {
+ const contents = Buffer.alloc(1024);
+ contents.write("go/example", 512);
+ const attributes = Buffer.concat([
+ paxRecords({ "GNU.sparse.map": "512,512" }),
+ paxRecords({ "GNU.sparse.map": "" }),
+ ]);
+ expect(() =>
+ plainTarEntries(
+ archive(
+ tarRecord(attributes, { name: "PaxHeaders/readme", type: 0x78 }),
+ tarRecord(contents, { name: "package/README.md" }),
+ ),
+ ),
+ ).toThrow(internalReferenceError);
+ });
+
+ test("checks markers across concatenated archive metadata", () => {
+ const record = tarRecord(Buffer.from("Public contents."), {
+ name: "package/README.md",
+ });
+ record.write("go/", record.length - 3);
+ expect(() =>
+ plainTarEntries(
+ archive(
+ record,
+ tarRecord(paxRecords({ uid: "0" }), { name: "example", type: 0x78 }),
+ ),
+ ),
+ ).toThrow(internalReferenceError);
+ });
+
+ test.each([false, true])(
+ "keeps public Brotli content semantics across retained sparse extents, split=%j",
+ (split) => {
+ const path = "package/runtime.mjs.br";
+ const size = cleanCompressedPayload.length;
+ const middle = cleanCompressedPayload.indexOf("Go/w") + 2;
+ expect(middle).toBeGreaterThan(2);
+ const bytes = archive(
+ tarRecord(
+ paxRecords({
+ "GNU.sparse.size": String(size),
+ "GNU.sparse.numblocks": split ? "2" : "1",
+ "GNU.sparse.map": split
+ ? `0,${middle},${middle},${size - middle}`
+ : `0,${size}`,
+ }),
+ { name: "PaxHeaders/runtime", type: 0x78 },
+ ),
+ tarRecord(cleanCompressedPayload, { name: path }),
+ );
+ expect(plainTarEntries(bytes)).toEqual([{ path, size }]);
+ expect(() =>
+ assertStoredSparseContents(
+ readTarArchive(bytes),
+ new Map([[path, cleanCompressedPayload]]),
+ ),
+ ).not.toThrow();
+ },
+ );
+
+ test("keeps retained extents between separate sparse metadata segments", () => {
+ const path = "package/logo.png";
+ const logo = readFileSync(
+ new URL(
+ "../../../plugins/codex-security/assets/logo.png",
+ import.meta.url,
+ ),
+ );
+ const firstPadding = Buffer.alloc(512 - 250);
+ firstPadding.write(".openai.", firstPadding.length - 8);
+ const secondPadding = Buffer.alloc(512 - 250);
+ secondPadding.write("org");
+ const bytes = archive(
+ tarRecord(
+ paxRecords({
+ "GNU.sparse.size": String(logo.length),
+ "GNU.sparse.numblocks": "3",
+ "GNU.sparse.map": `0,250,250,250,500,${logo.length - 500}`,
+ }),
+ { name: "PaxHeaders/logo", type: 0x78 },
+ ),
+ tarRecord(
+ Buffer.concat([
+ logo.subarray(0, 250),
+ firstPadding,
+ logo.subarray(250, 500),
+ secondPadding,
+ logo.subarray(500),
+ ]),
+ { name: path },
+ ),
+ );
+ expect(() =>
+ assertStoredSparseContents(
+ readTarArchive(bytes),
+ new Map([[path, logo]]),
+ ),
+ ).not.toThrow();
+ });
+
+ test("checks markers crossing discarded padding and retained sparse bytes", () => {
+ const path = "package/logo.png";
+ const logo = readFileSync(
+ new URL(
+ "../../../plugins/codex-security/assets/logo.png",
+ import.meta.url,
+ ),
+ );
+ expect(logo.subarray(387, 390).toString()).toBe("org");
+ const padding = Buffer.alloc(512 - 387);
+ padding.write(".openai.", padding.length - 8);
+ const bytes = archive(
+ tarRecord(
+ paxRecords({
+ "GNU.sparse.size": String(logo.length),
+ "GNU.sparse.numblocks": "2",
+ "GNU.sparse.map": `0,387,387,${logo.length - 387}`,
+ }),
+ { name: "PaxHeaders/logo", type: 0x78 },
+ ),
+ tarRecord(
+ Buffer.concat([logo.subarray(0, 387), padding, logo.subarray(387)]),
+ { name: path },
+ ),
+ );
+ expect(() =>
+ assertStoredSparseContents(
+ readTarArchive(bytes),
+ new Map([[path, logo]]),
+ ),
+ ).toThrow(internalReferenceError);
+ });
+
+ test("uses the native GNU sparse name while retaining stored content", () => {
+ const path = "package/README.md";
+ const stored = Buffer.alloc(1024);
+ stored.write("1\n512\n512\n");
+ stored.fill(0x78, 512);
+ expect(
+ plainTarEntries(
+ archive(
+ tarRecord(
+ paxRecords({
+ "GNU.sparse.major": "1",
+ "GNU.sparse.minor": "0",
+ "GNU.sparse.name": path,
+ "GNU.sparse.realsize": "1024",
+ }),
+ { name: "PaxHeaders/readme", type: 0x78 },
+ ),
+ tarRecord(stored, { name: "package/GNUSparseFile.1/readme" }),
+ ),
+ ),
+ ).toEqual([{ path, size: stored.byteLength }]);
+ });
+
+ test("accepts an empty size field for an empty file", () => {
+ expect(
+ plainTarEntries(
+ archive(
+ tarRecord(Buffer.alloc(0), {
+ name: "package/README.md",
+ sizeField: Buffer.alloc(12),
+ }),
+ ),
+ ),
+ ).toEqual([{ path: "package/README.md", size: 0 }]);
+ });
+
+ test("rejects invalid or binary size encodings", () => {
+ const base256 = Buffer.alloc(12);
+ base256[0] = 0x80;
+ base256[11] = 1;
+ for (const sizeField of [base256, Buffer.from("00000000008\0")]) {
+ expect(() =>
+ plainTarEntries(
+ archive(
+ tarRecord(Buffer.from("x"), {
+ name: "package/README.md",
+ sizeField,
+ }),
+ ),
+ ),
+ ).toThrow(invalidTarEntryError);
+ }
+ });
+
+ test.each([0, 0x30])("accepts basic V7 regular typeflag %i", (type) => {
+ expect(
+ plainTarEntries(
+ archive(
+ tarRecord(Buffer.from("readme"), {
+ name: "package/README.md",
+ type,
+ magic: "\0".repeat(6),
+ version: "\0\0",
+ }),
+ ),
+ ),
+ ).toEqual([{ path: "package/README.md", size: 6 }]);
+ });
+
+ test("rejects alternate ustar signatures", () => {
+ for (const options of [
+ { magic: "ustar " },
+ { magic: "ustar\0", version: " \0" },
+ ]) {
+ expect(() =>
+ plainTarEntries(
+ archive(
+ tarRecord(Buffer.from("x"), {
+ name: "package/README.md",
+ ...options,
+ }),
+ ),
+ ),
+ ).toThrow(invalidTarEntryError);
+ }
+ });
+
+ test("scans complete header text fields", () => {
+ expect(() =>
+ plainTarEntries(
+ archive(
+ tarRecord(Buffer.from("clean"), {
+ name: "package/README.md",
+ user: "public\0go/example",
+ }),
+ ),
+ ),
+ ).toThrow(internalReferenceError);
+ });
+
+ test("scans complete raw headers", () => {
+ for (const options of [
+ {
+ deviceNumbers: Buffer.concat([
+ Buffer.from("go/example"),
+ Buffer.alloc(6),
+ ]),
+ },
+ { reserved: Buffer.from("go/example") },
+ ]) {
+ expect(() =>
+ plainTarEntries(
+ archive(
+ tarRecord(Buffer.from("clean"), {
+ name: "package/README.md",
+ ...options,
+ }),
+ ),
+ ),
+ ).toThrow(internalReferenceError);
+ }
+ });
+
+ test("accepts explicit empty directory members", () => {
+ expect(
+ plainTarEntries(
+ archive(tarRecord(Buffer.alloc(0), { name: "package/", type: 0x35 })),
+ ),
+ ).toEqual([{ path: "package/", size: 0 }]);
+ expect(() =>
+ plainTarEntries(
+ archive(tarRecord(Buffer.from("x"), { name: "package/", type: 0x35 })),
+ ),
+ ).toThrow(invalidTarEntryError);
+ });
+
+ test("accepts GNU headers without treating timestamps as a path prefix", () => {
+ expect(
+ plainTarEntries(
+ archive(
+ tarRecord(Buffer.from("readme"), {
+ name: "package/README.md",
+ magic: "ustar ",
+ version: " \0",
+ prefix: "00000000000",
+ }),
+ ),
+ ),
+ ).toEqual([{ path: "package/README.md", size: 6 }]);
+ });
+
+ test("accepts padding, optional end markers, and zero blocks between files", () => {
+ const record = tarRecord(Buffer.from("x"), { name: "package/README.md" });
+ record[record.length - 1] = 1;
+ const secondRecord = tarRecord(Buffer.from("y"), {
+ name: "package/LICENSE",
+ });
+
+ expect(plainTarEntries(record)).toEqual([
+ { path: "package/README.md", size: 1 },
+ ]);
+ for (const trailingBytes of [1, 511]) {
+ expect(
+ plainTarEntries(
+ Buffer.concat([archive(record), Buffer.alloc(trailingBytes)]),
+ ),
+ ).toEqual([{ path: "package/README.md", size: 1 }]);
+ }
+ for (const zeroBlocks of [1, 2]) {
+ expect(
+ plainTarEntries(
+ Buffer.concat([
+ record,
+ Buffer.alloc(blockSize * zeroBlocks),
+ secondRecord,
+ ]),
+ ),
+ ).toEqual([
+ { path: "package/README.md", size: 1 },
+ { path: "package/LICENSE", size: 1 },
+ ]);
+ }
+ });
+
+ test("still scans padding and rejects partial blocks or truncated contents", () => {
+ const record = tarRecord(Buffer.from("x"), { name: "package/README.md" });
+ const paddingMarker = Buffer.from(record);
+ paddingMarker.write("go/example", blockSize + 1);
+ expect(() => plainTarEntries(paddingMarker)).toThrow(
+ internalReferenceError,
+ );
+ for (const bytes of [
+ Buffer.concat([archive(record), Buffer.from([1])]),
+ record.subarray(0, blockSize),
+ ]) {
+ expect(() => plainTarEntries(bytes)).toThrow(invalidTarEntryError);
+ }
+ });
+});
diff --git a/sdk/typescript/tests-ts/package-tar-fixtures.ts b/sdk/typescript/tests-ts/package-tar-fixtures.ts
new file mode 100644
index 000000000..550e1534b
--- /dev/null
+++ b/sdk/typescript/tests-ts/package-tar-fixtures.ts
@@ -0,0 +1,145 @@
+export const blockSize = 512;
+
+export function octal(value: number, width: number, terminator = "\0"): Buffer {
+ return Buffer.from(
+ value.toString(8).padStart(width - terminator.length, "0") + terminator,
+ );
+}
+
+function tarHeader({
+ name,
+ prefix = "",
+ size = 0,
+ mtime = 0,
+ mode = 0o644,
+ type = 0x30,
+ sizeField = octal(size, 12, " "),
+ magic = "ustar\0",
+ version = "00",
+ user = "",
+ deviceNumbers = Buffer.alloc(16),
+ reserved = Buffer.alloc(12),
+}: {
+ name: string;
+ prefix?: string;
+ size?: number;
+ mtime?: number;
+ mode?: number;
+ type?: number;
+ sizeField?: Buffer;
+ magic?: string;
+ version?: string;
+ user?: string;
+ deviceNumbers?: Buffer;
+ reserved?: Buffer;
+}): Buffer {
+ const header = Buffer.alloc(blockSize);
+ header.write(name, 0, 100, "utf8");
+ octal(mode, 8).copy(header, 100);
+ octal(0, 8).copy(header, 108);
+ octal(0, 8).copy(header, 116);
+ sizeField.copy(header, 124);
+ octal(mtime, 12).copy(header, 136);
+ header.fill(0x20, 148, 156);
+ header[156] = type;
+ header.write(magic, 257, "binary");
+ header.write(version, 263, "binary");
+ header.write(user, 265, 32, "utf8");
+ deviceNumbers.copy(header, 329, 0, 16);
+ header.write(prefix, 345, 155, "utf8");
+ reserved.copy(header, 500, 0, 12);
+ const checksum = header.reduce((sum, byte) => sum + byte, 0);
+ Buffer.from(checksum.toString(8).padStart(6, "0") + "\0 ").copy(header, 148);
+ return header;
+}
+
+export function tarRecord(
+ contents: Buffer,
+ options: Omit<Parameters<typeof tarHeader>[0], "size">,
+): Buffer {
+ return Buffer.concat([
+ tarHeader({ ...options, size: contents.length }),
+ contents,
+ Buffer.alloc(
+ Math.ceil(contents.length / blockSize) * blockSize - contents.length,
+ ),
+ ]);
+}
+
+export function oldGnuSparseRecord(
+ contents: Buffer,
+ name: string,
+ extents = [{ offset: 0, size: contents.length }],
+ metadataMarker?: "continuation" | "boundary" | "unused",
+): Buffer {
+ const stored = Buffer.concat(
+ extents.flatMap(({ offset, size }, index) => [
+ contents.subarray(offset, offset + size),
+ Buffer.alloc(index < extents.length - 1 ? (512 - (size % 512)) % 512 : 0),
+ ]),
+ );
+ const record = tarRecord(stored, {
+ name,
+ type: 0x53,
+ magic: "ustar ",
+ version: " \0",
+ });
+ const header = record.subarray(0, blockSize);
+ const continuations = [];
+ let remaining = extents;
+ let map = header;
+ let start = 386;
+ let count = 4;
+ do {
+ for (const [index, extent] of remaining.slice(0, count).entries()) {
+ octal(extent.offset, 12).copy(map, start + index * 24);
+ octal(extent.size, 12).copy(map, start + index * 24 + 12);
+ }
+ remaining = remaining.slice(count);
+ map[start + count * 24] = remaining.length > 0 ? 1 : 0;
+ if (remaining.length > 0) {
+ map = Buffer.alloc(blockSize);
+ continuations.push(map);
+ start = 0;
+ count = 21;
+ }
+ } while (remaining.length > 0);
+ if (metadataMarker === "unused") {
+ const used =
+ extents.length <= 4 ? extents.length : ((extents.length - 5) % 21) + 1;
+ map.write("public", start + (used + 1) * 24);
+ }
+ octal(contents.length, 12).copy(header, 483);
+ if (metadataMarker === "continuation")
+ continuations.at(-1)!.write("go/test", 505);
+ if (metadataMarker === "boundary") header.write("go/", 509);
+ header.fill(0x20, 148, 156);
+ octal(
+ header.reduce((sum, byte) => sum + byte, 0),
+ 8,
+ "\0 ",
+ ).copy(header, 148);
+ return Buffer.concat([header, ...continuations, record.subarray(blockSize)]);
+}
+
+export function archive(...records: Buffer[]): Buffer {
+ return Buffer.concat([...records, Buffer.alloc(blockSize * 2)]);
+}
+
+export function paxRecords(attributes: Record<string, string>): Buffer {
+ return Buffer.concat(
+ Object.entries(attributes).map(([key, value]) => {
+ const record = ` ${key}=${value}\n`;
+ let length = Buffer.byteLength(record) + 1;
+ while (length !== Buffer.byteLength(record) + String(length).length)
+ length = Buffer.byteLength(record) + String(length).length;
+ return Buffer.from(`${length}${record}`);
+ }),
+ );
+}
+
+// Synthetic alphanumeric text whose compressed bytes happen to contain " Go/w".
+export const cleanCompressedPayload = Buffer.from(
+ "G/8DAGRwm7EiGn34sGFNRM5DuwB8/2zECGTx7hk1/DY6puqEJWJumAz3HfjBg7bx5FxwfFCafaN2of0Oi2uTSSE+wAYOJS4Ii53urIa5BOIkTbR5t2YJ4/UM6dWzXcEvk92TZNz/Y7hNVxjrsXKemVnN4i1pUOdQiCYQNLU7Dbh77pXyrGIWSl+X+L7DUfFR+4Rz3GF85xCbj1NuwlDfIWj42+ueic5NDX4bPXmp46fE8bTdpAUmMvt48x99pm2wlp+oKNvvcb64GT4i7F9zCJMIdfmc06UCn7Go4jQ1WI2P+e05ZbpsWoTkPXnAFULsyUtefeZ1d5nbvE2CDXV3eF7Hxc2KgtQQv8j/CIEd+9vG3REfGcJVIVMqjo1SKzOfADpWQYh3p18YFMyF6R4qOYzPB+zjBy/ZQiCaNm1LpVYdJ7XL0wXPjAVOzJU4zeSUvNFqQ0TeXFGat6ikPEii43FRFSx3aQi2HnWI3rBd1tts0LlTjIWUq8+agHu/mwmJ+jWxAF74Yvij4++fUTHxm8PvqXOK+3QpR82rAPZ+pyRgKX1Z9k9XHlVwNXzJCd5laKwKOWnAOF6vKnrqhZz4aw1VqEJ+1x/4+AM++Pw58PdlRKBjKoLl7DKgEBCGKsd5FF8XPQz9pC3MWj1UbdVa/jLl5M1iIfl9P44zgKV1TaG1LEjuS09t7e0UFSrgXI0BWvBuyNfEras635PJBiZuI8mX6s5yt5NkocDkATyMNP1BnW+wKyTzduIFbd2yV45t/t6ttd5y5FeAAKnT0Jjsmk3zbYdl12fpKKDUvs363EVzTjWEhNaoTC3ntUZKNlYzX74UMDogR28vd8vvJniJvfamegrjMcf3fviFr9caYVQH+bq/TpxK078VtU0Z5iah0E2+ttjzWXHi//ftzs9e+xArq7zohSgVrMpgsL2kjfl/zaltHiNmTgogtQI6gRiI1ICa3kimcK2a/gj0cTkCVwKywUt6xu1N2x2lPi7dk/pnx8/iSkdyphyFA4OQov+d8maitV0XwU9/NKTW048C",
+ "base64",
+);
diff --git a/sdk/typescript/tests-ts/package-tar-listing.test.ts b/sdk/typescript/tests-ts/package-tar-listing.test.ts
index 4c0e585e5..11ba4ef2a 100644
--- a/sdk/typescript/tests-ts/package-tar-listing.test.ts
+++ b/sdk/typescript/tests-ts/package-tar-listing.test.ts
@@ -1,6 +1,244 @@
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import {
+ chmodSync,
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ readdirSync,
+ realpathSync,
+ rmSync,
+ writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { delimiter, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { brotliCompressSync, brotliDecompressSync, gzipSync } from "node:zlib";
import { describe, expect, test } from "bun:test";
+import {
+ archive,
+ cleanCompressedPayload,
+ octal,
+ oldGnuSparseRecord,
+ paxRecords,
+ tarRecord,
+} from "./package-tar-fixtures.js";
-import { regularTarListingLines } from "../scripts/package-tar-listing.mjs";
+const { assertTarListingSizes, regularTarListingLines } = (await import(
+ new URL("../scripts/package-tar-listing.mjs", import.meta.url).href
+)) as {
+ regularTarListingLines: (listing: string) => string[];
+ assertTarListingSizes: (lines: string[], maximum: number) => number[];
+};
+const { packageDistFiles } = (await import(
+ new URL("../scripts/package-dist-files.mjs", import.meta.url).href
+)) as { packageDistFiles: readonly string[] };
+const pluginContract = {
+ externalOwnedExact: [".codex-plugin/plugin.json"],
+ shippedExact: ["scripts/launch_codex_security_mcp"],
+};
+
+function packageTar({
+ trailingZeroBytes = 0,
+ sizeTerminator = " ",
+ type = 0x30,
+ compatibleLayout = false,
+ rootDirectoryMode = 0o755,
+ directoryMode = 0o755,
+ readmeMode = 0o644,
+ mtime = 0,
+ readmeSparse,
+ readmeLongName = false,
+ additionalRecords = [],
+ overrides = new Map(),
+}: {
+ trailingZeroBytes?: number;
+ sizeTerminator?: string;
+ type?: number;
+ compatibleLayout?: boolean;
+ rootDirectoryMode?: number;
+ directoryMode?: number;
+ readmeMode?: number;
+ mtime?: number;
+ readmeSparse?: "0.0" | "1.0";
+ readmeLongName?: boolean;
+ additionalRecords?: Buffer[];
+ overrides?: ReadonlyMap<string, Buffer>;
+} = {}): Buffer {
+ const executablePaths = [
+ "package/bin/codex-security.mjs",
+ "package/_bundled_plugin/scripts/launch_codex_security_mcp",
+ ];
+ const paths = [
+ "package/package.json",
+ "package/README.md",
+ "package/docs/cli.md",
+ "package/docs/findings-service.md",
+ "package/docs/dedupe-records.md",
+ "package/schemas/project-config.schema.json",
+ "package/LICENSE",
+ ...executablePaths,
+ ...packageDistFiles,
+ "package/_bundled_plugin/.codex-plugin/plugin.json",
+ ];
+ const records = paths.map((path) => {
+ const override = overrides.get(path);
+ if (override !== undefined) return override;
+ if (path === "package/README.md" && readmeSparse !== undefined) {
+ const contents = Buffer.alloc(512, 0x78);
+ const map = Buffer.alloc(512);
+ map.write("1\n512\n512\n");
+ const attributes: Record<string, string> =
+ readmeSparse === "0.0"
+ ? {
+ "GNU.sparse.size": "1024",
+ "GNU.sparse.numblocks": "1",
+ "GNU.sparse.map": "512,512",
+ }
+ : {
+ "GNU.sparse.major": "1",
+ "GNU.sparse.minor": "0",
+ "GNU.sparse.name": path,
+ "GNU.sparse.realsize": "1024",
+ };
+ return Buffer.concat([
+ tarRecord(paxRecords(attributes), {
+ name: "PaxHeaders/readme",
+ type: 0x78,
+ }),
+ tarRecord(
+ readmeSparse === "0.0" ? contents : Buffer.concat([map, contents]),
+ {
+ name:
+ readmeSparse === "0.0" ? path : "package/GNUSparseFile.1/readme",
+ },
+ ),
+ ]);
+ }
+ const contents =
+ path === "package/package.json"
+ ? Buffer.from(
+ JSON.stringify({
+ license: "Apache-2.0",
+ name: "@openai/codex-security",
+ }),
+ )
+ : path.endsWith(".json") || path.endsWith(".map")
+ ? Buffer.from("{}\n")
+ : Buffer.from("fixture\n");
+ const record = tarRecord(contents, {
+ name: path,
+ type,
+ mtime,
+ mode: executablePaths.includes(path)
+ ? 0o755
+ : path === "package/README.md"
+ ? readmeMode
+ : 0o644,
+ sizeField: octal(contents.length, 12, sizeTerminator),
+ ...(compatibleLayout ? { magic: "ustar ", version: " \0" } : {}),
+ });
+ if (compatibleLayout) record[record.length - 1] = 1;
+ if (path === "package/README.md" && readmeLongName)
+ return Buffer.concat([
+ tarRecord(Buffer.from(`${path}\0`), {
+ name: "././@LongLink",
+ type: 0x4c,
+ }),
+ record,
+ ]);
+ return record;
+ });
+ records.push(...additionalRecords);
+ if (compatibleLayout) {
+ const directories = new Set<string>();
+ for (const path of paths) {
+ const parts = path.split("/");
+ for (let index = 1; index < parts.length; index++) {
+ directories.add(`${parts.slice(0, index).join("/")}/`);
+ }
+ }
+ return Buffer.concat([
+ ...[...directories].map((name) =>
+ tarRecord(Buffer.alloc(0), {
+ name,
+ type: 0x35,
+ mode: name === "package/" ? rootDirectoryMode : directoryMode,
+ }),
+ ),
+ ...records.flatMap((record) => [Buffer.alloc(512), record]),
+ ]);
+ }
+ return archive(...records, Buffer.alloc(trailingZeroBytes));
+}
+
+function commandPath(command: string): string {
+ const lookup = spawnSync(
+ process.platform === "win32" ? "where.exe" : "which",
+ [command],
+ { encoding: "utf8", windowsHide: true },
+ );
+ if (lookup.error !== undefined) throw lookup.error;
+ const path = lookup.stdout.split(/\r?\n/u).find(Boolean);
+ if (lookup.status !== 0 || path === undefined) {
+ throw new Error(`Could not resolve ${command}.`);
+ }
+ return path;
+}
+
+function installPackage(
+ root: string,
+ archivePath: string,
+ environment: NodeJS.ProcessEnv,
+) {
+ const consumer = join(root, "consumer");
+ mkdirSync(consumer);
+ writeFileSync(
+ join(consumer, "package.json"),
+ JSON.stringify({ name: "synthetic-consumer", private: true }),
+ );
+ const resolved = spawnSync(
+ commandPath("node"),
+ [
+ "--input-type=module",
+ "-e",
+ `import { resolveNpm } from ${JSON.stringify(new URL("../scripts/package-smoke-npm.mjs", import.meta.url).href)}; console.log(JSON.stringify(await resolveNpm()));`,
+ ],
+ { encoding: "utf8", env: environment, windowsHide: true },
+ );
+ expect({ status: resolved.status, stderr: resolved.stderr }).toEqual({
+ status: 0,
+ stderr: "",
+ });
+ const npm = JSON.parse(resolved.stdout) as {
+ command: string;
+ args: string[];
+ };
+ return spawnSync(
+ npm.command,
+ [
+ ...npm.args,
+ "install",
+ "--global=false",
+ "--prefix",
+ consumer,
+ "--offline",
+ "--ignore-scripts",
+ "--package-lock=false",
+ "--no-audit",
+ "--no-fund",
+ archivePath,
+ ],
+ {
+ cwd: consumer,
+ env: environment,
+ encoding: "utf8",
+ timeout: 30_000,
+ windowsHide: true,
+ },
+ );
+}
describe("npm package tar listings", () => {
test("accepts regular entries with Unix or Windows line endings", () => {
@@ -17,9 +255,1252 @@ describe("npm package tar listings", () => {
]);
});
+ test.each([
+ "-rw-r--r-- 0/0 33554433 1970-01-01 00:00 package/README.md",
+ "-rw-r--r-- 0 0 0 33554433 Jan 1 1970 package/README.md",
+ ])("bounds native sparse logical sizes before extraction: %s", (line) => {
+ expect(() => assertTarListingSizes([line], 32 * 1024 * 1024)).toThrow(
+ "npm tarball contains an invalid tar entry",
+ );
+ const bounded = line.replace("33554433", "33554432");
+ expect(() =>
+ assertTarListingSizes([bounded], 32 * 1024 * 1024),
+ ).not.toThrow();
+ expect(() =>
+ assertTarListingSizes([bounded, bounded], 32 * 1024 * 1024),
+ ).toThrow();
+ });
+
+ test("retains bounded logical sparse sizes in listing order", () => {
+ expect(
+ assertTarListingSizes(
+ [
+ "drwxr-xr-x 0/0 0 1970-01-01 00:00 package/",
+ "-rw-r--r-- 0/0 1024 1970-01-01 00:00 package/README.md",
+ "-rw-r--r-- 0 0 0 512 Jan 1 1970 package/LICENSE",
+ ],
+ 1536,
+ ),
+ ).toEqual([0, 1024, 512]);
+ });
+
test("rejects symbolic links and other non-regular entries", () => {
expect(() =>
regularTarListingLines("lrwxrwxrwx package/link -> target\r\n"),
).toThrow("npm tarball contains a non-regular entry");
});
+
+ test.each([false, true])(
+ "rejects invalid package paths before Brotli expansion, complete=%p",
+ (complete) => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-path-order-"));
+ try {
+ const archivePath = join(root, "unexpected-brotli.tgz");
+ writeFileSync(
+ archivePath,
+ gzipSync(
+ Buffer.concat([
+ ...(complete ? [packageTar()] : []),
+ tarRecord(Buffer.from("Malformed compressed bytes."), {
+ name: "package/unexpected.br",
+ }),
+ ]),
+ ),
+ );
+ const contractPath = join(root, "plugin contract.json");
+ writeFileSync(contractPath, JSON.stringify(pluginContract));
+ const result = spawnSync(
+ commandPath("node"),
+ [
+ fileURLToPath(
+ new URL("../scripts/check-package.mjs", import.meta.url),
+ ),
+ archivePath,
+ contractPath,
+ ],
+ { cwd: root, encoding: "utf8", timeout: 30_000, windowsHide: true },
+ );
+ expect(result.status).toBe(1);
+ expect(result.stderr).toContain(
+ complete
+ ? "unexpected file: package/unexpected.br"
+ : "missing package/package.json",
+ );
+ } finally {
+ rmSync(root, { force: true, recursive: true });
+ }
+ },
+ );
+
+ test("escapes terminal controls in unexpected archive path diagnostics", () => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-terminal-path-"));
+ try {
+ const archivePath = join(root, "unexpected-path.tgz");
+ writeFileSync(
+ archivePath,
+ gzipSync(
+ Buffer.concat([
+ packageTar(),
+ tarRecord(Buffer.from("fixture"), {
+ name: "package/unexpected\u001b]52;c;synthetic\u0007\u009b",
+ }),
+ ]),
+ ),
+ );
+ const contractPath = join(root, "plugin contract.json");
+ writeFileSync(contractPath, JSON.stringify(pluginContract));
+ const result = spawnSync(
+ commandPath("node"),
+ [
+ fileURLToPath(
+ new URL("../scripts/check-package.mjs", import.meta.url),
+ ),
+ archivePath,
+ contractPath,
+ ],
+ { cwd: root, encoding: "utf8", timeout: 30_000, windowsHide: true },
+ );
+ expect(result.status).toBe(1);
+ expect(result.stderr).toContain("unexpected file: package/unexpected");
+ expect(result.stderr).not.toMatch(/[\u001b\u0007\u009b]/u);
+ expect(result.stderr).toContain("synthetic");
+ } finally {
+ rmSync(root, { force: true, recursive: true });
+ }
+ });
+
+ test("accepts equivalent bounded gzip representations", () => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-gzip-test-"));
+ try {
+ const tarBytes = packageTar({ trailingZeroBytes: 31 * 1024 * 1024 });
+ const archives = [
+ ["default", gzipSync(tarBytes)],
+ ["level-0", gzipSync(tarBytes, { level: 0 })],
+ ["sparse-0.0", gzipSync(packageTar({ readmeSparse: "0.0" }))],
+ ["sparse-1.0", gzipSync(packageTar({ readmeSparse: "1.0" }))],
+ ["gnu-long-name", gzipSync(packageTar({ readmeLongName: true }))],
+ ["npm-size-field", gzipSync(packageTar({ sizeTerminator: " \0" }))],
+ ["nul-regular-file", gzipSync(packageTar({ type: 0 }))],
+ [
+ "future-timestamps",
+ gzipSync(
+ packageTar({ mtime: Math.floor(Date.now() / 1000) + 365 * 86400 }),
+ ),
+ ],
+ ["unreadable-readme", gzipSync(packageTar({ readmeMode: 0 }))],
+ [
+ "read-only-nested-directories",
+ gzipSync(
+ packageTar({ compatibleLayout: true, directoryMode: 0o555 }),
+ ),
+ ],
+ ["posix-size-field", gzipSync(packageTar({ sizeTerminator: "\0" }))],
+ [
+ "compatible-tar-layout",
+ gzipSync(packageTar({ compatibleLayout: true })),
+ ],
+ [
+ "read-only-directories",
+ gzipSync(
+ packageTar({ compatibleLayout: true, rootDirectoryMode: 0o555 }),
+ ),
+ ],
+ ] as const;
+ expect(archives[0][1].length).toBeLessThan(1024 * 1024);
+ expect(archives[1][1].length).toBeGreaterThan(31 * 1024 * 1024);
+
+ const contractPath = join(root, "plugin contract.json");
+ writeFileSync(contractPath, JSON.stringify(pluginContract));
+ const environment: NodeJS.ProcessEnv = { ...process.env };
+ delete environment["CODEX_SECURITY_EXPECTED_GIT_HEAD"];
+ for (const [representation, contents] of archives) {
+ const archivePath = join(root, `${representation}.tgz`);
+ writeFileSync(archivePath, contents);
+ const result = spawnSync(
+ commandPath("node"),
+ [
+ fileURLToPath(
+ new URL("../scripts/check-package.mjs", import.meta.url),
+ ),
+ archivePath,
+ contractPath,
+ ],
+ {
+ cwd: root,
+ encoding: "utf8",
+ env: environment,
+ timeout: 30_000,
+ windowsHide: true,
+ },
+ );
+ expect({
+ representation,
+ status: result.status,
+ stderr: result.stderr,
+ }).toEqual({ representation, status: 0, stderr: "" });
+ }
+ } finally {
+ rmSync(root, { force: true, recursive: true });
+ }
+ });
+
+ test.each(["environment", "user-config"])(
+ "keeps npm fixture installs local with global %s settings",
+ (configuration) => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-npm-local-"));
+ try {
+ const archivePath = join(root, "package.tgz");
+ writeFileSync(archivePath, gzipSync(packageTar()));
+ const userConfig = join(root, "npmrc");
+ writeFileSync(
+ userConfig,
+ configuration === "user-config" ? "global=true\n" : "",
+ );
+ const environment: NodeJS.ProcessEnv = { ...process.env };
+ for (const key of Object.keys(environment)) {
+ if (/^npm_config_(?:global|prefix|userconfig|cache)$/iu.test(key))
+ delete environment[key];
+ }
+ const globalPrefix = join(root, "global-prefix");
+ Object.assign(environment, {
+ npm_config_cache: join(root, "npm-cache"),
+ npm_config_prefix: globalPrefix,
+ npm_config_userconfig: userConfig,
+ ...(configuration === "environment"
+ ? { npm_config_global: "true" }
+ : {}),
+ });
+ const installed = installPackage(root, archivePath, environment);
+ expect({
+ status: installed.status,
+ stderr: installed.status === 0 ? "" : installed.stderr,
+ }).toEqual({
+ status: 0,
+ stderr: "",
+ });
+ expect(
+ existsSync(
+ join(
+ root,
+ "consumer",
+ "node_modules",
+ "@openai",
+ "codex-security",
+ "package.json",
+ ),
+ ),
+ ).toBe(true);
+ expect(existsSync(globalPrefix)).toBe(false);
+ } finally {
+ rmSync(root, { force: true, recursive: true });
+ }
+ },
+ );
+
+ test.each([
+ "pax-full",
+ "pax-map-prefix",
+ "pax-hole",
+ "oldgnu",
+ "sparse-name",
+ "pax-path",
+ "global-path",
+ "pax-then-long-name",
+ "long-name-then-pax",
+ "pax-then-wrong-long-name",
+ "overwrite-text",
+ "overwrite-logo",
+ "ordinary-pax-then-wrong-long-name",
+ "ordinary-pax-then-correct-long-name",
+ "ordinary-long-name-then-pax",
+ ])("validates archive assets as npm installs them: %s", (representation) => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-npm-sparse-"));
+ try {
+ const logo = readFileSync(
+ new URL(
+ "../../../plugins/codex-security/assets/logo.png",
+ import.meta.url,
+ ),
+ );
+ const path = "package/_bundled_plugin/logo.png";
+ const renamed =
+ representation === "sparse-name" || representation === "pax-path";
+ const wrongPath = "package/synthetic-storage.png";
+ const ordinary = representation.startsWith("ordinary-");
+ const pathOrder = ordinary
+ ? representation.slice("ordinary-".length)
+ : representation;
+ const wrongLongName = pathOrder === "pax-then-wrong-long-name";
+ const orderedPath = representation.includes("then");
+ const overwrite = representation.startsWith("overwrite");
+ let stored =
+ representation === "overwrite-text" ? Buffer.from("fixture\n") : logo;
+ const attributes: Record<string, string> = ordinary
+ ? {}
+ : {
+ "GNU.sparse.size": String(stored.length),
+ "GNU.sparse.numblocks": "1",
+ "GNU.sparse.map": `0,${stored.length}`,
+ };
+ if (representation === "pax-map-prefix") {
+ const map = Buffer.alloc(512);
+ map.write(`1\n0\n${logo.length}\n`);
+ stored = Buffer.concat([map, logo]);
+ for (const key of Object.keys(attributes)) delete attributes[key];
+ Object.assign(attributes, {
+ "GNU.sparse.major": "1",
+ "GNU.sparse.minor": "0",
+ "GNU.sparse.name": path,
+ "GNU.sparse.realsize": String(logo.length),
+ });
+ } else if (representation === "pax-hole") {
+ const hole = logo.findIndex(
+ (byte, index) => index > 0 && index % 512 === 0 && byte === 0,
+ );
+ expect(hole).toBeGreaterThan(0);
+ stored = Buffer.concat([
+ logo.subarray(0, hole),
+ logo.subarray(hole + 1),
+ ]);
+ attributes["GNU.sparse.numblocks"] = "2";
+ attributes["GNU.sparse.map"] =
+ `0,${hole},${hole + 1},${logo.length - hole - 1}`;
+ }
+ if (
+ !ordinary &&
+ (renamed || orderedPath || representation === "global-path")
+ )
+ attributes["GNU.sparse.name"] = path;
+ if (overwrite) attributes["GNU.sparse.name"] = "package/README.md";
+ if (representation === "pax-path" || orderedPath)
+ attributes["path"] =
+ representation === "pax-then-long-name" ? wrongPath : path;
+ const longName = tarRecord(
+ Buffer.from(
+ `${representation === "pax-then-long-name" || representation === "ordinary-pax-then-correct-long-name" || pathOrder === "long-name-then-pax" ? path : wrongPath}\0`,
+ ),
+ { name: "././@LongLink", type: 0x4c },
+ );
+ const record =
+ representation === "oldgnu"
+ ? oldGnuSparseRecord(logo, path)
+ : Buffer.concat([
+ ...(representation === "global-path"
+ ? [
+ tarRecord(paxRecords({ path: wrongPath }), {
+ name: "GlobalHead",
+ type: 0x67,
+ }),
+ ]
+ : []),
+ ...(pathOrder === "long-name-then-pax" ? [longName] : []),
+ tarRecord(paxRecords(attributes), {
+ name: "PaxHeaders/asset",
+ type: 0x78,
+ }),
+ ...(pathOrder.startsWith("pax-then") ? [longName] : []),
+ tarRecord(stored, {
+ name: renamed || ordinary ? wrongPath : path,
+ }),
+ ]);
+ const archivePath = join(root, "package.tgz");
+ writeFileSync(
+ archivePath,
+ gzipSync(
+ packageTar({
+ overrides: overwrite
+ ? new Map([["package/README.md", Buffer.alloc(0)]])
+ : undefined,
+ additionalRecords: overwrite
+ ? [tarRecord(logo, { name: path }), record]
+ : [record],
+ }),
+ ),
+ );
+ const nativeRoot = join(root, "native");
+ mkdirSync(nativeRoot);
+ const extracted = spawnSync(
+ "tar",
+ ["-xzf", archivePath, "-C", nativeRoot],
+ { encoding: "buffer", timeout: 30_000, windowsHide: true },
+ );
+ expect({
+ status: extracted.status,
+ stderr: extracted.stderr.toString(),
+ }).toEqual({ status: 0, stderr: "" });
+ // These incompatible packages do not promise a portable native asset path.
+ if (!wrongLongName)
+ expect(readFileSync(join(nativeRoot, path)).equals(logo)).toBe(true);
+ const contractPath = join(root, "contract.json");
+ writeFileSync(
+ contractPath,
+ JSON.stringify({
+ ...pluginContract,
+ shippedExact: [...pluginContract.shippedExact, "logo.png"],
+ }),
+ );
+ const environment: NodeJS.ProcessEnv = {
+ ...process.env,
+ npm_config_cache: join(root, "npm-cache"),
+ };
+ delete environment["CODEX_SECURITY_EXPECTED_GIT_HEAD"];
+ const installed = installPackage(root, archivePath, environment);
+ const consumer = join(root, "consumer");
+ expect({
+ status: installed.status,
+ stderr: installed.status === 0 ? "" : installed.stderr,
+ }).toEqual({ status: 0, stderr: "" });
+ const installedPackage = join(
+ consumer,
+ "node_modules",
+ "@openai",
+ "codex-security",
+ );
+ const installedLogo = join(
+ installedPackage,
+ "_bundled_plugin",
+ "logo.png",
+ );
+ if (wrongLongName) {
+ expect(existsSync(installedLogo)).toBe(false);
+ expect(
+ readFileSync(
+ join(installedPackage, wrongPath.slice("package/".length)),
+ ).equals(logo),
+ ).toBe(true);
+ }
+ const compatible = [
+ "pax-full",
+ "pax-path",
+ "global-path",
+ "pax-then-long-name",
+ "long-name-then-pax",
+ "overwrite-logo",
+ "ordinary-pax-then-correct-long-name",
+ "ordinary-long-name-then-pax",
+ ].includes(representation);
+ expect(
+ existsSync(installedLogo) && readFileSync(installedLogo).equals(logo),
+ ).toBe(compatible);
+ if (overwrite)
+ expect(readFileSync(installedLogo).equals(stored)).toBe(true);
+ const checked = spawnSync(
+ commandPath("node"),
+ [
+ fileURLToPath(
+ new URL("../scripts/check-package.mjs", import.meta.url),
+ ),
+ archivePath,
+ contractPath,
+ ],
+ {
+ cwd: root,
+ env: environment,
+ encoding: "utf8",
+ timeout: 30_000,
+ windowsHide: true,
+ },
+ );
+ expect({
+ status: checked.status,
+ stderr: compatible ? checked.stderr : "",
+ }).toEqual({ status: compatible ? 0 : 1, stderr: "" });
+ } finally {
+ rmSync(root, { force: true, recursive: true });
+ }
+ });
+
+ test.each([
+ "oldgnu-manifest",
+ "pax-map-prefix",
+ "overwrite-json",
+ "overwrite-metadata",
+ "overwrite-git-head",
+ "pax-full",
+ "pax-path",
+ "oldgnu-readme",
+ ])("validates the manifest npm installs: %s", (representation) => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-npm-manifest-"));
+ try {
+ const path = "package/package.json";
+ const gitHead = "a".repeat(40);
+ const metadata = {
+ name: "@openai/codex-security",
+ version: "1.0.0",
+ license: "Apache-2.0",
+ gitHead,
+ };
+ const manifest = Buffer.from(JSON.stringify(metadata));
+ const overrides = new Map([[path, tarRecord(manifest, { name: path })]]);
+ if (representation.startsWith("oldgnu")) {
+ const oldPath =
+ representation === "oldgnu-manifest" ? path : "package/README.md";
+ overrides.set(
+ oldPath,
+ oldGnuSparseRecord(
+ oldPath === path ? manifest : Buffer.from("fixture\n"),
+ oldPath,
+ ),
+ );
+ } else {
+ const overwrite = representation.startsWith("overwrite");
+ const logicalPath = overwrite ? "package/README.md" : path;
+ let stored = overwrite
+ ? Buffer.from(
+ representation === "overwrite-json"
+ ? "fixture\n"
+ : JSON.stringify({
+ ...metadata,
+ ...(representation === "overwrite-metadata"
+ ? { license: "MIT" }
+ : { gitHead: "b".repeat(40) }),
+ }),
+ )
+ : manifest;
+ const attributes: Record<string, string> = {
+ "GNU.sparse.size": String(stored.length),
+ "GNU.sparse.numblocks": "1",
+ "GNU.sparse.map": `0,${stored.length}`,
+ "GNU.sparse.name": logicalPath,
+ };
+ if (representation === "pax-map-prefix") {
+ const map = Buffer.alloc(512);
+ map.write(`1\n0\n${stored.length}\n`);
+ for (const key of Object.keys(attributes)) delete attributes[key];
+ Object.assign(attributes, {
+ "GNU.sparse.major": "1",
+ "GNU.sparse.minor": "0",
+ "GNU.sparse.name": path,
+ "GNU.sparse.realsize": String(stored.length),
+ });
+ stored = Buffer.concat([map, stored]);
+ }
+ if (representation === "pax-path") attributes["path"] = path;
+ overrides.set(
+ logicalPath,
+ Buffer.concat([
+ tarRecord(paxRecords(attributes), {
+ name: "PaxHeaders/manifest",
+ type: 0x78,
+ }),
+ tarRecord(stored, {
+ name:
+ representation === "pax-path"
+ ? "package/synthetic-storage.json"
+ : path,
+ }),
+ ]),
+ );
+ }
+ const archivePath = join(root, "package.tgz");
+ writeFileSync(archivePath, gzipSync(packageTar({ overrides })));
+ const extracted = spawnSync("tar", ["-xOzf", archivePath, path], {
+ encoding: "buffer",
+ timeout: 30_000,
+ windowsHide: true,
+ });
+ expect({
+ status: extracted.status,
+ stderr: extracted.stderr.toString(),
+ }).toEqual({ status: 0, stderr: "" });
+ expect(extracted.stdout.equals(manifest)).toBe(true);
+ const environment: NodeJS.ProcessEnv = {
+ ...process.env,
+ npm_config_cache: join(root, "npm-cache"),
+ CODEX_SECURITY_EXPECTED_GIT_HEAD: gitHead,
+ };
+ const installed = installPackage(root, archivePath, environment);
+ const invalidJson = [
+ "oldgnu-manifest",
+ "pax-map-prefix",
+ "overwrite-json",
+ ].includes(representation);
+ expect(installed.error).toBeUndefined();
+ if (invalidJson) expect(installed.status).not.toBe(0);
+ else {
+ expect({
+ status: installed.status,
+ stderr: installed.status === 0 ? "" : installed.stderr,
+ }).toEqual({ status: 0, stderr: "" });
+ const actual = JSON.parse(
+ readFileSync(
+ join(
+ root,
+ "consumer",
+ "node_modules",
+ "@openai",
+ "codex-security",
+ "package.json",
+ ),
+ "utf8",
+ ),
+ );
+ expect(actual.license).toBe(
+ representation === "overwrite-metadata" ? "MIT" : metadata.license,
+ );
+ expect(actual.gitHead).toBe(
+ representation === "overwrite-git-head" ? "b".repeat(40) : gitHead,
+ );
+ }
+ const compatible = ["pax-full", "pax-path", "oldgnu-readme"].includes(
+ representation,
+ );
+ const contractPath = join(root, "contract.json");
+ writeFileSync(contractPath, JSON.stringify(pluginContract));
+ const checked = spawnSync(
+ commandPath("node"),
+ [
+ fileURLToPath(
+ new URL("../scripts/check-package.mjs", import.meta.url),
+ ),
+ archivePath,
+ contractPath,
+ ],
+ {
+ cwd: root,
+ env: environment,
+ encoding: "utf8",
+ timeout: 30_000,
+ windowsHide: true,
+ },
+ );
+ expect({
+ status: checked.status,
+ stderr: compatible ? checked.stderr : "",
+ }).toEqual({ status: compatible ? 0 : 1, stderr: "" });
+ if (representation === "overwrite-metadata")
+ expect(checked.stderr).toContain("expected public metadata");
+ if (representation === "overwrite-git-head")
+ expect(checked.stderr).toContain("gitHead must match release commit");
+ } finally {
+ rmSync(root, { force: true, recursive: true });
+ }
+ });
+
+ test("checks reconstructed sparse binary assets and their discarded metadata", () => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-sparse-assets-"));
+ try {
+ const logo = readFileSync(
+ new URL(
+ "../../../plugins/codex-security/assets/logo.png",
+ import.meta.url,
+ ),
+ );
+ const oldGnuText = Buffer.alloc(32 * 1024);
+ const oldGnuExtents = Array.from({ length: 32 }, (_, index) => {
+ oldGnuText.write("Public contents.", index * 1024);
+ return { offset: index * 1024, size: 512 };
+ });
+ oldGnuExtents.push({ offset: oldGnuText.length, size: 0 });
+ const unicodeCompressedPayload = Buffer.from(cleanCompressedPayload);
+ Buffer.from("😀").copy(
+ unicodeCompressedPayload,
+ unicodeCompressedPayload.indexOf("Go/w") - 4,
+ );
+ const leadingMarker = cleanCompressedPayload.indexOf("Go/w");
+ expect(leadingMarker).toBeGreaterThan(0);
+ const repositoryCompressedPayload = Buffer.from(cleanCompressedPayload);
+ const repositoryMarker = Buffer.from(
+ ["github.com", "openai", "openai"].join("/"),
+ );
+ repositoryMarker.copy(repositoryCompressedPayload, 300);
+ const repositoryMarkerEnd = 300 + repositoryMarker.length;
+ expect(
+ brotliDecompressSync(repositoryCompressedPayload).toString("utf8"),
+ ).toMatch(/^[A-Za-z0-9]+$/u);
+ const middle = Math.floor(cleanCompressedPayload.length / 2);
+ const cases = [
+ {
+ name: "brotli",
+ files: [["runtime.mjs.br", cleanCompressedPayload, true]],
+ },
+ {
+ name: "unicode-brotli",
+ files: [["runtime.mjs.br", unicodeCompressedPayload, true]],
+ },
+ {
+ name: "sparse-brotli-leading-delimiter",
+ files: [
+ [
+ "runtime.mjs.br.part-000",
+ cleanCompressedPayload.subarray(0, leadingMarker),
+ false,
+ ],
+ [
+ "runtime.mjs.br.part-001",
+ cleanCompressedPayload.subarray(leadingMarker),
+ true,
+ ],
+ ],
+ },
+ {
+ name: "sparse-brotli-trailing-delimiter",
+ files: [
+ [
+ "runtime.mjs.br.part-000",
+ repositoryCompressedPayload.subarray(0, repositoryMarkerEnd),
+ true,
+ ],
+ [
+ "runtime.mjs.br.part-001",
+ repositoryCompressedPayload.subarray(repositoryMarkerEnd),
+ false,
+ ],
+ ],
+ },
+ {
+ name: "mixed-split-brotli",
+ files: [
+ [
+ "runtime.mjs.br.part-000",
+ cleanCompressedPayload.subarray(0, middle),
+ true,
+ ],
+ [
+ "runtime.mjs.br.part-001",
+ cleanCompressedPayload.subarray(middle),
+ false,
+ ],
+ ],
+ },
+ {
+ name: "mixed-sparse-split-tail",
+ files: [
+ [
+ "runtime.mjs.br.part-000",
+ cleanCompressedPayload.subarray(0, middle),
+ true,
+ ],
+ [
+ "runtime.mjs.br.part-001",
+ cleanCompressedPayload.subarray(middle),
+ "0.1-tail",
+ ],
+ ],
+ error: "npm tarball contains trailing Brotli data",
+ },
+ { name: "png", files: [["logo.png", logo, true]] },
+ {
+ name: "pax-sparse-zero-byte-hole-png",
+ files: [["logo.png", logo, "0.1-hole"]],
+ error: "npm tarball contains an unexpected PNG asset",
+ },
+ {
+ name: "pax-sparse-zero-byte-hole-tail",
+ files: [["logo.png", logo, "0.1-hole"]],
+ tailMarker: true,
+ error:
+ /npm tarball contains (?:an internal reference\.|an invalid tar entry)/u,
+ },
+ {
+ name: "oldgnu-brotli",
+ files: [["runtime.mjs.br", cleanCompressedPayload, "oldgnu"]],
+ error: "npm tarball contains an invalid tar entry",
+ },
+ {
+ name: "oldgnu-png",
+ files: [["logo.png", logo, "oldgnu"]],
+ error: "npm tarball contains an invalid tar entry",
+ },
+ {
+ name: "oldgnu-continuations",
+ files: [["helpers.mjs", oldGnuText, "oldgnu"]],
+ extents: oldGnuExtents,
+ },
+ {
+ name: "oldgnu-unused-main-slot",
+ files: [["runtime.mjs.br", cleanCompressedPayload, "oldgnu"]],
+ metadataMarker: "unused",
+ error: "npm tarball contains an invalid tar entry",
+ },
+ {
+ name: "oldgnu-unused-continuation-slot",
+ files: [["helpers.mjs", oldGnuText, "oldgnu"]],
+ extents: oldGnuExtents,
+ metadataMarker: "unused",
+ },
+ {
+ name: "oldgnu-continuation-marker",
+ files: [["helpers.mjs", oldGnuText, "oldgnu"]],
+ extents: oldGnuExtents,
+ metadataMarker: "continuation",
+ error: "npm tarball contains an internal reference.",
+ },
+ {
+ name: "oldgnu-metadata-boundary-marker",
+ files: [["helpers.mjs", oldGnuText, "oldgnu"]],
+ extents: oldGnuExtents,
+ metadataMarker: "boundary",
+ error: "npm tarball contains an internal reference.",
+ },
+ {
+ name: "oldgnu-text-body-boundary-marker",
+ files: [
+ ["helpers.mjs", Buffer.from("synthetic-reference\n"), "oldgnu"],
+ ],
+ metadataMarker: "boundary",
+ error: "npm tarball contains an internal reference.",
+ },
+ {
+ name: "global-sparse-keys",
+ files: [["runtime.mjs.br", cleanCompressedPayload, false]],
+ globalSparse: true,
+ },
+ {
+ name: "expanded-marker",
+ files: [
+ [
+ "runtime.mjs.br",
+ brotliCompressSync(Buffer.from("go/synthetic-reference")),
+ true,
+ ],
+ ],
+ error: "npm tarball contains an internal reference.",
+ },
+ {
+ name: "sparse-header-map-boundary-marker",
+ files: [["runtime.mjs.br", cleanCompressedPayload, "1.0"]],
+ headerMapMarker: true,
+ error: "npm tarball contains an internal reference.",
+ },
+ {
+ name: "sparse-tail-header-boundary-marker",
+ files: [
+ ["runtime.mjs.br", cleanCompressedPayload, "1.0"],
+ ["extra.mjs", Buffer.from("Public contents."), false],
+ ],
+ tailBoundaryMarker: true,
+ error:
+ /npm tarball contains (?:an internal reference\.|an invalid tar entry:)/u,
+ },
+ {
+ name: "map-padding-marker",
+ files: [["runtime.mjs.br", cleanCompressedPayload, "1.0"]],
+ mapMarker: true,
+ error: "npm tarball contains an internal reference.",
+ },
+ {
+ name: "discarded-tail-marker",
+ files: [["logo.png", logo, "1.0"]],
+ tailMarker: true,
+ error:
+ /npm tarball contains (?:an internal reference\.|an invalid tar entry:)/u,
+ },
+ ] satisfies {
+ name: string;
+ files: [
+ string,
+ Buffer,
+ boolean | "1.0" | "0.1-tail" | "0.1-hole" | "oldgnu",
+ ][];
+ extents?: { offset: number; size: number }[];
+ metadataMarker?: "continuation" | "boundary" | "unused";
+ globalSparse?: boolean;
+ mapMarker?: boolean;
+ headerMapMarker?: boolean;
+ tailMarker?: boolean;
+ tailBoundaryMarker?: boolean;
+ error?: string | RegExp;
+ }[];
+ for (const scenario of cases) {
+ const records = scenario.files.map(([name, contents, sparse]) => {
+ const path = `package/_bundled_plugin/${name}`;
+ if (!sparse) return tarRecord(contents, { name: path });
+ if (sparse === "oldgnu")
+ return oldGnuSparseRecord(
+ contents,
+ path,
+ scenario.extents,
+ scenario.metadataMarker,
+ );
+ if (sparse === "0.1-hole") {
+ const hole = contents.findIndex(
+ (byte, index) => index > 0 && index % 512 === 0 && byte === 0,
+ );
+ expect(hole).toBeGreaterThan(0);
+ const stored = Buffer.concat([
+ contents.subarray(0, hole),
+ contents.subarray(hole + 1),
+ scenario.tailMarker
+ ? Buffer.from("go/synthetic-reference")
+ : Buffer.alloc(0),
+ ]);
+ return Buffer.concat([
+ tarRecord(
+ paxRecords({
+ "GNU.sparse.size": String(contents.length),
+ "GNU.sparse.numblocks": "2",
+ "GNU.sparse.map": `0,${hole},${hole + 1},${contents.length - hole - 1}`,
+ }),
+ { name: "PaxHeaders/asset", type: 0x78 },
+ ),
+ tarRecord(stored, { name: path }),
+ ]);
+ }
+ if (sparse === true) {
+ return Buffer.concat([
+ tarRecord(
+ paxRecords({
+ "GNU.sparse.size": String(contents.length),
+ "GNU.sparse.numblocks": "1",
+ "GNU.sparse.map": `0,${contents.length}`,
+ }),
+ { name: "PaxHeaders/asset", type: 0x78 },
+ ),
+ tarRecord(contents, { name: path }),
+ ]);
+ }
+ if (sparse === "0.1-tail") {
+ const storedContents = Buffer.concat([
+ contents,
+ Buffer.from("go/synthetic-reference"),
+ ]);
+ return Buffer.concat([
+ tarRecord(
+ paxRecords({
+ "GNU.sparse.size": String(storedContents.length),
+ "GNU.sparse.numblocks": "1",
+ "GNU.sparse.map": `0,${storedContents.length}`,
+ }),
+ { name: "PaxHeaders/asset", type: 0x78 },
+ ),
+ tarRecord(storedContents, { name: path }),
+ ]);
+ }
+ const map = Buffer.alloc(512);
+ map.write(`1\n0\n${contents.length}\n`);
+ if (scenario.mapMarker) map.write("go/synthetic-reference", 128);
+ const tail = scenario.tailBoundaryMarker
+ ? Buffer.alloc((512 - (contents.length % 512)) % 512 || 512)
+ : scenario.tailMarker
+ ? Buffer.from("go/synthetic-reference")
+ : Buffer.alloc(0);
+ if (scenario.tailBoundaryMarker) tail.write("go/", tail.length - 3);
+ return Buffer.concat([
+ tarRecord(
+ paxRecords({
+ "GNU.sparse.major": "1",
+ "GNU.sparse.minor": "0",
+ "GNU.sparse.name": path,
+ "GNU.sparse.realsize": String(contents.length),
+ }),
+ { name: "PaxHeaders/asset", type: 0x78 },
+ ),
+ tarRecord(Buffer.concat([map, contents, tail]), {
+ name: "package/GNUSparseFile.1/asset",
+ ...(scenario.headerMapMarker
+ ? {
+ reserved: Buffer.concat([
+ Buffer.alloc(9),
+ Buffer.from("go/"),
+ ]),
+ }
+ : {}),
+ }),
+ ]);
+ });
+ const archivePath = join(root, `${scenario.name}.tgz`);
+ writeFileSync(
+ archivePath,
+ gzipSync(
+ Buffer.concat([
+ ...(scenario.globalSparse
+ ? [
+ tarRecord(
+ paxRecords({
+ "GNU.sparse.major": "1",
+ "GNU.sparse.minor": "0",
+ }),
+ { name: "GlobalHead", type: 0x67 },
+ ),
+ ]
+ : []),
+ packageTar({ additionalRecords: records }),
+ ]),
+ ),
+ );
+ const contractPath = join(root, "contract.json");
+ writeFileSync(
+ contractPath,
+ JSON.stringify({
+ ...pluginContract,
+ shippedExact: [
+ ...pluginContract.shippedExact,
+ ...scenario.files.map(([name]) => name),
+ ],
+ }),
+ );
+ const environment = { ...process.env };
+ delete environment["CODEX_SECURITY_EXPECTED_GIT_HEAD"];
+ const result = spawnSync(
+ commandPath("node"),
+ [
+ fileURLToPath(
+ new URL("../scripts/check-package.mjs", import.meta.url),
+ ),
+ archivePath,
+ contractPath,
+ ],
+ {
+ cwd: root,
+ env: environment,
+ encoding: "utf8",
+ timeout: 30_000,
+ windowsHide: true,
+ },
+ );
+ if (scenario.error) {
+ expect({ scenario: scenario.name, status: result.status }).toEqual({
+ scenario: scenario.name,
+ status: 1,
+ });
+ if (typeof scenario.error === "string")
+ expect(result.stderr).toContain(scenario.error);
+ else expect(result.stderr).toMatch(scenario.error);
+ } else {
+ expect({
+ scenario: scenario.name,
+ status: result.status,
+ stderr: result.stderr,
+ }).toEqual({ scenario: scenario.name, status: 0, stderr: "" });
+ }
+ }
+ } finally {
+ rmSync(root, { force: true, recursive: true });
+ }
+ });
+
+ test.each([0x78, 0x67])(
+ "preserves inherited character locale for Unicode pax owners, type=%i",
+ (type) => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-owner-test-"));
+ try {
+ const archivePath = join(root, "unicode owner.tgz");
+ writeFileSync(
+ archivePath,
+ gzipSync(
+ Buffer.concat([
+ tarRecord(
+ paxRecords({
+ uname: "Synthetic é owner",
+ gname: "Synthetic é group",
+ }),
+ { name: "OwnerHead", type },
+ ),
+ packageTar(),
+ ]),
+ ),
+ );
+ const contractPath = join(root, "plugin contract.json");
+ writeFileSync(contractPath, JSON.stringify(pluginContract));
+ const environment: NodeJS.ProcessEnv = { ...process.env };
+ delete environment["CODEX_SECURITY_EXPECTED_GIT_HEAD"];
+ if (process.platform !== "win32")
+ environment["LC_ALL"] =
+ process.platform === "darwin" ? "en_US.UTF-8" : "C.UTF-8";
+ const result = spawnSync(
+ commandPath("node"),
+ [
+ fileURLToPath(
+ new URL("../scripts/check-package.mjs", import.meta.url),
+ ),
+ archivePath,
+ contractPath,
+ ],
+ {
+ cwd: root,
+ env: environment,
+ encoding: "utf8",
+ timeout: 30_000,
+ windowsHide: true,
+ },
+ );
+ expect({ status: result.status, stderr: result.stderr }).toEqual({
+ status: 0,
+ stderr: "",
+ });
+ } finally {
+ rmSync(root, { force: true, recursive: true });
+ }
+ },
+ );
+
+ test("streams each archive without resolving tar from its directory", () => {
+ const root = mkdtempSync(join(tmpdir(), "codex-package-tar-test-"));
+ try {
+ const archiveDirectory = join(
+ root,
+ process.platform === "win32" ? "package archive" : "D: package archive",
+ );
+ mkdirSync(archiveDirectory, { recursive: true });
+ const archivePath = join(
+ archiveDirectory,
+ process.platform === "win32"
+ ? "-fixture package.tgz"
+ : "-fixture: package.tgz",
+ );
+ const contractPath = join(root, "plugin contract.json");
+ const logPath = join(root, "tar calls.jsonl");
+ const adjacentTarMarker = join(root, "archive tar ran");
+ const tarBytes = packageTar({
+ compatibleLayout: true,
+ rootDirectoryMode: 0o555,
+ readmeMode: 0,
+ });
+ const archiveContents = gzipSync(tarBytes, { level: 0 });
+ writeFileSync(archivePath, archiveContents);
+ writeFileSync(contractPath, JSON.stringify(pluginContract));
+
+ const nodePath = commandPath("node");
+ const environment: NodeJS.ProcessEnv = { ...process.env };
+ delete environment["CODEX_SECURITY_EXPECTED_GIT_HEAD"];
+ environment["PATH"] = `.${delimiter}${process.env["PATH"] ?? ""}`;
+ const extractionDirectory = join(root, "extraction");
+ mkdirSync(extractionDirectory, { mode: 0o700 });
+ for (const variable of ["TMPDIR", "TMP", "TEMP"]) {
+ environment[variable] = extractionDirectory;
+ }
+ if (process.platform === "win32") {
+ for (const name of ["tar.com", "tar.exe"]) {
+ writeFileSync(join(archiveDirectory, name), "not an executable");
+ }
+ } else {
+ const adjacentTar = join(archiveDirectory, "tar");
+ writeFileSync(
+ adjacentTar,
+ `#!/usr/bin/env node
+require("node:fs").writeFileSync(process.env.ADJACENT_TAR_MARKER, "ran");
+process.exit(99);
+`,
+ );
+ chmodSync(adjacentTar, 0o755);
+
+ const proxySource = `#!/usr/bin/env node
+const { spawnSync } = require("node:child_process");
+const { createHash } = require("node:crypto");
+const { appendFileSync, readFileSync } = require("node:fs");
+const input = readFileSync(0);
+appendFileSync(
+ process.env.TAR_PROXY_LOG,
+ JSON.stringify({
+ args: process.argv.slice(2),
+ cwd: process.cwd(),
+ inputLength: input.length,
+ inputSha256: createHash("sha256").update(input).digest("hex"),
+ }) + "\\n",
+);
+const result = spawnSync(process.env.REAL_TAR, process.argv.slice(2), {
+ env: process.env,
+ input,
+ stdio: ["pipe", "inherit", "inherit"],
+ windowsHide: true,
+});
+if (result.error !== undefined) throw result.error;
+if (result.status === 0 && process.argv.includes("-xzf") && process.env.TAR_PROXY_FAIL_EXTRACT) {
+ process.stderr.write("synthetic extraction failure\\n");
+ process.exit(23);
+}
+process.exit(result.status ?? 1);
+`;
+ const proxyPath = join(root, "tar");
+ writeFileSync(proxyPath, proxySource);
+ chmodSync(proxyPath, 0o755);
+ environment["ADJACENT_TAR_MARKER"] = adjacentTarMarker;
+ environment["REAL_TAR"] = commandPath("tar");
+ environment["TAR_PROXY_LOG"] = logPath;
+ }
+
+ const result = spawnSync(
+ nodePath,
+ [
+ fileURLToPath(
+ new URL("../scripts/check-package.mjs", import.meta.url),
+ ),
+ archivePath,
+ contractPath,
+ ],
+ {
+ cwd: root,
+ encoding: "utf8",
+ env: environment,
+ timeout: 30_000,
+ windowsHide: true,
+ },
+ );
+ expect(existsSync(adjacentTarMarker)).toBe(false);
+ expect({ status: result.status, stderr: result.stderr }).toEqual({
+ status: 0,
+ stderr: "",
+ });
+ expect(readdirSync(extractionDirectory)).toEqual([]);
+ if (process.platform === "win32") return;
+
+ const calls = readFileSync(logPath, "utf8")
+ .trim()
+ .split(/\r?\n/u)
+ .map(
+ (line) =>
+ JSON.parse(line) as {
+ args: string[];
+ cwd: string;
+ inputLength: number;
+ inputSha256: string;
+ },
+ );
+ const archiveSha256 = createHash("sha256")
+ .update(archiveContents)
+ .digest("hex");
+ expect(calls).toHaveLength(3);
+ for (const call of calls) {
+ expect(call.args.filter((arg) => arg === "-")).toEqual(["-"]);
+ expect(call.args).not.toContain(archivePath);
+ expect(realpathSync(call.cwd)).toBe(realpathSync(root));
+ expect(call.inputLength).toBe(archiveContents.length);
+ expect(call.inputSha256).toBe(archiveSha256);
+ }
+ expect(calls[0]?.args).toEqual(["--ignore-zeros", "-tzf", "-"]);
+ expect(calls[1]?.args).toEqual([
+ "--ignore-zeros",
+ "--numeric-owner",
+ "-tvzf",
+ "-",
+ ]);
+ expect(calls[2]?.args.slice(0, 10)).toEqual([
+ "--ignore-zeros",
+ "-m",
+ "--keep-old-files",
+ "--no-same-owner",
+ "--no-same-permissions",
+ "--no-acls",
+ "--no-xattrs",
+ "-xzf",
+ "-",
+ "-C",
+ ]);
+
+ environment["TAR_PROXY_FAIL_EXTRACT"] = "true";
+ const failed = spawnSync(
+ nodePath,
+ [
+ fileURLToPath(
+ new URL("../scripts/check-package.mjs", import.meta.url),
+ ),
+ archivePath,
+ contractPath,
+ ],
+ {
+ cwd: root,
+ encoding: "utf8",
+ env: environment,
+ timeout: 30_000,
+ windowsHide: true,
+ },
+ );
+ expect(failed.status).not.toBe(0);
+ expect(failed.stderr).toContain("synthetic extraction failure");
+ expect(readdirSync(extractionDirectory)).toEqual([]);
+ } finally {
+ rmSync(root, { force: true, recursive: true });
+ }
+ });
});