OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(package): inspect unpacked files for internal references (#618)
Michael D'Angelo5efc3d941613
diff --git a/sdk/typescript/scripts/check-package.mjs b/sdk/typescript/scripts/check-package.mjsindex 6d2d01090..d0a28a973 100644--- a/sdk/typescript/scripts/check-package.mjs+++ b/sdk/typescript/scripts/check-package.mjs@@ -1,15 +1,31 @@import { spawnSync } from "node:child_process";-import { createHash } from "node:crypto";-import { readFileSync } from "node:fs";+import {+ chmodSync,+ lstatSync,+ mkdtempSync,+ readFileSync,+ readdirSync,+ rmSync,+} from "node:fs";+import { tmpdir } from "node:os";+import { join, resolve } from "node:path";import { fileURLToPath } from "node:url";import { gunzipSync } from "node:zlib";+import { packageDistFiles } from "./package-dist-files.mjs";import {assertPublicPackageContents,MAX_EXPANDED_ASSET_BYTES,} from "./package-public-content.mjs";import { assertExpectedGitHead } from "./package-provenance.mjs";import { packageSmokeTimeouts } from "./package-smoke-timeouts.mjs";-import { regularTarListingLines } from "./package-tar-listing.mjs";+import {+ assertStoredSparseContents,+ readTarArchive,+} from "./package-tar-entries.mjs";+import {+ assertTarListingSizes,+ regularTarListingLines,+} from "./package-tar-listing.mjs";import { pluginContractFiles } from "./plugin-contract.mjs";const PACKAGE_SMOKE_PROCESS_TIMEOUT_MS =@@ -30,12 +46,33 @@ if (archive === undefined || args.length > 2) {);}-const archiveBytes = gunzipSync(readFileSync(archive), {+const archivePath = resolve(archive);+const compressedArchive = readFileSync(archivePath);+const archiveBytes = gunzipSync(compressedArchive, {maxOutputLength: MAX_EXPANDED_ASSET_BYTES,});-const PUBLIC_LOGO_SHA256 =- "9b9c2b09b2fa064611fb62307d321d5c2ea70cf0789f7ce34cdb0fc0d9190b3a";-const tarOptions = { maxBuffer: archiveBytes.byteLength + 1024 };+const storedArchive = readTarArchive(archiveBytes);+const rawEntries = storedArchive.entries;+validatePackagePaths(rawEntries.map(({ path }) => path));+assertPublicPackageContents(+ storedArchive.files,+ Buffer.concat(storedArchive.metadata.filter(Buffer.isBuffer)),+);+const processEnvironment = { ...process.env };+delete processEnvironment.TAR_OPTIONS;+const characterLocale =+ processEnvironment.LC_ALL || processEnvironment.LC_CTYPE;+delete processEnvironment.LC_ALL;+const tarOptions = {+ env: {+ ...processEnvironment,+ LC_CTYPE: characterLocale,+ LC_MESSAGES: "C",+ LC_NUMERIC: "C",+ },+ input: compressedArchive,+ maxBuffer: archiveBytes.byteLength + 1024,+};function tar(args, encoding = "buffer") {const result = spawnSync("tar", ["--ignore-zeros", ...args], {...tarOptions,@@ -51,216 +88,91 @@ function tar(args, encoding = "buffer") {return result.stdout;}-let offset = 0;-const archiveFiles = new Map();-const archiveMetadata = [];-for (; offset + 512 <= archiveBytes.byteLength;) {- const header = archiveBytes.subarray(offset, offset + 512);- if (header.every((byte) => byte === 0)) {- archiveMetadata.push(header);- offset += 512;- continue;+function invalidTarEntry() {+ throw new Error("npm tarball contains an invalid tar entry.");+}++function validatePackagePaths(entries) {+ const files = new Set(entries);+ if (files.size !== entries.length)+ throw new Error("npm tarball contains duplicate paths.");+ const required = [+ "package/package.json",+ "package/README.md",+ "package/docs/cli.md",+ "package/docs/findings-service.md",+ "package/docs/dedupe-records.md",+ "package/LICENSE",+ "package/bin/codex-security.mjs",+ "package/dist/index.js",+ "package/dist/index.d.ts",+ "package/dist/cli.js",+ "package/schemas/project-config.schema.json",+ "package/_bundled_plugin/.codex-plugin/plugin.json",+ ];++ for (const file of required) {+ if (!files.has(file)) throw new Error(`npm tarball is missing ${file}.`);}- const name = header.subarray(0, 100).toString("utf8").split("\0", 1)[0];- const prefix = header.subarray(345, 500).toString("utf8").split("\0", 1)[0];- const path = prefix === "" ? name : `${prefix}/${name}`;- const sizeField = header- .subarray(124, 136)- .toString("ascii")- .split("\0", 1)[0]- .trim();- if (!/^[0-7]*$/u.test(sizeField)) {- throw new Error("npm tarball contains an invalid tar entry.");++ const contract = JSON.parse(readFileSync(contractPath, "utf8"));+ const pluginPaths = pluginContractFiles(contract);+ if (new Set(pluginPaths).size !== pluginPaths.length) {+ throw new Error("Plugin projection contract contains duplicate paths.");}- if (path.endsWith("/") && header[156] !== 0x35) {- throw new Error("npm tarball contains an invalid tar entry.");++ const pluginEntries = new Set();+ for (const file of pluginPaths) {+ const pluginArchivePath = `package/_bundled_plugin/${file}`;+ pluginEntries.add(pluginArchivePath);+ if (!files.has(pluginArchivePath)) {+ throw new Error(`npm tarball is missing ${pluginArchivePath}.`);+ }}- const size = Number.parseInt(sizeField || "0", 8);- const contentsStart = offset + 512;- const nextOffset = contentsStart + Math.ceil(size / 512) * 512;- if (nextOffset > archiveBytes.byteLength) {- throw new Error("npm tarball contains an invalid tar entry.");++ const distFiles = new Set(packageDistFiles);+ for (const file of distFiles) {+ if (!files.has(file)) throw new Error(`npm tarball is missing ${file}.`);}- if (header[156] === 0 || header[156] === 0x30) {- archiveFiles.set(- path,- archiveBytes.subarray(contentsStart, contentsStart + size),- );- archiveMetadata.push(- header,- archiveBytes.subarray(contentsStart + size, nextOffset),- );- } else {- archiveMetadata.push(archiveBytes.subarray(offset, nextOffset));+ const allowedFiles = new Set([...required, ...distFiles, ...pluginEntries]);+ for (const file of [...allowedFiles]) {+ const parts = file.split("/");+ for (let index = 1; index < parts.length; index++) {+ allowedFiles.add(`${parts.slice(0, index).join("/")}/`);+ }}- offset = nextOffset;-}-if (archiveBytes.subarray(offset).some((byte) => byte !== 0)) {- throw new Error("npm tarball contains trailing tar data.");-}--function archiveFile(path) {- const contents = archiveFiles.get(path);- if (contents === undefined) {- throw new Error("npm tarball contains an invalid tar entry: " + path + ".");+ const unsafePath = /(?:^|\/)\.{1,2}(?:\/|$)/u;+ for (const file of files) {+ if (+ !allowedFiles.has(file) ||+ unsafePath.test(file) ||+ file.includes("\\")+ ) {+ const displayPath = file.replace(+ /[\u0000-\u001f\u007f-\u009f]/gu,+ (character) =>+ `\\x${character.charCodeAt(0).toString(16).padStart(2, "0")}`,+ );+ throw new Error(+ `npm tarball contains an unexpected file: ${displayPath}.`,+ );+ }}- return contents;}-const entries = tar(["-tzf", archive], "utf8").split(/\r?\n/u).filter(Boolean);+const entries = tar(["-tzf", "-"], "utf8").split(/\r?\n/u).filter(Boolean);const files = new Set(entries);if (files.size !== entries.length) {throw new Error("npm tarball contains duplicate paths.");}-const required = [- "package/package.json",- "package/README.md",- "package/docs/cli.md",- "package/docs/findings-service.md",- "package/docs/dedupe-records.md",- "package/LICENSE",- "package/bin/codex-security.mjs",- "package/dist/index.js",- "package/dist/index.d.ts",- "package/dist/cli.js",- "package/schemas/project-config.schema.json",- "package/_bundled_plugin/.codex-plugin/plugin.json",-];--for (const file of required) {- if (!files.has(file)) throw new Error(`npm tarball is missing ${file}.`);-}--const contract = JSON.parse(readFileSync(contractPath, "utf8"));-const pluginPaths = pluginContractFiles(contract);-if (new Set(pluginPaths).size !== pluginPaths.length) {- throw new Error("Plugin projection contract contains duplicate paths.");-}--const allowedFiles = new Set([- ...required,- ...pluginPaths.map((file) => `package/_bundled_plugin/${file}`),- ...[- "api",- "artifact-export",- "auth",- "bulk-scan-discovery",- "cli",- "cli-help",- "cli-scan-logs-json",- "cli-signals",- "classify-severity",- "classify-scan-severity",- "severity-store",- "cloud-publish",- "codex-prompt",- "codex-sdk-environment",- "component-plan",- "component-scan",- "config",- "config-path",- "contract",- "cost",- "cost-model",- "custom-validation",- "custom-validation-prompt",- "custom-publish",- "deep-progress",- "deep-scan",- "deep-config",- "deep-scan-defaults",- "project-config",- "project-config-schema",- "prompt-files",- "provider-profile",- "scan-modes",- "scan-settings",- "errors",- "feedback",- "finding-catalogue",- "findings-import",- "github",- "index",- "import-scan",- "knowledge-base",- "linear",- "models",- "multiscan",- "mock-scan",- "owner-evidence",- "patch-tui",- "publication",- "publication-events",- "publication-store",- "publish",- "result",- "record",- "request-metadata",- "runtime",- "scan-activity",- "scan-comparison",- "scan-dashboard",- "scan-history-renderer",- "scan-inputs",- "scan-logs",- "security-policy",- "security-policy-cli",- "suggest-owners",- "scan-sessions",- "server/api",- "deduplication/codex-review",- "deduplication/checkpointed-review",- "deduplication/refusal",- "deduplication/retry",- "deduplication/deduplication",- "finding-retrieval",- "finding-workflow",- "findings-client",- "findings-errors",- "finding-dedupe-groups",- "deduplication/deduplication-prompts",- "deduplication/deduplication-reviewer",- "deduplication/diagnostics",- "deduplication/scan",- "deduplication/local",- "deduplication/finding-schema",- "deduplication/records",- "deduplication/records-protocol",- "deduplication/review",- "saved-scan",- "saved-scan-bootstrap",- "server/embeddings",- "server/errors",- "server/sqlite-store",- "server/storage",- "targets",- "thread-source",- "trusted-executable",- "value",- "version",- "windows-path",- "worker-progress",- ].flatMap((module) =>- ["js", "js.map", "d.ts", "d.ts.map"].map(- (extension) => `package/dist/${module}.${extension}`,- ),- ),-]);-for (const file of [...allowedFiles]) {- if (!files.has(file)) throw new Error(`npm tarball is missing ${file}.`);- const parts = file.split("/");- for (let index = 1; index < parts.length; index++) {- allowedFiles.add(`${parts.slice(0, index).join("/")}/`);- }-}-const unsafePath = /(?:^|\/)\.{1,2}(?:\/|$)/u;-for (const file of files) {- if (!allowedFiles.has(file) || unsafePath.test(file) || file.includes("\\")) {- throw new Error(`npm tarball contains an unexpected file: ${file}.`);- }+if (+ rawEntries.length !== entries.length ||+ rawEntries.some(({ path }, index) => path !== entries[index])+) {+ invalidTarEntry();}-const listing = tar(["-tvzf", archive], "utf8");+const listing = tar(["--numeric-owner", "-tvzf", "-"], "utf8");const listingLines = regularTarListingLines(listing);if (listingLines.length !== entries.length ||@@ -268,8 +180,12 @@ if ((line, index) => line.startsWith("d") !== entries[index].endsWith("/"),)) {- throw new Error("npm tarball contains an invalid tar entry.");+ invalidTarEntry();}+const listingSizes = assertTarListingSizes(+ listingLines,+ MAX_EXPANDED_ASSET_BYTES,+);for (const [path, name] of [["package/bin/codex-security.mjs", "CLI"],["package/_bundled_plugin/scripts/launch_codex_security_mcp", "MCP"],@@ -280,9 +196,104 @@ for (const [path, name] of [throw new Error(`npm package ${name} launcher is not executable.`);}}-const packageJson = JSON.parse(- archiveFile("package/package.json").toString("utf8"),-);++function privateExtractionDirectories(directory) {+ chmodSync(directory, 0o700);+ for (const entry of readdirSync(directory, { withFileTypes: true })) {+ if (entry.isDirectory()) {+ privateExtractionDirectories(join(directory, entry.name));+ }+ }+}++function extractedArchiveFiles() {+ const logicalSizes = new Map();+ const expectedPaths = new Map();+ for (const [index, { path }] of rawEntries.entries()) {+ const directory = path.endsWith("/");+ const extractedPath = directory ? path.slice(0, -1) : path;+ const type = directory ? "directory" : "file";+ const previousType = expectedPaths.get(extractedPath);+ if (previousType !== undefined && previousType !== type) invalidTarEntry();+ if (!directory) logicalSizes.set(path, listingSizes[index]);+ expectedPaths.set(extractedPath, type);+ const parts = extractedPath.split("/");+ for (let index = 1; index < parts.length; index++) {+ const directory = parts.slice(0, index).join("/");+ if (logicalSizes.has(directory)) invalidTarEntry();+ expectedPaths.set(directory, "directory");+ }+ }++ const extractionRoot = mkdtempSync(+ join(tmpdir(), "codex-security-package-check-"),+ );+ try {+ chmodSync(extractionRoot, 0o700);+ try {+ tar([+ "-m",+ "--keep-old-files",+ "--no-same-owner",+ "--no-same-permissions",+ "--no-acls",+ "--no-xattrs",+ "-xzf",+ "-",+ "-C",+ extractionRoot,+ ...logicalSizes.keys(),+ ]);+ } finally {+ privateExtractionDirectories(extractionRoot);+ }++ const archiveFiles = new Map();+ let expandedBytes = 0;+ function visit(directory, relative = "") {+ for (const name of readdirSync(directory)) {+ const path = relative === "" ? name : `${relative}/${name}`;+ const expectedType = expectedPaths.get(path);+ if (expectedType === undefined) invalidTarEntry();+ const extractedPath = join(extractionRoot, path);+ const stats = lstatSync(extractedPath);+ expectedPaths.delete(path);++ if (expectedType === "directory") {+ if (!stats.isDirectory()) invalidTarEntry();+ visit(extractedPath, path);+ continue;+ }++ if (+ !stats.isFile() ||+ stats.nlink !== 1 ||+ stats.size !== logicalSizes.get(path) ||+ stats.size > MAX_EXPANDED_ASSET_BYTES ||+ expandedBytes > MAX_EXPANDED_ASSET_BYTES - stats.size+ ) {+ invalidTarEntry();+ }+ expandedBytes += stats.size;+ chmodSync(extractedPath, 0o600);+ archiveFiles.set(path, readFileSync(extractedPath));+ }+ }+ visit(extractionRoot);++ if (expectedPaths.size !== 0 || archiveFiles.size !== logicalSizes.size) {+ invalidTarEntry();+ }+ return archiveFiles;+ } finally {+ rmSync(extractionRoot, { force: true, recursive: true });+ }+}++const archiveFiles = extractedArchiveFiles();+const npmManifest = storedArchive.npmFiles.get("package/package.json");+if (npmManifest === undefined) invalidTarEntry();+const packageJson = JSON.parse(npmManifest.toString("utf8"));if (packageJson.name !== "@openai/codex-security" ||packageJson.license !== "Apache-2.0"@@ -294,21 +305,24 @@ assertExpectedGitHead(process.env.CODEX_SECURITY_EXPECTED_GIT_HEAD,);-for (const file of files) {- if (/\.png$/iu.test(file)) {- const digest = createHash("sha256").update(archiveFile(file)).digest("hex");- if (digest !== PUBLIC_LOGO_SHA256) {- throw new Error(`npm tarball contains an unexpected PNG asset: ${file}.`);- }- }+assertPublicPackageContents(archiveFiles);+assertStoredSparseContents(storedArchive, archiveFiles);+for (const path of archiveFiles.keys()) {+ if (+ /\.(?:png|br(?:\.part-[0-9]+)?)$/iu.test(path) &&+ !storedArchive.npmFiles.has(path)+ )+ invalidTarEntry();}--assertPublicPackageContents(archiveFiles, Buffer.concat(archiveMetadata));+assertPublicPackageContents(storedArchive.npmFiles);if (args.length === 1) {const smoke = spawnSync(process.execPath,- [fileURLToPath(new URL("./smoke-package.mjs", import.meta.url)), archive],+ [+ fileURLToPath(new URL("./smoke-package.mjs", import.meta.url)),+ archivePath,+ ],{stdio: "inherit",timeout: PACKAGE_SMOKE_PROCESS_TIMEOUT_MS,diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjsnew file mode 100644index 000000000..ec33e1c6d--- /dev/null+++ b/sdk/typescript/scripts/package-dist-files.mjs@@ -0,0 +1,107 @@+const distModules = [+ "api",+ "artifact-export",+ "auth",+ "bulk-scan-discovery",+ "cli",+ "cli-help",+ "cli-scan-logs-json",+ "cli-signals",+ "classify-severity",+ "classify-scan-severity",+ "severity-store",+ "cloud-publish",+ "codex-prompt",+ "codex-sdk-environment",+ "component-plan",+ "component-scan",+ "config",+ "config-path",+ "contract",+ "cost",+ "cost-model",+ "custom-validation",+ "custom-validation-prompt",+ "custom-publish",+ "deep-progress",+ "deep-scan",+ "deep-config",+ "deep-scan-defaults",+ "project-config",+ "project-config-schema",+ "prompt-files",+ "provider-profile",+ "scan-modes",+ "scan-settings",+ "errors",+ "feedback",+ "finding-catalogue",+ "findings-import",+ "github",+ "index",+ "import-scan",+ "knowledge-base",+ "linear",+ "models",+ "multiscan",+ "mock-scan",+ "owner-evidence",+ "patch-tui",+ "publication",+ "publication-events",+ "publication-store",+ "publish",+ "result",+ "record",+ "request-metadata",+ "runtime",+ "scan-activity",+ "scan-comparison",+ "scan-dashboard",+ "scan-history-renderer",+ "scan-inputs",+ "scan-logs",+ "security-policy",+ "security-policy-cli",+ "suggest-owners",+ "scan-sessions",+ "server/api",+ "deduplication/codex-review",+ "deduplication/checkpointed-review",+ "deduplication/refusal",+ "deduplication/retry",+ "deduplication/deduplication",+ "finding-retrieval",+ "finding-workflow",+ "findings-client",+ "findings-errors",+ "finding-dedupe-groups",+ "deduplication/deduplication-prompts",+ "deduplication/deduplication-reviewer",+ "deduplication/diagnostics",+ "deduplication/scan",+ "deduplication/local",+ "deduplication/finding-schema",+ "deduplication/records",+ "deduplication/records-protocol",+ "deduplication/review",+ "saved-scan",+ "saved-scan-bootstrap",+ "server/embeddings",+ "server/errors",+ "server/sqlite-store",+ "server/storage",+ "targets",+ "thread-source",+ "trusted-executable",+ "value",+ "version",+ "windows-path",+ "worker-progress",+];++export const packageDistFiles = distModules.flatMap((module) =>+ ["js", "js.map", "d.ts", "d.ts.map"].map(+ (extension) => `package/dist/${module}.${extension}`,+ ),+);diff --git a/sdk/typescript/scripts/package-public-content.mjs b/sdk/typescript/scripts/package-public-content.mjsindex 2d8458b1e..2005dce34 100644--- a/sdk/typescript/scripts/package-public-content.mjs+++ b/sdk/typescript/scripts/package-public-content.mjs@@ -1,9 +1,13 @@+import { createHash } from "node:crypto";import { brotliDecompressSync } from "node:zlib";export const MAX_EXPANDED_ASSET_BYTES = 32 * 1024 * 1024;+const PUBLIC_LOGO_SHA256 =+ "9b9c2b09b2fa064611fb62307d321d5c2ea70cf0789f7ce34cdb0fc0d9190b3a";+const internalMarker =- /(?:internal\.api\.openai\.org|gateway\.[a-z0-9.-]*internal|\.openai\.org|openai\.firewall\.socket\.dev|socket\x2dfirewall\x2dregistry|openai\.(?:enterprise\.)?slack\.com|app\.slack\.com\/client|(?:app\.notion\.com\/p|notion\.so)\/openai|linear\.app\/openai|(?:github\.com[:/]|api\.github\.com\/repos\/|raw\.githubusercontent\.com\/)openai\/openai(?:\.git)?(?:[^a-z0-9_-]|$)|LicenseRef\x2dProprietary|\/Users\/|\/home\/dev-user|flow\.apps\.openai\.org|(?:^|[^a-z0-9_-])go\/[a-z0-9_-]+)/iu;+ /(?:internal\.api\.openai\.org|gateway\.[a-z0-9.-]*internal|\.openai\.org|openai\.firewall\.socket\.dev|socket\x2dfirewall\x2dregistry|openai\.(?:enterprise\.)?slack\.com|app\.slack\.com\/client|(?:app\.notion\.com\/p|notion\.so)\/openai|linear\.app\/openai|(?:github\.com[:/]|api\.github\.com\/repos\/|raw\.githubusercontent\.com\/)openai\/openai(?:\.git)?(?![a-z0-9_-])|LicenseRef\x2dProprietary|\/Users\/|\/home\/dev-user|flow\.apps\.openai\.org|(?<![a-z0-9_-])go\/[a-z0-9_-]+)/iu;/*** @param {Map<string, Buffer>} archiveFiles@@ -25,6 +29,13 @@ export function assertPublicPackageContents(compressedParts.set(name, parts);} else if (/\.br$/iu.test(path)) {assertPublicBrotli(bytes, path);+ } else if (/\.png$/iu.test(path)) {+ if (+ createHash("sha256").update(bytes).digest("hex") !== PUBLIC_LOGO_SHA256+ )+ throw new Error(+ `npm tarball contains an unexpected PNG asset: ${path}.`,+ );} else {assertPublicText(bytes.toString("utf8"));}@@ -49,8 +60,17 @@ function assertPublicBrotli(bytes, path) {assertPublicText(result.buffer.toString("utf8"));}-function assertPublicText(contents) {- if (internalMarker.test(contents)) {- throw new Error("npm tarball contains an internal reference.");+export function assertPublicText(contents, binaryRanges = []) {+ const pattern = new RegExp(internalMarker.source, "giu");+ let match;+ while ((match = pattern.exec(contents)) !== null) {+ if (+ !binaryRanges.some(+ ({ start, end }) =>+ match.index >= start && match.index + match[0].length <= end,+ )+ )+ throw new Error("npm tarball contains an internal reference.");+ pattern.lastIndex = match.index + 1;}}diff --git a/sdk/typescript/scripts/package-tar-entries.mjs b/sdk/typescript/scripts/package-tar-entries.mjsnew file mode 100644index 000000000..5b6e81ade--- /dev/null+++ b/sdk/typescript/scripts/package-tar-entries.mjs@@ -0,0 +1,299 @@+import { assertPublicText } from "./package-public-content.mjs";++const blockSize = 512;+function invalidTarEntry() {+ throw new Error("npm tarball contains an invalid tar entry.");+}++function headerText(header, start, end) {+ return header.subarray(start, end).toString("utf8").split("\0", 1)[0];+}++function octalNumber(header, start, end) {+ const field = headerText(header, start, end).trim();+ if (!/^[0-7]*$/u.test(field)) invalidTarEntry();+ return Number.parseInt(field || "0", 8);+}++function paxAttributes(contents) {+ const attributes = new Map();+ let offset = 0;+ while (offset < contents.byteLength) {+ const separator = contents.indexOf(0x20, offset);+ const lengthField = contents.subarray(offset, separator).toString("ascii");+ if (separator === -1 || !/^[0-9]+$/u.test(lengthField)) invalidTarEntry();+ const end = offset + Number(lengthField);+ const equals = contents.indexOf(0x3d, separator + 1);+ if (+ end > contents.byteLength ||+ equals <= separator + 1 ||+ equals >= end - 1 ||+ contents[end - 1] !== 0x0a+ )+ invalidTarEntry();+ attributes.set(+ contents.subarray(separator + 1, equals).toString("utf8"),+ contents.subarray(equals + 1, end - 1).toString("utf8"),+ );+ offset = end;+ }+ return attributes;+}++function sparseMap(contents, paxMap) {+ if (paxMap) {+ const values = paxMap.split(",").map(Number);+ const extents = [];+ for (let index = 0; index < values.length; index += 2)+ extents.push({ offset: values[index], size: values[index + 1] });+ return { contents, extents, dataOffset: 0 };+ }+ let offset = 0;+ function number() {+ while (offset < contents.length) {+ const end = contents.indexOf(0x0a, offset);+ if (end === -1) invalidTarEntry();+ const line = contents.subarray(offset, end).toString("ascii");+ offset = end + 1;+ if (line.startsWith("#")) continue;+ if (!/^[0-9]*$/u.test(line)) invalidTarEntry();+ return Number(line);+ }+ invalidTarEntry();+ }+ const count = number();+ const extents = [];+ for (let index = 0; index < count; index++) {+ extents.push({ offset: number(), size: number() });+ }+ const dataOffset = Math.ceil(offset / blockSize) * blockSize;+ if (dataOffset > contents.length) invalidTarEntry();+ return { contents, extents, dataOffset };+}++export function assertStoredSparseContents(archive, extractedFiles) {+ const sparseMetadata = new Map();+ for (const [path, { contents, extents, dataOffset }] of archive.sparseFiles) {+ const extracted = extractedFiles.get(path);+ let metadata;+ // GNU tar pads stored extents; libarchive also accepts packed extents.+ for (const padded of [false, true]) {+ let offset = dataOffset;+ const parts = [contents.subarray(0, dataOffset)];+ const matches = extents.every((extent, index) => {+ const end = offset + extent.size;+ if (+ !contents+ .subarray(offset, end)+ .equals(+ extracted.subarray(extent.offset, extent.offset + extent.size),+ )+ )+ return false;+ const next =+ padded && index < extents.length - 1+ ? Math.ceil(end / blockSize) * blockSize+ : end;+ parts.push(+ { binary: contents.subarray(offset, end) },+ contents.subarray(end, next),+ );+ offset = next;+ return true;+ });+ if (matches) {+ parts.push(contents.subarray(offset));+ metadata = parts;+ break;+ }+ }+ if (metadata === undefined) invalidTarEntry();+ sparseMetadata.set(path, metadata);+ }+ const parts = archive.metadata.flatMap((part) =>+ typeof part === "string" ? sparseMetadata.get(part) : part,+ );+ const binaryRanges = [];+ let text = "";+ for (const part of parts) {+ const binary = !Buffer.isBuffer(part);+ const decoded = (binary ? part.binary : part).toString("utf8");+ if (binary) {+ const previous = binaryRanges.at(-1);+ if (previous?.end === text.length) previous.end += decoded.length;+ else+ binaryRanges.push({+ start: text.length,+ end: text.length + decoded.length,+ });+ }+ text += decoded;+ }+ assertPublicText(text, binaryRanges);+}++export function readTarArchive(archiveBytes) {+ const entries = [];+ const archiveFiles = new Map();+ const archiveMetadata = [];+ const sparseFiles = new Map();+ const npmFiles = new Map();+ let offset = 0;+ const globalAttributes = new Map();+ const nextAttributes = new Map();+ let nextName;+ let nextNpmPath;++ while (offset + blockSize <= archiveBytes.byteLength) {+ const header = archiveBytes.subarray(offset, offset + blockSize);+ if (header.every((byte) => byte === 0)) {+ archiveMetadata.push(header);+ offset += blockSize;+ continue;+ }++ const signature = header.subarray(257, 265).toString("latin1");+ const directory = header[156] === 0x35;+ const oldSparse = header[156] === 0x53;+ const longName = header[156] === 0x4c;+ const extended = header[156] === 0x78 || header[156] === 0x67 || longName;+ if (+ (header[156] !== 0 &&+ header[156] !== 0x30 &&+ !directory &&+ !extended &&+ !oldSparse) ||+ (signature !== "ustar\0" + "00" &&+ signature !== "ustar \0" &&+ signature !== "\0".repeat(8))+ ) {+ invalidTarEntry();+ }++ const name = headerText(header, 0, 100);+ // GNU headers use this area for timestamps and sparse-file metadata.+ const prefix =+ signature === "ustar\0" + "00" ? headerText(header, 345, 500) : "";+ const attribute = (key) =>+ nextAttributes.has(key)+ ? nextAttributes.get(key) || undefined+ : globalAttributes.get(key);+ const headerPath = prefix === "" ? name : `${prefix}/${name}`;+ const path =+ attribute("GNU.sparse.name") ??+ attribute("path") ??+ nextName ??+ headerPath;+ if (!extended && (path === "" || path.endsWith("/") !== directory))+ invalidTarEntry();+ assertPublicText(path);++ const headerSize = octalNumber(header, 124, 136);+ const paxSize = extended ? undefined : attribute("size");+ if (paxSize !== undefined && !/^[0-9]+$/u.test(paxSize)) invalidTarEntry();+ const size = paxSize === undefined ? headerSize : Number(paxSize);+ let contentsStart = offset + blockSize;+ const oldSparseExtents = [];+ if (oldSparse) {+ let map = header;+ let start = 386;+ let count = 4;+ for (;;) {+ let index = 0;+ for (; index < count; index++) {+ const field = start + index * 24;+ if (map[field + 12] === 0) break;+ const offset = octalNumber(map, field, field + 12);+ const size = octalNumber(map, field + 12, field + 24);+ if (offset !== 0 || size !== 0)+ oldSparseExtents.push({ offset, size });+ }+ if (index < count || map[start + count * 24] === 0) break;+ map = archiveBytes.subarray(contentsStart, contentsStart + blockSize);+ if (map.length !== blockSize) invalidTarEntry();+ contentsStart += blockSize;+ start = 0;+ count = 21;+ }+ }+ const contentsEnd = contentsStart + size;+ const nextOffset = contentsStart + Math.ceil(size / blockSize) * blockSize;+ if (nextOffset > archiveBytes.byteLength || (directory && size !== 0)) {+ invalidTarEntry();+ }++ if (extended) {+ const contents = archiveBytes.subarray(contentsStart, contentsEnd);+ archiveMetadata.push(archiveBytes.subarray(offset, nextOffset));+ if (longName) {+ nextName = headerText(contents, 0, contents.length);+ nextNpmPath = nextName;+ } else {+ const destination =+ header[156] === 0x67 ? globalAttributes : nextAttributes;+ for (const [key, value] of paxAttributes(contents)) {+ // npm merges local PAX paths and GNU long names in record order.+ if (destination === nextAttributes && key === "path")+ nextNpmPath = value || undefined;+ if (destination === globalAttributes && value === "")+ destination.delete(key);+ else destination.set(key, value);+ }+ }+ } else {+ if (directory)+ archiveMetadata.push(archiveBytes.subarray(offset, nextOffset));+ else {+ const contents = archiveBytes.subarray(contentsStart, contentsEnd);+ // npm ignores old-GNU sparse entries and GNU sparse name overrides.+ const npmPath = oldSparse ? undefined : (nextNpmPath ?? headerPath);+ if (npmPath !== undefined) npmFiles.set(npmPath, contents);+ archiveMetadata.push(archiveBytes.subarray(offset, contentsStart));+ const paxSparseMap = nextAttributes.get("GNU.sparse.map");+ if (+ (oldSparse ||+ paxSparseMap ||+ Number.parseInt(nextAttributes.get("GNU.sparse.major"), 10) ===+ 1) &&+ /\.(?:png|br(?:\.part-[0-9]+)?)$/iu.test(path)+ ) {+ sparseFiles.set(+ path,+ oldSparse+ ? { contents, extents: oldSparseExtents, dataOffset: 0 }+ : sparseMap(contents, paxSparseMap),+ );+ // Retain sparse framing in order with the surrounding headers and padding.+ archiveMetadata.push(path);+ } else {+ archiveFiles.set(path, contents);+ if (oldSparse) archiveMetadata.push(contents);+ }+ archiveMetadata.push(archiveBytes.subarray(contentsEnd, nextOffset));+ }+ entries.push({ path, size });+ nextAttributes.clear();+ nextName = undefined;+ nextNpmPath = undefined;+ }+ offset = nextOffset;+ }++ if (archiveBytes.subarray(offset).some((byte) => byte !== 0))+ invalidTarEntry();+ const deferredStreams = new Set(+ [...sparseFiles.keys()].map((path) => path.replace(/\.part-[0-9]+$/iu, "")),+ );+ for (const path of archiveFiles.keys()) {+ if (deferredStreams.has(path.replace(/\.part-[0-9]+$/iu, ""))) {+ archiveFiles.delete(path);+ }+ }+ return {+ entries,+ files: archiveFiles,+ metadata: archiveMetadata,+ sparseFiles,+ npmFiles,+ };+}diff --git a/sdk/typescript/scripts/package-tar-listing.mjs b/sdk/typescript/scripts/package-tar-listing.mjsindex d30d31782..6a0cac2b9 100644--- a/sdk/typescript/scripts/package-tar-listing.mjs+++ b/sdk/typescript/scripts/package-tar-listing.mjs@@ -8,3 +8,24 @@ export function regularTarListingLines(listing) {}return lines;}++export function assertTarListingSizes(lines, maximum) {+ let expandedBytes = 0;+ const sizes = [];+ for (const line of lines) {+ if (line.startsWith("d")) {+ sizes.push(0);+ continue;+ }+ // GNU combines numeric owners; BSD lists the link count, UID and GID.+ const match = /^\S+\s+(?:\d+\/\d+\s+|\d+\s+\d+\s+\d+\s+)(\d+)\s/u.exec(+ line,+ );+ const size = match === null ? NaN : Number(match[1]);+ if (!Number.isSafeInteger(size) || size > maximum - expandedBytes)+ throw new Error("npm tarball contains an invalid tar entry.");+ expandedBytes += size;+ sizes.push(size);+ }+ return sizes;+}diff --git a/sdk/typescript/tests-ts/package-public-content.test.ts b/sdk/typescript/tests-ts/package-public-content.test.tsindex 8100d34bc..44733c02e 100644--- a/sdk/typescript/tests-ts/package-public-content.test.ts+++ b/sdk/typescript/tests-ts/package-public-content.test.ts@@ -1,13 +1,10 @@+import { readFileSync } from "node:fs";import { brotliCompressSync, brotliDecompressSync } from "node:zlib";import { describe, expect, test } from "bun:test";import { assertPublicPackageContents } from "../scripts/package-public-content.mjs";-// Synthetic alphanumeric text whose compressed bytes happen to contain " Go/w".-const cleanCompressedPayload = Buffer.from(- "G/8DAGRwm7EiGn34sGFNRM5DuwB8/2zECGTx7hk1/DY6puqEJWJumAz3HfjBg7bx5FxwfFCafaN2of0Oi2uTSSE+wAYOJS4Ii53urIa5BOIkTbR5t2YJ4/UM6dWzXcEvk92TZNz/Y7hNVxjrsXKemVnN4i1pUOdQiCYQNLU7Dbh77pXyrGIWSl+X+L7DUfFR+4Rz3GF85xCbj1NuwlDfIWj42+ueic5NDX4bPXmp46fE8bTdpAUmMvt48x99pm2wlp+oKNvvcb64GT4i7F9zCJMIdfmc06UCn7Go4jQ1WI2P+e05ZbpsWoTkPXnAFULsyUtefeZ1d5nbvE2CDXV3eF7Hxc2KgtQQv8j/CIEd+9vG3REfGcJVIVMqjo1SKzOfADpWQYh3p18YFMyF6R4qOYzPB+zjBy/ZQiCaNm1LpVYdJ7XL0wXPjAVOzJU4zeSUvNFqQ0TeXFGat6ikPEii43FRFSx3aQi2HnWI3rBd1tts0LlTjIWUq8+agHu/mwmJ+jWxAF74Yvij4++fUTHxm8PvqXOK+3QpR82rAPZ+pyRgKX1Z9k9XHlVwNXzJCd5laKwKOWnAOF6vKnrqhZz4aw1VqEJ+1x/4+AM++Pw58PdlRKBjKoLl7DKgEBCGKsd5FF8XPQz9pC3MWj1UbdVa/jLl5M1iIfl9P44zgKV1TaG1LEjuS09t7e0UFSrgXI0BWvBuyNfEras635PJBiZuI8mX6s5yt5NkocDkATyMNP1BnW+wKyTzduIFbd2yV45t/t6ttd5y5FeAAKnT0Jjsmk3zbYdl12fpKKDUvs363EVzTjWEhNaoTC3ntUZKNlYzX74UMDogR28vd8vvJniJvfamegrjMcf3fviFr9caYVQH+bq/TpxK078VtU0Z5iah0E2+ttjzWXHi//ftzs9e+xArq7zohSgVrMpgsL2kjfl/zaltHiNmTgogtQI6gRiI1ICa3kimcK2a/gj0cTkCVwKywUt6xu1N2x2lPi7dk/pnx8/iSkdyphyFA4OQov+d8maitV0XwU9/NKTW048C",- "base64",-);+import { cleanCompressedPayload } from "./package-tar-fixtures.js";function compressedFiles(bytes: Buffer, split: boolean): Map<string, Buffer> {if (!split) return new Map([["package/runtime.mjs.br", bytes]]);@@ -59,6 +56,31 @@ describe("npm package public contents", () => {},);+ test("checks the approved PNG digest before exempting binary contents", () => {+ const approved = readFileSync(+ new URL(+ "../../../plugins/codex-security/assets/logo.png",+ import.meta.url,+ ),+ );+ expect(() =>+ assertPublicPackageContents(new Map([["package/logo.png", approved]])),+ ).not.toThrow();+ const stored = Buffer.from(approved);+ expect(stored.subarray(171, 178)).toEqual(Buffer.alloc(7));+ stored.set(Buffer.from("\0go/x\0\0"), 171);+ expect(() =>+ assertPublicPackageContents(new Map([["package/logo.png", stored]])),+ ).toThrow("npm tarball contains an unexpected PNG asset");+ });++ test("keeps the expanded Brotli size bound", () => {+ const bytes = brotliCompressSync(Buffer.alloc(32 * 1024 * 1024 + 1));+ expect(() =>+ assertPublicPackageContents(compressedFiles(bytes, false)),+ ).toThrow();+ });+test("checks tar metadata", () => {expect(() =>assertPublicPackageContents(new Map(), Buffer.from("go/synthetic-owner")),diff --git a/sdk/typescript/tests-ts/package-tar-entries.test.ts b/sdk/typescript/tests-ts/package-tar-entries.test.tsnew file mode 100644index 000000000..f1f790b70--- /dev/null+++ b/sdk/typescript/tests-ts/package-tar-entries.test.ts@@ -0,0 +1,632 @@+import { readFileSync } from "node:fs";+import { describe, expect, test } from "bun:test";+import {+ archive,+ blockSize,+ cleanCompressedPayload,+ octal,+ paxRecords,+ tarRecord,+} from "./package-tar-fixtures.js";++type PlainTarEntry = {+ path: string;+ size: number;+};++type PackageTarEntries = {+ readTarArchive: (archiveBytes: Buffer) => {+ entries: PlainTarEntry[];+ files: Map<string, Buffer>;+ metadata: (Buffer | string)[];+ };+ assertStoredSparseContents: (+ archive: ReturnType<PackageTarEntries["readTarArchive"]>,+ files: Map<string, Buffer>,+ ) => void;+};++const { readTarArchive, assertStoredSparseContents } = (await import(+ new URL("../scripts/package-tar-entries.mjs", import.meta.url).href+)) as PackageTarEntries;+const { assertPublicPackageContents } = (await import(+ new URL("../scripts/package-public-content.mjs", import.meta.url).href+)) as {+ assertPublicPackageContents: (+ files: Map<string, Buffer>,+ metadata: Buffer,+ ) => void;+};++function plainTarEntries(bytes: Buffer): PlainTarEntry[] {+ const archive = readTarArchive(bytes);+ assertPublicPackageContents(+ archive.files,+ Buffer.concat(+ archive.metadata.filter((part): part is Buffer => Buffer.isBuffer(part)),+ ),+ );+ return archive.entries;+}++const invalidTarEntryError = "npm tarball contains an invalid tar entry.";+const internalReferenceError = "npm tarball contains an internal reference.";++describe("plain npm tar entries", () => {+ test.each([" ", " \0", "\0"])(+ "accepts package size fields ending in %j",+ (terminator) => {+ expect(+ plainTarEntries(+ archive(+ tarRecord(Buffer.from("readme"), {+ name: "package/README.md",+ sizeField: octal(6, 12, terminator),+ }),+ ),+ ),+ ).toEqual([{ path: "package/README.md", size: 6 }]);+ },+ );++ test.each([0, 0x30])(+ "accepts regular typeflag %i and prefix paths",+ (type) => {+ const prefix = `package/${"nested/".repeat(13)}deep`;+ const longPath = `${prefix}/README.md`;+ expect(longPath.length).toBeGreaterThan(100);++ const bytes = archive(+ tarRecord(Buffer.from("license"), { name: "package/LICENSE", type }),+ tarRecord(Buffer.from("readme"), { name: "README.md", prefix, type }),+ );++ expect(plainTarEntries(bytes)).toEqual([+ { path: "package/LICENSE", size: 7 },+ { path: longPath, size: 6 },+ ]);+ },+ );++ test("rejects unsupported entry types and malformed extended headers", () => {+ for (const type of [0x31, 0x32, 0x33, 0x34, 0x36, 0x44, 0x4b, 0x67, 0x78]) {+ expect(() =>+ plainTarEntries(+ archive(+ tarRecord(Buffer.from("x"), {+ name: "package/README.md",+ type,+ }),+ ),+ ),+ ).toThrow(invalidTarEntryError);+ }+ });++ test("uses GNU long names for one member and scans their complete payload", () => {+ const path = `package/${"nested/".repeat(16)}README.md`;+ const records = (tail: string) =>+ archive(+ tarRecord(Buffer.from(`${path}\0${tail}`), {+ name: "././@LongLink",+ type: 0x4c,+ }),+ tarRecord(Buffer.from("readme"), { name: "placeholder" }),+ tarRecord(Buffer.from("license"), { name: "package/LICENSE" }),+ );+ expect(plainTarEntries(records(""))).toEqual([+ { path, size: 6 },+ { path: "package/LICENSE", size: 7 },+ ]);+ expect(() => plainTarEntries(records("go/example"))).toThrow(+ internalReferenceError,+ );+ });++ test.each([0x78, 0x67])("accepts POSIX pax metadata typeflag %i", (type) => {+ const attributes = paxRecords({ ctime: "0.123456789" });+ expect(+ plainTarEntries(+ archive(+ tarRecord(attributes, { name: "package/PaxHeaders/README.md", type }),+ tarRecord(Buffer.from("readme"), { name: "package/README.md" }),+ ),+ ),+ ).toEqual([{ path: "package/README.md", size: 6 }]);+ });++ test("uses local pax paths and sizes and clears them after their member", () => {+ const path = `package/${"nested/".repeat(16)}README.md`;+ expect(+ plainTarEntries(+ archive(+ tarRecord(paxRecords({ path, size: "6" }), {+ name: "package/PaxHeaders/README.md",+ type: 0x78,+ }),+ tarRecord(Buffer.from("readme"), {+ name: "placeholder",+ sizeField: octal(0, 12),+ }),+ tarRecord(Buffer.from("license"), { name: "package/LICENSE" }),+ ),+ ),+ ).toEqual([+ { path, size: 6 },+ { path: "package/LICENSE", size: 7 },+ ]);+ });++ test("retains global pax attributes and honors local overrides", () => {+ expect(+ plainTarEntries(+ archive(+ tarRecord(paxRecords({ path: "package/README.md" }), {+ name: "GlobalHead",+ type: 0x67,+ }),+ tarRecord(Buffer.from("readme"), { name: "placeholder" }),+ tarRecord(paxRecords({ path: "package/LICENSE" }), {+ name: "package/PaxHeaders/LICENSE",+ type: 0x78,+ }),+ tarRecord(Buffer.from("license"), { name: "placeholder" }),+ ),+ ),+ ).toEqual([+ { path: "package/README.md", size: 6 },+ { path: "package/LICENSE", size: 7 },+ ]);+ });++ test("scans the complete pax metadata payload", () => {+ expect(() =>+ plainTarEntries(+ archive(+ tarRecord(paxRecords({ comment: "go/example" }), {+ name: "package/PaxHeaders/README.md",+ type: 0x78,+ }),+ tarRecord(Buffer.from("readme"), { name: "package/README.md" }),+ ),+ ),+ ).toThrow(internalReferenceError);+ });++ test.each([false, true])(+ "scans discarded sparse-map padding, marker=%j",+ (marker) => {+ const map = Buffer.alloc(blockSize);+ map.write("1\n512\n512\n");+ if (marker) map.write("go/example", 32);+ const bytes = archive(+ tarRecord(+ paxRecords({+ "GNU.sparse.major": "1",+ "GNU.sparse.minor": "0",+ "GNU.sparse.name": "package/README.md",+ "GNU.sparse.realsize": "1024",+ }),+ { name: "package/PaxHeaders/README.md", type: 0x78 },+ ),+ tarRecord(Buffer.concat([map, Buffer.alloc(blockSize)]), {+ name: "package/README.md",+ }),+ );+ if (marker)+ expect(() => plainTarEntries(bytes)).toThrow(internalReferenceError);+ else+ expect(plainTarEntries(bytes)).toEqual([+ { path: "package/README.md", size: 1024 },+ ]);+ },+ );++ for (const format of [+ "gnu-01",+ "gnu-nul",+ "bsd-comments",+ "bsd-solaris",+ ] as const) {+ test.each([false, true])(+ `scans all stored sparse bytes for ${format}, marker=%j`,+ (marker) => {+ let attributes: Record<string, string>;+ let contents: Buffer;+ if (format === "gnu-01") {+ attributes = {+ "GNU.sparse.size": "1024",+ "GNU.sparse.numblocks": "1",+ "GNU.sparse.map": "512,512",+ };+ contents = Buffer.alloc(1024);+ if (marker) contents.write("go/example", 512);+ } else if (format === "bsd-solaris") {+ attributes = { "SUN.holesdata": " 512 1024" };+ contents = Buffer.alloc(1024);+ if (marker) contents.write("go/example", 0);+ } else {+ attributes = {+ "GNU.sparse.major": format === "gnu-nul" ? "1\0" : "1",+ "GNU.sparse.minor": "0",+ "GNU.sparse.name": "package/README.md",+ "GNU.sparse.realsize": format === "bsd-comments" ? "1536" : "1024",+ };+ contents = Buffer.alloc(format === "bsd-comments" ? 1536 : 1024);+ if (format === "bsd-comments") {+ const comments = `${`# ${"x".repeat(60)}\n`.repeat(9)}# ${marker ? "go/example" : "public note"}\n1\n1024\n512\n`;+ contents.write(comments);+ } else {+ contents.write("1\n512\n512\n");+ if (marker) contents.write("go/example", 32);+ }+ }+ const bytes = archive(+ tarRecord(paxRecords(attributes), {+ name: "package/PaxHeaders/README.md",+ type: 0x78,+ }),+ tarRecord(contents, { name: "package/README.md" }),+ );+ if (marker)+ expect(() => plainTarEntries(bytes)).toThrow(internalReferenceError);+ else+ expect(plainTarEntries(bytes)).toEqual([+ { path: "package/README.md", size: contents.length },+ ]);+ },+ );+ }++ test("checks stored old GNU sparse bodies", () => {+ const contents = Buffer.alloc(1024);+ contents.write("go/example");+ const record = tarRecord(contents, {+ name: "package/README.md",+ magic: "ustar ",+ version: " \0",+ });+ for (const offset of [386, 410]) {+ octal(0, 12).copy(record, offset);+ octal(512, 12).copy(record, offset + 12);+ }+ octal(1024, 12).copy(record, 483);+ expect(() => plainTarEntries(archive(record))).toThrow(+ internalReferenceError,+ );+ });++ test("checks stored bytes after an empty repeated sparse attribute", () => {+ const contents = Buffer.alloc(1024);+ contents.write("go/example", 512);+ const attributes = Buffer.concat([+ paxRecords({ "GNU.sparse.map": "512,512" }),+ paxRecords({ "GNU.sparse.map": "" }),+ ]);+ expect(() =>+ plainTarEntries(+ archive(+ tarRecord(attributes, { name: "PaxHeaders/readme", type: 0x78 }),+ tarRecord(contents, { name: "package/README.md" }),+ ),+ ),+ ).toThrow(internalReferenceError);+ });++ test("checks markers across concatenated archive metadata", () => {+ const record = tarRecord(Buffer.from("Public contents."), {+ name: "package/README.md",+ });+ record.write("go/", record.length - 3);+ expect(() =>+ plainTarEntries(+ archive(+ record,+ tarRecord(paxRecords({ uid: "0" }), { name: "example", type: 0x78 }),+ ),+ ),+ ).toThrow(internalReferenceError);+ });++ test.each([false, true])(+ "keeps public Brotli content semantics across retained sparse extents, split=%j",+ (split) => {+ const path = "package/runtime.mjs.br";+ const size = cleanCompressedPayload.length;+ const middle = cleanCompressedPayload.indexOf("Go/w") + 2;+ expect(middle).toBeGreaterThan(2);+ const bytes = archive(+ tarRecord(+ paxRecords({+ "GNU.sparse.size": String(size),+ "GNU.sparse.numblocks": split ? "2" : "1",+ "GNU.sparse.map": split+ ? `0,${middle},${middle},${size - middle}`+ : `0,${size}`,+ }),+ { name: "PaxHeaders/runtime", type: 0x78 },+ ),+ tarRecord(cleanCompressedPayload, { name: path }),+ );+ expect(plainTarEntries(bytes)).toEqual([{ path, size }]);+ expect(() =>+ assertStoredSparseContents(+ readTarArchive(bytes),+ new Map([[path, cleanCompressedPayload]]),+ ),+ ).not.toThrow();+ },+ );++ test("keeps retained extents between separate sparse metadata segments", () => {+ const path = "package/logo.png";+ const logo = readFileSync(+ new URL(+ "../../../plugins/codex-security/assets/logo.png",+ import.meta.url,+ ),+ );+ const firstPadding = Buffer.alloc(512 - 250);+ firstPadding.write(".openai.", firstPadding.length - 8);+ const secondPadding = Buffer.alloc(512 - 250);+ secondPadding.write("org");+ const bytes = archive(+ tarRecord(+ paxRecords({+ "GNU.sparse.size": String(logo.length),+ "GNU.sparse.numblocks": "3",+ "GNU.sparse.map": `0,250,250,250,500,${logo.length - 500}`,+ }),+ { name: "PaxHeaders/logo", type: 0x78 },+ ),+ tarRecord(+ Buffer.concat([+ logo.subarray(0, 250),+ firstPadding,+ logo.subarray(250, 500),+ secondPadding,+ logo.subarray(500),+ ]),+ { name: path },+ ),+ );+ expect(() =>+ assertStoredSparseContents(+ readTarArchive(bytes),+ new Map([[path, logo]]),+ ),+ ).not.toThrow();+ });++ test("checks markers crossing discarded padding and retained sparse bytes", () => {+ const path = "package/logo.png";+ const logo = readFileSync(+ new URL(+ "../../../plugins/codex-security/assets/logo.png",+ import.meta.url,+ ),+ );+ expect(logo.subarray(387, 390).toString()).toBe("org");+ const padding = Buffer.alloc(512 - 387);+ padding.write(".openai.", padding.length - 8);+ const bytes = archive(+ tarRecord(+ paxRecords({+ "GNU.sparse.size": String(logo.length),+ "GNU.sparse.numblocks": "2",+ "GNU.sparse.map": `0,387,387,${logo.length - 387}`,+ }),+ { name: "PaxHeaders/logo", type: 0x78 },+ ),+ tarRecord(+ Buffer.concat([logo.subarray(0, 387), padding, logo.subarray(387)]),+ { name: path },+ ),+ );+ expect(() =>+ assertStoredSparseContents(+ readTarArchive(bytes),+ new Map([[path, logo]]),+ ),+ ).toThrow(internalReferenceError);+ });++ test("uses the native GNU sparse name while retaining stored content", () => {+ const path = "package/README.md";+ const stored = Buffer.alloc(1024);+ stored.write("1\n512\n512\n");+ stored.fill(0x78, 512);+ expect(+ plainTarEntries(+ archive(+ tarRecord(+ paxRecords({+ "GNU.sparse.major": "1",+ "GNU.sparse.minor": "0",+ "GNU.sparse.name": path,+ "GNU.sparse.realsize": "1024",+ }),+ { name: "PaxHeaders/readme", type: 0x78 },+ ),+ tarRecord(stored, { name: "package/GNUSparseFile.1/readme" }),+ ),+ ),+ ).toEqual([{ path, size: stored.byteLength }]);+ });++ test("accepts an empty size field for an empty file", () => {+ expect(+ plainTarEntries(+ archive(+ tarRecord(Buffer.alloc(0), {+ name: "package/README.md",+ sizeField: Buffer.alloc(12),+ }),+ ),+ ),+ ).toEqual([{ path: "package/README.md", size: 0 }]);+ });++ test("rejects invalid or binary size encodings", () => {+ const base256 = Buffer.alloc(12);+ base256[0] = 0x80;+ base256[11] = 1;+ for (const sizeField of [base256, Buffer.from("00000000008\0")]) {+ expect(() =>+ plainTarEntries(+ archive(+ tarRecord(Buffer.from("x"), {+ name: "package/README.md",+ sizeField,+ }),+ ),+ ),+ ).toThrow(invalidTarEntryError);+ }+ });++ test.each([0, 0x30])("accepts basic V7 regular typeflag %i", (type) => {+ expect(+ plainTarEntries(+ archive(+ tarRecord(Buffer.from("readme"), {+ name: "package/README.md",+ type,+ magic: "\0".repeat(6),+ version: "\0\0",+ }),+ ),+ ),+ ).toEqual([{ path: "package/README.md", size: 6 }]);+ });++ test("rejects alternate ustar signatures", () => {+ for (const options of [+ { magic: "ustar " },+ { magic: "ustar\0", version: " \0" },+ ]) {+ expect(() =>+ plainTarEntries(+ archive(+ tarRecord(Buffer.from("x"), {+ name: "package/README.md",+ ...options,+ }),+ ),+ ),+ ).toThrow(invalidTarEntryError);+ }+ });++ test("scans complete header text fields", () => {+ expect(() =>+ plainTarEntries(+ archive(+ tarRecord(Buffer.from("clean"), {+ name: "package/README.md",+ user: "public\0go/example",+ }),+ ),+ ),+ ).toThrow(internalReferenceError);+ });++ test("scans complete raw headers", () => {+ for (const options of [+ {+ deviceNumbers: Buffer.concat([+ Buffer.from("go/example"),+ Buffer.alloc(6),+ ]),+ },+ { reserved: Buffer.from("go/example") },+ ]) {+ expect(() =>+ plainTarEntries(+ archive(+ tarRecord(Buffer.from("clean"), {+ name: "package/README.md",+ ...options,+ }),+ ),+ ),+ ).toThrow(internalReferenceError);+ }+ });++ test("accepts explicit empty directory members", () => {+ expect(+ plainTarEntries(+ archive(tarRecord(Buffer.alloc(0), { name: "package/", type: 0x35 })),+ ),+ ).toEqual([{ path: "package/", size: 0 }]);+ expect(() =>+ plainTarEntries(+ archive(tarRecord(Buffer.from("x"), { name: "package/", type: 0x35 })),+ ),+ ).toThrow(invalidTarEntryError);+ });++ test("accepts GNU headers without treating timestamps as a path prefix", () => {+ expect(+ plainTarEntries(+ archive(+ tarRecord(Buffer.from("readme"), {+ name: "package/README.md",+ magic: "ustar ",+ version: " \0",+ prefix: "00000000000",+ }),+ ),+ ),+ ).toEqual([{ path: "package/README.md", size: 6 }]);+ });++ test("accepts padding, optional end markers, and zero blocks between files", () => {+ const record = tarRecord(Buffer.from("x"), { name: "package/README.md" });+ record[record.length - 1] = 1;+ const secondRecord = tarRecord(Buffer.from("y"), {+ name: "package/LICENSE",+ });++ expect(plainTarEntries(record)).toEqual([+ { path: "package/README.md", size: 1 },+ ]);+ for (const trailingBytes of [1, 511]) {+ expect(+ plainTarEntries(+ Buffer.concat([archive(record), Buffer.alloc(trailingBytes)]),+ ),+ ).toEqual([{ path: "package/README.md", size: 1 }]);+ }+ for (const zeroBlocks of [1, 2]) {+ expect(+ plainTarEntries(+ Buffer.concat([+ record,+ Buffer.alloc(blockSize * zeroBlocks),+ secondRecord,+ ]),+ ),+ ).toEqual([+ { path: "package/README.md", size: 1 },+ { path: "package/LICENSE", size: 1 },+ ]);+ }+ });++ test("still scans padding and rejects partial blocks or truncated contents", () => {+ const record = tarRecord(Buffer.from("x"), { name: "package/README.md" });+ const paddingMarker = Buffer.from(record);+ paddingMarker.write("go/example", blockSize + 1);+ expect(() => plainTarEntries(paddingMarker)).toThrow(+ internalReferenceError,+ );+ for (const bytes of [+ Buffer.concat([archive(record), Buffer.from([1])]),+ record.subarray(0, blockSize),+ ]) {+ expect(() => plainTarEntries(bytes)).toThrow(invalidTarEntryError);+ }+ });+});diff --git a/sdk/typescript/tests-ts/package-tar-fixtures.ts b/sdk/typescript/tests-ts/package-tar-fixtures.tsnew file mode 100644index 000000000..550e1534b--- /dev/null+++ b/sdk/typescript/tests-ts/package-tar-fixtures.ts@@ -0,0 +1,145 @@+export const blockSize = 512;++export function octal(value: number, width: number, terminator = "\0"): Buffer {+ return Buffer.from(+ value.toString(8).padStart(width - terminator.length, "0") + terminator,+ );+}++function tarHeader({+ name,+ prefix = "",+ size = 0,+ mtime = 0,+ mode = 0o644,+ type = 0x30,+ sizeField = octal(size, 12, " "),+ magic = "ustar\0",+ version = "00",+ user = "",+ deviceNumbers = Buffer.alloc(16),+ reserved = Buffer.alloc(12),+}: {+ name: string;+ prefix?: string;+ size?: number;+ mtime?: number;+ mode?: number;+ type?: number;+ sizeField?: Buffer;+ magic?: string;+ version?: string;+ user?: string;+ deviceNumbers?: Buffer;+ reserved?: Buffer;+}): Buffer {+ const header = Buffer.alloc(blockSize);+ header.write(name, 0, 100, "utf8");+ octal(mode, 8).copy(header, 100);+ octal(0, 8).copy(header, 108);+ octal(0, 8).copy(header, 116);+ sizeField.copy(header, 124);+ octal(mtime, 12).copy(header, 136);+ header.fill(0x20, 148, 156);+ header[156] = type;+ header.write(magic, 257, "binary");+ header.write(version, 263, "binary");+ header.write(user, 265, 32, "utf8");+ deviceNumbers.copy(header, 329, 0, 16);+ header.write(prefix, 345, 155, "utf8");+ reserved.copy(header, 500, 0, 12);+ const checksum = header.reduce((sum, byte) => sum + byte, 0);+ Buffer.from(checksum.toString(8).padStart(6, "0") + "\0 ").copy(header, 148);+ return header;+}++export function tarRecord(+ contents: Buffer,+ options: Omit<Parameters<typeof tarHeader>[0], "size">,+): Buffer {+ return Buffer.concat([+ tarHeader({ ...options, size: contents.length }),+ contents,+ Buffer.alloc(+ Math.ceil(contents.length / blockSize) * blockSize - contents.length,+ ),+ ]);+}++export function oldGnuSparseRecord(+ contents: Buffer,+ name: string,+ extents = [{ offset: 0, size: contents.length }],+ metadataMarker?: "continuation" | "boundary" | "unused",+): Buffer {+ const stored = Buffer.concat(+ extents.flatMap(({ offset, size }, index) => [+ contents.subarray(offset, offset + size),+ Buffer.alloc(index < extents.length - 1 ? (512 - (size % 512)) % 512 : 0),+ ]),+ );+ const record = tarRecord(stored, {+ name,+ type: 0x53,+ magic: "ustar ",+ version: " \0",+ });+ const header = record.subarray(0, blockSize);+ const continuations = [];+ let remaining = extents;+ let map = header;+ let start = 386;+ let count = 4;+ do {+ for (const [index, extent] of remaining.slice(0, count).entries()) {+ octal(extent.offset, 12).copy(map, start + index * 24);+ octal(extent.size, 12).copy(map, start + index * 24 + 12);+ }+ remaining = remaining.slice(count);+ map[start + count * 24] = remaining.length > 0 ? 1 : 0;+ if (remaining.length > 0) {+ map = Buffer.alloc(blockSize);+ continuations.push(map);+ start = 0;+ count = 21;+ }+ } while (remaining.length > 0);+ if (metadataMarker === "unused") {+ const used =+ extents.length <= 4 ? extents.length : ((extents.length - 5) % 21) + 1;+ map.write("public", start + (used + 1) * 24);+ }+ octal(contents.length, 12).copy(header, 483);+ if (metadataMarker === "continuation")+ continuations.at(-1)!.write("go/test", 505);+ if (metadataMarker === "boundary") header.write("go/", 509);+ header.fill(0x20, 148, 156);+ octal(+ header.reduce((sum, byte) => sum + byte, 0),+ 8,+ "\0 ",+ ).copy(header, 148);+ return Buffer.concat([header, ...continuations, record.subarray(blockSize)]);+}++export function archive(...records: Buffer[]): Buffer {+ return Buffer.concat([...records, Buffer.alloc(blockSize * 2)]);+}++export function paxRecords(attributes: Record<string, string>): Buffer {+ return Buffer.concat(+ Object.entries(attributes).map(([key, value]) => {+ const record = ` ${key}=${value}\n`;+ let length = Buffer.byteLength(record) + 1;+ while (length !== Buffer.byteLength(record) + String(length).length)+ length = Buffer.byteLength(record) + String(length).length;+ return Buffer.from(`${length}${record}`);+ }),+ );+}++// Synthetic alphanumeric text whose compressed bytes happen to contain " Go/w".+export const cleanCompressedPayload = Buffer.from(+ "G/8DAGRwm7EiGn34sGFNRM5DuwB8/2zECGTx7hk1/DY6puqEJWJumAz3HfjBg7bx5FxwfFCafaN2of0Oi2uTSSE+wAYOJS4Ii53urIa5BOIkTbR5t2YJ4/UM6dWzXcEvk92TZNz/Y7hNVxjrsXKemVnN4i1pUOdQiCYQNLU7Dbh77pXyrGIWSl+X+L7DUfFR+4Rz3GF85xCbj1NuwlDfIWj42+ueic5NDX4bPXmp46fE8bTdpAUmMvt48x99pm2wlp+oKNvvcb64GT4i7F9zCJMIdfmc06UCn7Go4jQ1WI2P+e05ZbpsWoTkPXnAFULsyUtefeZ1d5nbvE2CDXV3eF7Hxc2KgtQQv8j/CIEd+9vG3REfGcJVIVMqjo1SKzOfADpWQYh3p18YFMyF6R4qOYzPB+zjBy/ZQiCaNm1LpVYdJ7XL0wXPjAVOzJU4zeSUvNFqQ0TeXFGat6ikPEii43FRFSx3aQi2HnWI3rBd1tts0LlTjIWUq8+agHu/mwmJ+jWxAF74Yvij4++fUTHxm8PvqXOK+3QpR82rAPZ+pyRgKX1Z9k9XHlVwNXzJCd5laKwKOWnAOF6vKnrqhZz4aw1VqEJ+1x/4+AM++Pw58PdlRKBjKoLl7DKgEBCGKsd5FF8XPQz9pC3MWj1UbdVa/jLl5M1iIfl9P44zgKV1TaG1LEjuS09t7e0UFSrgXI0BWvBuyNfEras635PJBiZuI8mX6s5yt5NkocDkATyMNP1BnW+wKyTzduIFbd2yV45t/t6ttd5y5FeAAKnT0Jjsmk3zbYdl12fpKKDUvs363EVzTjWEhNaoTC3ntUZKNlYzX74UMDogR28vd8vvJniJvfamegrjMcf3fviFr9caYVQH+bq/TpxK078VtU0Z5iah0E2+ttjzWXHi//ftzs9e+xArq7zohSgVrMpgsL2kjfl/zaltHiNmTgogtQI6gRiI1ICa3kimcK2a/gj0cTkCVwKywUt6xu1N2x2lPi7dk/pnx8/iSkdyphyFA4OQov+d8maitV0XwU9/NKTW048C",+ "base64",+);diff --git a/sdk/typescript/tests-ts/package-tar-listing.test.ts b/sdk/typescript/tests-ts/package-tar-listing.test.tsindex 4c0e585e5..11ba4ef2a 100644--- a/sdk/typescript/tests-ts/package-tar-listing.test.ts+++ b/sdk/typescript/tests-ts/package-tar-listing.test.ts@@ -1,6 +1,244 @@+import { spawnSync } from "node:child_process";+import { createHash } from "node:crypto";+import {+ chmodSync,+ existsSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ readdirSync,+ realpathSync,+ rmSync,+ writeFileSync,+} from "node:fs";+import { tmpdir } from "node:os";+import { delimiter, join } from "node:path";+import { fileURLToPath } from "node:url";+import { brotliCompressSync, brotliDecompressSync, gzipSync } from "node:zlib";import { describe, expect, test } from "bun:test";+import {+ archive,+ cleanCompressedPayload,+ octal,+ oldGnuSparseRecord,+ paxRecords,+ tarRecord,+} from "./package-tar-fixtures.js";-import { regularTarListingLines } from "../scripts/package-tar-listing.mjs";+const { assertTarListingSizes, regularTarListingLines } = (await import(+ new URL("../scripts/package-tar-listing.mjs", import.meta.url).href+)) as {+ regularTarListingLines: (listing: string) => string[];+ assertTarListingSizes: (lines: string[], maximum: number) => number[];+};+const { packageDistFiles } = (await import(+ new URL("../scripts/package-dist-files.mjs", import.meta.url).href+)) as { packageDistFiles: readonly string[] };+const pluginContract = {+ externalOwnedExact: [".codex-plugin/plugin.json"],+ shippedExact: ["scripts/launch_codex_security_mcp"],+};++function packageTar({+ trailingZeroBytes = 0,+ sizeTerminator = " ",+ type = 0x30,+ compatibleLayout = false,+ rootDirectoryMode = 0o755,+ directoryMode = 0o755,+ readmeMode = 0o644,+ mtime = 0,+ readmeSparse,+ readmeLongName = false,+ additionalRecords = [],+ overrides = new Map(),+}: {+ trailingZeroBytes?: number;+ sizeTerminator?: string;+ type?: number;+ compatibleLayout?: boolean;+ rootDirectoryMode?: number;+ directoryMode?: number;+ readmeMode?: number;+ mtime?: number;+ readmeSparse?: "0.0" | "1.0";+ readmeLongName?: boolean;+ additionalRecords?: Buffer[];+ overrides?: ReadonlyMap<string, Buffer>;+} = {}): Buffer {+ const executablePaths = [+ "package/bin/codex-security.mjs",+ "package/_bundled_plugin/scripts/launch_codex_security_mcp",+ ];+ const paths = [+ "package/package.json",+ "package/README.md",+ "package/docs/cli.md",+ "package/docs/findings-service.md",+ "package/docs/dedupe-records.md",+ "package/schemas/project-config.schema.json",+ "package/LICENSE",+ ...executablePaths,+ ...packageDistFiles,+ "package/_bundled_plugin/.codex-plugin/plugin.json",+ ];+ const records = paths.map((path) => {+ const override = overrides.get(path);+ if (override !== undefined) return override;+ if (path === "package/README.md" && readmeSparse !== undefined) {+ const contents = Buffer.alloc(512, 0x78);+ const map = Buffer.alloc(512);+ map.write("1\n512\n512\n");+ const attributes: Record<string, string> =+ readmeSparse === "0.0"+ ? {+ "GNU.sparse.size": "1024",+ "GNU.sparse.numblocks": "1",+ "GNU.sparse.map": "512,512",+ }+ : {+ "GNU.sparse.major": "1",+ "GNU.sparse.minor": "0",+ "GNU.sparse.name": path,+ "GNU.sparse.realsize": "1024",+ };+ return Buffer.concat([+ tarRecord(paxRecords(attributes), {+ name: "PaxHeaders/readme",+ type: 0x78,+ }),+ tarRecord(+ readmeSparse === "0.0" ? contents : Buffer.concat([map, contents]),+ {+ name:+ readmeSparse === "0.0" ? path : "package/GNUSparseFile.1/readme",+ },+ ),+ ]);+ }+ const contents =+ path === "package/package.json"+ ? Buffer.from(+ JSON.stringify({+ license: "Apache-2.0",+ name: "@openai/codex-security",+ }),+ )+ : path.endsWith(".json") || path.endsWith(".map")+ ? Buffer.from("{}\n")+ : Buffer.from("fixture\n");+ const record = tarRecord(contents, {+ name: path,+ type,+ mtime,+ mode: executablePaths.includes(path)+ ? 0o755+ : path === "package/README.md"+ ? readmeMode+ : 0o644,+ sizeField: octal(contents.length, 12, sizeTerminator),+ ...(compatibleLayout ? { magic: "ustar ", version: " \0" } : {}),+ });+ if (compatibleLayout) record[record.length - 1] = 1;+ if (path === "package/README.md" && readmeLongName)+ return Buffer.concat([+ tarRecord(Buffer.from(`${path}\0`), {+ name: "././@LongLink",+ type: 0x4c,+ }),+ record,+ ]);+ return record;+ });+ records.push(...additionalRecords);+ if (compatibleLayout) {+ const directories = new Set<string>();+ for (const path of paths) {+ const parts = path.split("/");+ for (let index = 1; index < parts.length; index++) {+ directories.add(`${parts.slice(0, index).join("/")}/`);+ }+ }+ return Buffer.concat([+ ...[...directories].map((name) =>+ tarRecord(Buffer.alloc(0), {+ name,+ type: 0x35,+ mode: name === "package/" ? rootDirectoryMode : directoryMode,+ }),+ ),+ ...records.flatMap((record) => [Buffer.alloc(512), record]),+ ]);+ }+ return archive(...records, Buffer.alloc(trailingZeroBytes));+}++function commandPath(command: string): string {+ const lookup = spawnSync(+ process.platform === "win32" ? "where.exe" : "which",+ [command],+ { encoding: "utf8", windowsHide: true },+ );+ if (lookup.error !== undefined) throw lookup.error;+ const path = lookup.stdout.split(/\r?\n/u).find(Boolean);+ if (lookup.status !== 0 || path === undefined) {+ throw new Error(`Could not resolve ${command}.`);+ }+ return path;+}++function installPackage(+ root: string,+ archivePath: string,+ environment: NodeJS.ProcessEnv,+) {+ const consumer = join(root, "consumer");+ mkdirSync(consumer);+ writeFileSync(+ join(consumer, "package.json"),+ JSON.stringify({ name: "synthetic-consumer", private: true }),+ );+ const resolved = spawnSync(+ commandPath("node"),+ [+ "--input-type=module",+ "-e",+ `import { resolveNpm } from ${JSON.stringify(new URL("../scripts/package-smoke-npm.mjs", import.meta.url).href)}; console.log(JSON.stringify(await resolveNpm()));`,+ ],+ { encoding: "utf8", env: environment, windowsHide: true },+ );+ expect({ status: resolved.status, stderr: resolved.stderr }).toEqual({+ status: 0,+ stderr: "",+ });+ const npm = JSON.parse(resolved.stdout) as {+ command: string;+ args: string[];+ };+ return spawnSync(+ npm.command,+ [+ ...npm.args,+ "install",+ "--global=false",+ "--prefix",+ consumer,+ "--offline",+ "--ignore-scripts",+ "--package-lock=false",+ "--no-audit",+ "--no-fund",+ archivePath,+ ],+ {+ cwd: consumer,+ env: environment,+ encoding: "utf8",+ timeout: 30_000,+ windowsHide: true,+ },+ );+}describe("npm package tar listings", () => {test("accepts regular entries with Unix or Windows line endings", () => {@@ -17,9 +255,1252 @@ describe("npm package tar listings", () => {]);});+ test.each([+ "-rw-r--r-- 0/0 33554433 1970-01-01 00:00 package/README.md",+ "-rw-r--r-- 0 0 0 33554433 Jan 1 1970 package/README.md",+ ])("bounds native sparse logical sizes before extraction: %s", (line) => {+ expect(() => assertTarListingSizes([line], 32 * 1024 * 1024)).toThrow(+ "npm tarball contains an invalid tar entry",+ );+ const bounded = line.replace("33554433", "33554432");+ expect(() =>+ assertTarListingSizes([bounded], 32 * 1024 * 1024),+ ).not.toThrow();+ expect(() =>+ assertTarListingSizes([bounded, bounded], 32 * 1024 * 1024),+ ).toThrow();+ });++ test("retains bounded logical sparse sizes in listing order", () => {+ expect(+ assertTarListingSizes(+ [+ "drwxr-xr-x 0/0 0 1970-01-01 00:00 package/",+ "-rw-r--r-- 0/0 1024 1970-01-01 00:00 package/README.md",+ "-rw-r--r-- 0 0 0 512 Jan 1 1970 package/LICENSE",+ ],+ 1536,+ ),+ ).toEqual([0, 1024, 512]);+ });+test("rejects symbolic links and other non-regular entries", () => {expect(() =>regularTarListingLines("lrwxrwxrwx package/link -> target\r\n"),).toThrow("npm tarball contains a non-regular entry");});++ test.each([false, true])(+ "rejects invalid package paths before Brotli expansion, complete=%p",+ (complete) => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-path-order-"));+ try {+ const archivePath = join(root, "unexpected-brotli.tgz");+ writeFileSync(+ archivePath,+ gzipSync(+ Buffer.concat([+ ...(complete ? [packageTar()] : []),+ tarRecord(Buffer.from("Malformed compressed bytes."), {+ name: "package/unexpected.br",+ }),+ ]),+ ),+ );+ const contractPath = join(root, "plugin contract.json");+ writeFileSync(contractPath, JSON.stringify(pluginContract));+ const result = spawnSync(+ commandPath("node"),+ [+ fileURLToPath(+ new URL("../scripts/check-package.mjs", import.meta.url),+ ),+ archivePath,+ contractPath,+ ],+ { cwd: root, encoding: "utf8", timeout: 30_000, windowsHide: true },+ );+ expect(result.status).toBe(1);+ expect(result.stderr).toContain(+ complete+ ? "unexpected file: package/unexpected.br"+ : "missing package/package.json",+ );+ } finally {+ rmSync(root, { force: true, recursive: true });+ }+ },+ );++ test("escapes terminal controls in unexpected archive path diagnostics", () => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-terminal-path-"));+ try {+ const archivePath = join(root, "unexpected-path.tgz");+ writeFileSync(+ archivePath,+ gzipSync(+ Buffer.concat([+ packageTar(),+ tarRecord(Buffer.from("fixture"), {+ name: "package/unexpected\u001b]52;c;synthetic\u0007\u009b",+ }),+ ]),+ ),+ );+ const contractPath = join(root, "plugin contract.json");+ writeFileSync(contractPath, JSON.stringify(pluginContract));+ const result = spawnSync(+ commandPath("node"),+ [+ fileURLToPath(+ new URL("../scripts/check-package.mjs", import.meta.url),+ ),+ archivePath,+ contractPath,+ ],+ { cwd: root, encoding: "utf8", timeout: 30_000, windowsHide: true },+ );+ expect(result.status).toBe(1);+ expect(result.stderr).toContain("unexpected file: package/unexpected");+ expect(result.stderr).not.toMatch(/[\u001b\u0007\u009b]/u);+ expect(result.stderr).toContain("synthetic");+ } finally {+ rmSync(root, { force: true, recursive: true });+ }+ });++ test("accepts equivalent bounded gzip representations", () => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-gzip-test-"));+ try {+ const tarBytes = packageTar({ trailingZeroBytes: 31 * 1024 * 1024 });+ const archives = [+ ["default", gzipSync(tarBytes)],+ ["level-0", gzipSync(tarBytes, { level: 0 })],+ ["sparse-0.0", gzipSync(packageTar({ readmeSparse: "0.0" }))],+ ["sparse-1.0", gzipSync(packageTar({ readmeSparse: "1.0" }))],+ ["gnu-long-name", gzipSync(packageTar({ readmeLongName: true }))],+ ["npm-size-field", gzipSync(packageTar({ sizeTerminator: " \0" }))],+ ["nul-regular-file", gzipSync(packageTar({ type: 0 }))],+ [+ "future-timestamps",+ gzipSync(+ packageTar({ mtime: Math.floor(Date.now() / 1000) + 365 * 86400 }),+ ),+ ],+ ["unreadable-readme", gzipSync(packageTar({ readmeMode: 0 }))],+ [+ "read-only-nested-directories",+ gzipSync(+ packageTar({ compatibleLayout: true, directoryMode: 0o555 }),+ ),+ ],+ ["posix-size-field", gzipSync(packageTar({ sizeTerminator: "\0" }))],+ [+ "compatible-tar-layout",+ gzipSync(packageTar({ compatibleLayout: true })),+ ],+ [+ "read-only-directories",+ gzipSync(+ packageTar({ compatibleLayout: true, rootDirectoryMode: 0o555 }),+ ),+ ],+ ] as const;+ expect(archives[0][1].length).toBeLessThan(1024 * 1024);+ expect(archives[1][1].length).toBeGreaterThan(31 * 1024 * 1024);++ const contractPath = join(root, "plugin contract.json");+ writeFileSync(contractPath, JSON.stringify(pluginContract));+ const environment: NodeJS.ProcessEnv = { ...process.env };+ delete environment["CODEX_SECURITY_EXPECTED_GIT_HEAD"];+ for (const [representation, contents] of archives) {+ const archivePath = join(root, `${representation}.tgz`);+ writeFileSync(archivePath, contents);+ const result = spawnSync(+ commandPath("node"),+ [+ fileURLToPath(+ new URL("../scripts/check-package.mjs", import.meta.url),+ ),+ archivePath,+ contractPath,+ ],+ {+ cwd: root,+ encoding: "utf8",+ env: environment,+ timeout: 30_000,+ windowsHide: true,+ },+ );+ expect({+ representation,+ status: result.status,+ stderr: result.stderr,+ }).toEqual({ representation, status: 0, stderr: "" });+ }+ } finally {+ rmSync(root, { force: true, recursive: true });+ }+ });++ test.each(["environment", "user-config"])(+ "keeps npm fixture installs local with global %s settings",+ (configuration) => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-npm-local-"));+ try {+ const archivePath = join(root, "package.tgz");+ writeFileSync(archivePath, gzipSync(packageTar()));+ const userConfig = join(root, "npmrc");+ writeFileSync(+ userConfig,+ configuration === "user-config" ? "global=true\n" : "",+ );+ const environment: NodeJS.ProcessEnv = { ...process.env };+ for (const key of Object.keys(environment)) {+ if (/^npm_config_(?:global|prefix|userconfig|cache)$/iu.test(key))+ delete environment[key];+ }+ const globalPrefix = join(root, "global-prefix");+ Object.assign(environment, {+ npm_config_cache: join(root, "npm-cache"),+ npm_config_prefix: globalPrefix,+ npm_config_userconfig: userConfig,+ ...(configuration === "environment"+ ? { npm_config_global: "true" }+ : {}),+ });+ const installed = installPackage(root, archivePath, environment);+ expect({+ status: installed.status,+ stderr: installed.status === 0 ? "" : installed.stderr,+ }).toEqual({+ status: 0,+ stderr: "",+ });+ expect(+ existsSync(+ join(+ root,+ "consumer",+ "node_modules",+ "@openai",+ "codex-security",+ "package.json",+ ),+ ),+ ).toBe(true);+ expect(existsSync(globalPrefix)).toBe(false);+ } finally {+ rmSync(root, { force: true, recursive: true });+ }+ },+ );++ test.each([+ "pax-full",+ "pax-map-prefix",+ "pax-hole",+ "oldgnu",+ "sparse-name",+ "pax-path",+ "global-path",+ "pax-then-long-name",+ "long-name-then-pax",+ "pax-then-wrong-long-name",+ "overwrite-text",+ "overwrite-logo",+ "ordinary-pax-then-wrong-long-name",+ "ordinary-pax-then-correct-long-name",+ "ordinary-long-name-then-pax",+ ])("validates archive assets as npm installs them: %s", (representation) => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-npm-sparse-"));+ try {+ const logo = readFileSync(+ new URL(+ "../../../plugins/codex-security/assets/logo.png",+ import.meta.url,+ ),+ );+ const path = "package/_bundled_plugin/logo.png";+ const renamed =+ representation === "sparse-name" || representation === "pax-path";+ const wrongPath = "package/synthetic-storage.png";+ const ordinary = representation.startsWith("ordinary-");+ const pathOrder = ordinary+ ? representation.slice("ordinary-".length)+ : representation;+ const wrongLongName = pathOrder === "pax-then-wrong-long-name";+ const orderedPath = representation.includes("then");+ const overwrite = representation.startsWith("overwrite");+ let stored =+ representation === "overwrite-text" ? Buffer.from("fixture\n") : logo;+ const attributes: Record<string, string> = ordinary+ ? {}+ : {+ "GNU.sparse.size": String(stored.length),+ "GNU.sparse.numblocks": "1",+ "GNU.sparse.map": `0,${stored.length}`,+ };+ if (representation === "pax-map-prefix") {+ const map = Buffer.alloc(512);+ map.write(`1\n0\n${logo.length}\n`);+ stored = Buffer.concat([map, logo]);+ for (const key of Object.keys(attributes)) delete attributes[key];+ Object.assign(attributes, {+ "GNU.sparse.major": "1",+ "GNU.sparse.minor": "0",+ "GNU.sparse.name": path,+ "GNU.sparse.realsize": String(logo.length),+ });+ } else if (representation === "pax-hole") {+ const hole = logo.findIndex(+ (byte, index) => index > 0 && index % 512 === 0 && byte === 0,+ );+ expect(hole).toBeGreaterThan(0);+ stored = Buffer.concat([+ logo.subarray(0, hole),+ logo.subarray(hole + 1),+ ]);+ attributes["GNU.sparse.numblocks"] = "2";+ attributes["GNU.sparse.map"] =+ `0,${hole},${hole + 1},${logo.length - hole - 1}`;+ }+ if (+ !ordinary &&+ (renamed || orderedPath || representation === "global-path")+ )+ attributes["GNU.sparse.name"] = path;+ if (overwrite) attributes["GNU.sparse.name"] = "package/README.md";+ if (representation === "pax-path" || orderedPath)+ attributes["path"] =+ representation === "pax-then-long-name" ? wrongPath : path;+ const longName = tarRecord(+ Buffer.from(+ `${representation === "pax-then-long-name" || representation === "ordinary-pax-then-correct-long-name" || pathOrder === "long-name-then-pax" ? path : wrongPath}\0`,+ ),+ { name: "././@LongLink", type: 0x4c },+ );+ const record =+ representation === "oldgnu"+ ? oldGnuSparseRecord(logo, path)+ : Buffer.concat([+ ...(representation === "global-path"+ ? [+ tarRecord(paxRecords({ path: wrongPath }), {+ name: "GlobalHead",+ type: 0x67,+ }),+ ]+ : []),+ ...(pathOrder === "long-name-then-pax" ? [longName] : []),+ tarRecord(paxRecords(attributes), {+ name: "PaxHeaders/asset",+ type: 0x78,+ }),+ ...(pathOrder.startsWith("pax-then") ? [longName] : []),+ tarRecord(stored, {+ name: renamed || ordinary ? wrongPath : path,+ }),+ ]);+ const archivePath = join(root, "package.tgz");+ writeFileSync(+ archivePath,+ gzipSync(+ packageTar({+ overrides: overwrite+ ? new Map([["package/README.md", Buffer.alloc(0)]])+ : undefined,+ additionalRecords: overwrite+ ? [tarRecord(logo, { name: path }), record]+ : [record],+ }),+ ),+ );+ const nativeRoot = join(root, "native");+ mkdirSync(nativeRoot);+ const extracted = spawnSync(+ "tar",+ ["-xzf", archivePath, "-C", nativeRoot],+ { encoding: "buffer", timeout: 30_000, windowsHide: true },+ );+ expect({+ status: extracted.status,+ stderr: extracted.stderr.toString(),+ }).toEqual({ status: 0, stderr: "" });+ // These incompatible packages do not promise a portable native asset path.+ if (!wrongLongName)+ expect(readFileSync(join(nativeRoot, path)).equals(logo)).toBe(true);+ const contractPath = join(root, "contract.json");+ writeFileSync(+ contractPath,+ JSON.stringify({+ ...pluginContract,+ shippedExact: [...pluginContract.shippedExact, "logo.png"],+ }),+ );+ const environment: NodeJS.ProcessEnv = {+ ...process.env,+ npm_config_cache: join(root, "npm-cache"),+ };+ delete environment["CODEX_SECURITY_EXPECTED_GIT_HEAD"];+ const installed = installPackage(root, archivePath, environment);+ const consumer = join(root, "consumer");+ expect({+ status: installed.status,+ stderr: installed.status === 0 ? "" : installed.stderr,+ }).toEqual({ status: 0, stderr: "" });+ const installedPackage = join(+ consumer,+ "node_modules",+ "@openai",+ "codex-security",+ );+ const installedLogo = join(+ installedPackage,+ "_bundled_plugin",+ "logo.png",+ );+ if (wrongLongName) {+ expect(existsSync(installedLogo)).toBe(false);+ expect(+ readFileSync(+ join(installedPackage, wrongPath.slice("package/".length)),+ ).equals(logo),+ ).toBe(true);+ }+ const compatible = [+ "pax-full",+ "pax-path",+ "global-path",+ "pax-then-long-name",+ "long-name-then-pax",+ "overwrite-logo",+ "ordinary-pax-then-correct-long-name",+ "ordinary-long-name-then-pax",+ ].includes(representation);+ expect(+ existsSync(installedLogo) && readFileSync(installedLogo).equals(logo),+ ).toBe(compatible);+ if (overwrite)+ expect(readFileSync(installedLogo).equals(stored)).toBe(true);+ const checked = spawnSync(+ commandPath("node"),+ [+ fileURLToPath(+ new URL("../scripts/check-package.mjs", import.meta.url),+ ),+ archivePath,+ contractPath,+ ],+ {+ cwd: root,+ env: environment,+ encoding: "utf8",+ timeout: 30_000,+ windowsHide: true,+ },+ );+ expect({+ status: checked.status,+ stderr: compatible ? checked.stderr : "",+ }).toEqual({ status: compatible ? 0 : 1, stderr: "" });+ } finally {+ rmSync(root, { force: true, recursive: true });+ }+ });++ test.each([+ "oldgnu-manifest",+ "pax-map-prefix",+ "overwrite-json",+ "overwrite-metadata",+ "overwrite-git-head",+ "pax-full",+ "pax-path",+ "oldgnu-readme",+ ])("validates the manifest npm installs: %s", (representation) => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-npm-manifest-"));+ try {+ const path = "package/package.json";+ const gitHead = "a".repeat(40);+ const metadata = {+ name: "@openai/codex-security",+ version: "1.0.0",+ license: "Apache-2.0",+ gitHead,+ };+ const manifest = Buffer.from(JSON.stringify(metadata));+ const overrides = new Map([[path, tarRecord(manifest, { name: path })]]);+ if (representation.startsWith("oldgnu")) {+ const oldPath =+ representation === "oldgnu-manifest" ? path : "package/README.md";+ overrides.set(+ oldPath,+ oldGnuSparseRecord(+ oldPath === path ? manifest : Buffer.from("fixture\n"),+ oldPath,+ ),+ );+ } else {+ const overwrite = representation.startsWith("overwrite");+ const logicalPath = overwrite ? "package/README.md" : path;+ let stored = overwrite+ ? Buffer.from(+ representation === "overwrite-json"+ ? "fixture\n"+ : JSON.stringify({+ ...metadata,+ ...(representation === "overwrite-metadata"+ ? { license: "MIT" }+ : { gitHead: "b".repeat(40) }),+ }),+ )+ : manifest;+ const attributes: Record<string, string> = {+ "GNU.sparse.size": String(stored.length),+ "GNU.sparse.numblocks": "1",+ "GNU.sparse.map": `0,${stored.length}`,+ "GNU.sparse.name": logicalPath,+ };+ if (representation === "pax-map-prefix") {+ const map = Buffer.alloc(512);+ map.write(`1\n0\n${stored.length}\n`);+ for (const key of Object.keys(attributes)) delete attributes[key];+ Object.assign(attributes, {+ "GNU.sparse.major": "1",+ "GNU.sparse.minor": "0",+ "GNU.sparse.name": path,+ "GNU.sparse.realsize": String(stored.length),+ });+ stored = Buffer.concat([map, stored]);+ }+ if (representation === "pax-path") attributes["path"] = path;+ overrides.set(+ logicalPath,+ Buffer.concat([+ tarRecord(paxRecords(attributes), {+ name: "PaxHeaders/manifest",+ type: 0x78,+ }),+ tarRecord(stored, {+ name:+ representation === "pax-path"+ ? "package/synthetic-storage.json"+ : path,+ }),+ ]),+ );+ }+ const archivePath = join(root, "package.tgz");+ writeFileSync(archivePath, gzipSync(packageTar({ overrides })));+ const extracted = spawnSync("tar", ["-xOzf", archivePath, path], {+ encoding: "buffer",+ timeout: 30_000,+ windowsHide: true,+ });+ expect({+ status: extracted.status,+ stderr: extracted.stderr.toString(),+ }).toEqual({ status: 0, stderr: "" });+ expect(extracted.stdout.equals(manifest)).toBe(true);+ const environment: NodeJS.ProcessEnv = {+ ...process.env,+ npm_config_cache: join(root, "npm-cache"),+ CODEX_SECURITY_EXPECTED_GIT_HEAD: gitHead,+ };+ const installed = installPackage(root, archivePath, environment);+ const invalidJson = [+ "oldgnu-manifest",+ "pax-map-prefix",+ "overwrite-json",+ ].includes(representation);+ expect(installed.error).toBeUndefined();+ if (invalidJson) expect(installed.status).not.toBe(0);+ else {+ expect({+ status: installed.status,+ stderr: installed.status === 0 ? "" : installed.stderr,+ }).toEqual({ status: 0, stderr: "" });+ const actual = JSON.parse(+ readFileSync(+ join(+ root,+ "consumer",+ "node_modules",+ "@openai",+ "codex-security",+ "package.json",+ ),+ "utf8",+ ),+ );+ expect(actual.license).toBe(+ representation === "overwrite-metadata" ? "MIT" : metadata.license,+ );+ expect(actual.gitHead).toBe(+ representation === "overwrite-git-head" ? "b".repeat(40) : gitHead,+ );+ }+ const compatible = ["pax-full", "pax-path", "oldgnu-readme"].includes(+ representation,+ );+ const contractPath = join(root, "contract.json");+ writeFileSync(contractPath, JSON.stringify(pluginContract));+ const checked = spawnSync(+ commandPath("node"),+ [+ fileURLToPath(+ new URL("../scripts/check-package.mjs", import.meta.url),+ ),+ archivePath,+ contractPath,+ ],+ {+ cwd: root,+ env: environment,+ encoding: "utf8",+ timeout: 30_000,+ windowsHide: true,+ },+ );+ expect({+ status: checked.status,+ stderr: compatible ? checked.stderr : "",+ }).toEqual({ status: compatible ? 0 : 1, stderr: "" });+ if (representation === "overwrite-metadata")+ expect(checked.stderr).toContain("expected public metadata");+ if (representation === "overwrite-git-head")+ expect(checked.stderr).toContain("gitHead must match release commit");+ } finally {+ rmSync(root, { force: true, recursive: true });+ }+ });++ test("checks reconstructed sparse binary assets and their discarded metadata", () => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-sparse-assets-"));+ try {+ const logo = readFileSync(+ new URL(+ "../../../plugins/codex-security/assets/logo.png",+ import.meta.url,+ ),+ );+ const oldGnuText = Buffer.alloc(32 * 1024);+ const oldGnuExtents = Array.from({ length: 32 }, (_, index) => {+ oldGnuText.write("Public contents.", index * 1024);+ return { offset: index * 1024, size: 512 };+ });+ oldGnuExtents.push({ offset: oldGnuText.length, size: 0 });+ const unicodeCompressedPayload = Buffer.from(cleanCompressedPayload);+ Buffer.from("😀").copy(+ unicodeCompressedPayload,+ unicodeCompressedPayload.indexOf("Go/w") - 4,+ );+ const leadingMarker = cleanCompressedPayload.indexOf("Go/w");+ expect(leadingMarker).toBeGreaterThan(0);+ const repositoryCompressedPayload = Buffer.from(cleanCompressedPayload);+ const repositoryMarker = Buffer.from(+ ["github.com", "openai", "openai"].join("/"),+ );+ repositoryMarker.copy(repositoryCompressedPayload, 300);+ const repositoryMarkerEnd = 300 + repositoryMarker.length;+ expect(+ brotliDecompressSync(repositoryCompressedPayload).toString("utf8"),+ ).toMatch(/^[A-Za-z0-9]+$/u);+ const middle = Math.floor(cleanCompressedPayload.length / 2);+ const cases = [+ {+ name: "brotli",+ files: [["runtime.mjs.br", cleanCompressedPayload, true]],+ },+ {+ name: "unicode-brotli",+ files: [["runtime.mjs.br", unicodeCompressedPayload, true]],+ },+ {+ name: "sparse-brotli-leading-delimiter",+ files: [+ [+ "runtime.mjs.br.part-000",+ cleanCompressedPayload.subarray(0, leadingMarker),+ false,+ ],+ [+ "runtime.mjs.br.part-001",+ cleanCompressedPayload.subarray(leadingMarker),+ true,+ ],+ ],+ },+ {+ name: "sparse-brotli-trailing-delimiter",+ files: [+ [+ "runtime.mjs.br.part-000",+ repositoryCompressedPayload.subarray(0, repositoryMarkerEnd),+ true,+ ],+ [+ "runtime.mjs.br.part-001",+ repositoryCompressedPayload.subarray(repositoryMarkerEnd),+ false,+ ],+ ],+ },+ {+ name: "mixed-split-brotli",+ files: [+ [+ "runtime.mjs.br.part-000",+ cleanCompressedPayload.subarray(0, middle),+ true,+ ],+ [+ "runtime.mjs.br.part-001",+ cleanCompressedPayload.subarray(middle),+ false,+ ],+ ],+ },+ {+ name: "mixed-sparse-split-tail",+ files: [+ [+ "runtime.mjs.br.part-000",+ cleanCompressedPayload.subarray(0, middle),+ true,+ ],+ [+ "runtime.mjs.br.part-001",+ cleanCompressedPayload.subarray(middle),+ "0.1-tail",+ ],+ ],+ error: "npm tarball contains trailing Brotli data",+ },+ { name: "png", files: [["logo.png", logo, true]] },+ {+ name: "pax-sparse-zero-byte-hole-png",+ files: [["logo.png", logo, "0.1-hole"]],+ error: "npm tarball contains an unexpected PNG asset",+ },+ {+ name: "pax-sparse-zero-byte-hole-tail",+ files: [["logo.png", logo, "0.1-hole"]],+ tailMarker: true,+ error:+ /npm tarball contains (?:an internal reference\.|an invalid tar entry)/u,+ },+ {+ name: "oldgnu-brotli",+ files: [["runtime.mjs.br", cleanCompressedPayload, "oldgnu"]],+ error: "npm tarball contains an invalid tar entry",+ },+ {+ name: "oldgnu-png",+ files: [["logo.png", logo, "oldgnu"]],+ error: "npm tarball contains an invalid tar entry",+ },+ {+ name: "oldgnu-continuations",+ files: [["helpers.mjs", oldGnuText, "oldgnu"]],+ extents: oldGnuExtents,+ },+ {+ name: "oldgnu-unused-main-slot",+ files: [["runtime.mjs.br", cleanCompressedPayload, "oldgnu"]],+ metadataMarker: "unused",+ error: "npm tarball contains an invalid tar entry",+ },+ {+ name: "oldgnu-unused-continuation-slot",+ files: [["helpers.mjs", oldGnuText, "oldgnu"]],+ extents: oldGnuExtents,+ metadataMarker: "unused",+ },+ {+ name: "oldgnu-continuation-marker",+ files: [["helpers.mjs", oldGnuText, "oldgnu"]],+ extents: oldGnuExtents,+ metadataMarker: "continuation",+ error: "npm tarball contains an internal reference.",+ },+ {+ name: "oldgnu-metadata-boundary-marker",+ files: [["helpers.mjs", oldGnuText, "oldgnu"]],+ extents: oldGnuExtents,+ metadataMarker: "boundary",+ error: "npm tarball contains an internal reference.",+ },+ {+ name: "oldgnu-text-body-boundary-marker",+ files: [+ ["helpers.mjs", Buffer.from("synthetic-reference\n"), "oldgnu"],+ ],+ metadataMarker: "boundary",+ error: "npm tarball contains an internal reference.",+ },+ {+ name: "global-sparse-keys",+ files: [["runtime.mjs.br", cleanCompressedPayload, false]],+ globalSparse: true,+ },+ {+ name: "expanded-marker",+ files: [+ [+ "runtime.mjs.br",+ brotliCompressSync(Buffer.from("go/synthetic-reference")),+ true,+ ],+ ],+ error: "npm tarball contains an internal reference.",+ },+ {+ name: "sparse-header-map-boundary-marker",+ files: [["runtime.mjs.br", cleanCompressedPayload, "1.0"]],+ headerMapMarker: true,+ error: "npm tarball contains an internal reference.",+ },+ {+ name: "sparse-tail-header-boundary-marker",+ files: [+ ["runtime.mjs.br", cleanCompressedPayload, "1.0"],+ ["extra.mjs", Buffer.from("Public contents."), false],+ ],+ tailBoundaryMarker: true,+ error:+ /npm tarball contains (?:an internal reference\.|an invalid tar entry:)/u,+ },+ {+ name: "map-padding-marker",+ files: [["runtime.mjs.br", cleanCompressedPayload, "1.0"]],+ mapMarker: true,+ error: "npm tarball contains an internal reference.",+ },+ {+ name: "discarded-tail-marker",+ files: [["logo.png", logo, "1.0"]],+ tailMarker: true,+ error:+ /npm tarball contains (?:an internal reference\.|an invalid tar entry:)/u,+ },+ ] satisfies {+ name: string;+ files: [+ string,+ Buffer,+ boolean | "1.0" | "0.1-tail" | "0.1-hole" | "oldgnu",+ ][];+ extents?: { offset: number; size: number }[];+ metadataMarker?: "continuation" | "boundary" | "unused";+ globalSparse?: boolean;+ mapMarker?: boolean;+ headerMapMarker?: boolean;+ tailMarker?: boolean;+ tailBoundaryMarker?: boolean;+ error?: string | RegExp;+ }[];+ for (const scenario of cases) {+ const records = scenario.files.map(([name, contents, sparse]) => {+ const path = `package/_bundled_plugin/${name}`;+ if (!sparse) return tarRecord(contents, { name: path });+ if (sparse === "oldgnu")+ return oldGnuSparseRecord(+ contents,+ path,+ scenario.extents,+ scenario.metadataMarker,+ );+ if (sparse === "0.1-hole") {+ const hole = contents.findIndex(+ (byte, index) => index > 0 && index % 512 === 0 && byte === 0,+ );+ expect(hole).toBeGreaterThan(0);+ const stored = Buffer.concat([+ contents.subarray(0, hole),+ contents.subarray(hole + 1),+ scenario.tailMarker+ ? Buffer.from("go/synthetic-reference")+ : Buffer.alloc(0),+ ]);+ return Buffer.concat([+ tarRecord(+ paxRecords({+ "GNU.sparse.size": String(contents.length),+ "GNU.sparse.numblocks": "2",+ "GNU.sparse.map": `0,${hole},${hole + 1},${contents.length - hole - 1}`,+ }),+ { name: "PaxHeaders/asset", type: 0x78 },+ ),+ tarRecord(stored, { name: path }),+ ]);+ }+ if (sparse === true) {+ return Buffer.concat([+ tarRecord(+ paxRecords({+ "GNU.sparse.size": String(contents.length),+ "GNU.sparse.numblocks": "1",+ "GNU.sparse.map": `0,${contents.length}`,+ }),+ { name: "PaxHeaders/asset", type: 0x78 },+ ),+ tarRecord(contents, { name: path }),+ ]);+ }+ if (sparse === "0.1-tail") {+ const storedContents = Buffer.concat([+ contents,+ Buffer.from("go/synthetic-reference"),+ ]);+ return Buffer.concat([+ tarRecord(+ paxRecords({+ "GNU.sparse.size": String(storedContents.length),+ "GNU.sparse.numblocks": "1",+ "GNU.sparse.map": `0,${storedContents.length}`,+ }),+ { name: "PaxHeaders/asset", type: 0x78 },+ ),+ tarRecord(storedContents, { name: path }),+ ]);+ }+ const map = Buffer.alloc(512);+ map.write(`1\n0\n${contents.length}\n`);+ if (scenario.mapMarker) map.write("go/synthetic-reference", 128);+ const tail = scenario.tailBoundaryMarker+ ? Buffer.alloc((512 - (contents.length % 512)) % 512 || 512)+ : scenario.tailMarker+ ? Buffer.from("go/synthetic-reference")+ : Buffer.alloc(0);+ if (scenario.tailBoundaryMarker) tail.write("go/", tail.length - 3);+ return Buffer.concat([+ tarRecord(+ paxRecords({+ "GNU.sparse.major": "1",+ "GNU.sparse.minor": "0",+ "GNU.sparse.name": path,+ "GNU.sparse.realsize": String(contents.length),+ }),+ { name: "PaxHeaders/asset", type: 0x78 },+ ),+ tarRecord(Buffer.concat([map, contents, tail]), {+ name: "package/GNUSparseFile.1/asset",+ ...(scenario.headerMapMarker+ ? {+ reserved: Buffer.concat([+ Buffer.alloc(9),+ Buffer.from("go/"),+ ]),+ }+ : {}),+ }),+ ]);+ });+ const archivePath = join(root, `${scenario.name}.tgz`);+ writeFileSync(+ archivePath,+ gzipSync(+ Buffer.concat([+ ...(scenario.globalSparse+ ? [+ tarRecord(+ paxRecords({+ "GNU.sparse.major": "1",+ "GNU.sparse.minor": "0",+ }),+ { name: "GlobalHead", type: 0x67 },+ ),+ ]+ : []),+ packageTar({ additionalRecords: records }),+ ]),+ ),+ );+ const contractPath = join(root, "contract.json");+ writeFileSync(+ contractPath,+ JSON.stringify({+ ...pluginContract,+ shippedExact: [+ ...pluginContract.shippedExact,+ ...scenario.files.map(([name]) => name),+ ],+ }),+ );+ const environment = { ...process.env };+ delete environment["CODEX_SECURITY_EXPECTED_GIT_HEAD"];+ const result = spawnSync(+ commandPath("node"),+ [+ fileURLToPath(+ new URL("../scripts/check-package.mjs", import.meta.url),+ ),+ archivePath,+ contractPath,+ ],+ {+ cwd: root,+ env: environment,+ encoding: "utf8",+ timeout: 30_000,+ windowsHide: true,+ },+ );+ if (scenario.error) {+ expect({ scenario: scenario.name, status: result.status }).toEqual({+ scenario: scenario.name,+ status: 1,+ });+ if (typeof scenario.error === "string")+ expect(result.stderr).toContain(scenario.error);+ else expect(result.stderr).toMatch(scenario.error);+ } else {+ expect({+ scenario: scenario.name,+ status: result.status,+ stderr: result.stderr,+ }).toEqual({ scenario: scenario.name, status: 0, stderr: "" });+ }+ }+ } finally {+ rmSync(root, { force: true, recursive: true });+ }+ });++ test.each([0x78, 0x67])(+ "preserves inherited character locale for Unicode pax owners, type=%i",+ (type) => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-owner-test-"));+ try {+ const archivePath = join(root, "unicode owner.tgz");+ writeFileSync(+ archivePath,+ gzipSync(+ Buffer.concat([+ tarRecord(+ paxRecords({+ uname: "Synthetic é owner",+ gname: "Synthetic é group",+ }),+ { name: "OwnerHead", type },+ ),+ packageTar(),+ ]),+ ),+ );+ const contractPath = join(root, "plugin contract.json");+ writeFileSync(contractPath, JSON.stringify(pluginContract));+ const environment: NodeJS.ProcessEnv = { ...process.env };+ delete environment["CODEX_SECURITY_EXPECTED_GIT_HEAD"];+ if (process.platform !== "win32")+ environment["LC_ALL"] =+ process.platform === "darwin" ? "en_US.UTF-8" : "C.UTF-8";+ const result = spawnSync(+ commandPath("node"),+ [+ fileURLToPath(+ new URL("../scripts/check-package.mjs", import.meta.url),+ ),+ archivePath,+ contractPath,+ ],+ {+ cwd: root,+ env: environment,+ encoding: "utf8",+ timeout: 30_000,+ windowsHide: true,+ },+ );+ expect({ status: result.status, stderr: result.stderr }).toEqual({+ status: 0,+ stderr: "",+ });+ } finally {+ rmSync(root, { force: true, recursive: true });+ }+ },+ );++ test("streams each archive without resolving tar from its directory", () => {+ const root = mkdtempSync(join(tmpdir(), "codex-package-tar-test-"));+ try {+ const archiveDirectory = join(+ root,+ process.platform === "win32" ? "package archive" : "D: package archive",+ );+ mkdirSync(archiveDirectory, { recursive: true });+ const archivePath = join(+ archiveDirectory,+ process.platform === "win32"+ ? "-fixture package.tgz"+ : "-fixture: package.tgz",+ );+ const contractPath = join(root, "plugin contract.json");+ const logPath = join(root, "tar calls.jsonl");+ const adjacentTarMarker = join(root, "archive tar ran");+ const tarBytes = packageTar({+ compatibleLayout: true,+ rootDirectoryMode: 0o555,+ readmeMode: 0,+ });+ const archiveContents = gzipSync(tarBytes, { level: 0 });+ writeFileSync(archivePath, archiveContents);+ writeFileSync(contractPath, JSON.stringify(pluginContract));++ const nodePath = commandPath("node");+ const environment: NodeJS.ProcessEnv = { ...process.env };+ delete environment["CODEX_SECURITY_EXPECTED_GIT_HEAD"];+ environment["PATH"] = `.${delimiter}${process.env["PATH"] ?? ""}`;+ const extractionDirectory = join(root, "extraction");+ mkdirSync(extractionDirectory, { mode: 0o700 });+ for (const variable of ["TMPDIR", "TMP", "TEMP"]) {+ environment[variable] = extractionDirectory;+ }+ if (process.platform === "win32") {+ for (const name of ["tar.com", "tar.exe"]) {+ writeFileSync(join(archiveDirectory, name), "not an executable");+ }+ } else {+ const adjacentTar = join(archiveDirectory, "tar");+ writeFileSync(+ adjacentTar,+ `#!/usr/bin/env node+require("node:fs").writeFileSync(process.env.ADJACENT_TAR_MARKER, "ran");+process.exit(99);+`,+ );+ chmodSync(adjacentTar, 0o755);++ const proxySource = `#!/usr/bin/env node+const { spawnSync } = require("node:child_process");+const { createHash } = require("node:crypto");+const { appendFileSync, readFileSync } = require("node:fs");+const input = readFileSync(0);+appendFileSync(+ process.env.TAR_PROXY_LOG,+ JSON.stringify({+ args: process.argv.slice(2),+ cwd: process.cwd(),+ inputLength: input.length,+ inputSha256: createHash("sha256").update(input).digest("hex"),+ }) + "\\n",+);+const result = spawnSync(process.env.REAL_TAR, process.argv.slice(2), {+ env: process.env,+ input,+ stdio: ["pipe", "inherit", "inherit"],+ windowsHide: true,+});+if (result.error !== undefined) throw result.error;+if (result.status === 0 && process.argv.includes("-xzf") && process.env.TAR_PROXY_FAIL_EXTRACT) {+ process.stderr.write("synthetic extraction failure\\n");+ process.exit(23);+}+process.exit(result.status ?? 1);+`;+ const proxyPath = join(root, "tar");+ writeFileSync(proxyPath, proxySource);+ chmodSync(proxyPath, 0o755);+ environment["ADJACENT_TAR_MARKER"] = adjacentTarMarker;+ environment["REAL_TAR"] = commandPath("tar");+ environment["TAR_PROXY_LOG"] = logPath;+ }++ const result = spawnSync(+ nodePath,+ [+ fileURLToPath(+ new URL("../scripts/check-package.mjs", import.meta.url),+ ),+ archivePath,+ contractPath,+ ],+ {+ cwd: root,+ encoding: "utf8",+ env: environment,+ timeout: 30_000,+ windowsHide: true,+ },+ );+ expect(existsSync(adjacentTarMarker)).toBe(false);+ expect({ status: result.status, stderr: result.stderr }).toEqual({+ status: 0,+ stderr: "",+ });+ expect(readdirSync(extractionDirectory)).toEqual([]);+ if (process.platform === "win32") return;++ const calls = readFileSync(logPath, "utf8")+ .trim()+ .split(/\r?\n/u)+ .map(+ (line) =>+ JSON.parse(line) as {+ args: string[];+ cwd: string;+ inputLength: number;+ inputSha256: string;+ },+ );+ const archiveSha256 = createHash("sha256")+ .update(archiveContents)+ .digest("hex");+ expect(calls).toHaveLength(3);+ for (const call of calls) {+ expect(call.args.filter((arg) => arg === "-")).toEqual(["-"]);+ expect(call.args).not.toContain(archivePath);+ expect(realpathSync(call.cwd)).toBe(realpathSync(root));+ expect(call.inputLength).toBe(archiveContents.length);+ expect(call.inputSha256).toBe(archiveSha256);+ }+ expect(calls[0]?.args).toEqual(["--ignore-zeros", "-tzf", "-"]);+ expect(calls[1]?.args).toEqual([+ "--ignore-zeros",+ "--numeric-owner",+ "-tvzf",+ "-",+ ]);+ expect(calls[2]?.args.slice(0, 10)).toEqual([+ "--ignore-zeros",+ "-m",+ "--keep-old-files",+ "--no-same-owner",+ "--no-same-permissions",+ "--no-acls",+ "--no-xattrs",+ "-xzf",+ "-",+ "-C",+ ]);++ environment["TAR_PROXY_FAIL_EXTRACT"] = "true";+ const failed = spawnSync(+ nodePath,+ [+ fileURLToPath(+ new URL("../scripts/check-package.mjs", import.meta.url),+ ),+ archivePath,+ contractPath,+ ],+ {+ cwd: root,+ encoding: "utf8",+ env: environment,+ timeout: 30_000,+ windowsHide: true,+ },+ );+ expect(failed.status).not.toBe(0);+ expect(failed.stderr).toContain("synthetic extraction failure");+ expect(readdirSync(extractionDirectory)).toEqual([]);+ } finally {+ rmSync(root, { force: true, recursive: true });+ }+ });});