OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
fix(plugin): remove arbitrary scan report and source limits (#381)
ianw-oai7d29a42d1dc6
diff --git a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.pyindex 0d545f275..e2df3fef9 100644--- a/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py+++ b/sdk/typescript/_bundled_plugin/scripts/finalize_scan_contract.py@@ -20,7 +20,7 @@ import sysfrom collections.abc import Iteratorfrom datetime import datetime, timezonefrom pathlib import Path, PurePosixPath-from typing import Any, BinaryIO, TextIO+from typing import Any, TextIOfrom urllib.parse import quote, urlsplitSCHEMA_VERSION = "1.0"@@ -58,33 +58,8 @@ GITHUB_HASH_BLOCK_SIZE = 100GITHUB_HASH_MOD = 37GITHUB_HASH_MASK = (1 << 64) - 1GITHUB_HASH_EOF = 65535-GITHUB_HASH_MAX_LINES = 100_000SOURCE_READ_CHUNK_SIZE = 64 * 1024-SOURCE_READ_MAX_BYTES = 10 * 1024 * 1024-CONTRACT_DOCUMENT_MAX_BYTES = {- "scan-manifest.json": 16 * 1024 * 1024,- "findings.json": 128 * 1024 * 1024,- "coverage.json": 32 * 1024 * 1024,-}-SCHEMA_DOCUMENT_MAX_BYTES = 4 * 1024 * 1024-JSON_DOCUMENT_READ_CHUNK_SIZE = 64 * 1024-MAX_JSON_DEPTH = 256MAX_JSON_INTEGER = (1 << 53) - 1-MAX_SCHEMA_NODES = 8192-MAX_SCHEMA_COLLECTION_ENTRIES = 4096-MAX_SCHEMA_APPLICATOR_EDGES = 128-SAFE_SCHEMA_PATTERNS = {- r"^(?![^:/?#]+://[^/?#]*@)[^?#]+$",- r"^codex-security-snapshot/v1:sha256:[a-f0-9]{64}$",- r"^(?!/)(?!.*(?:^|/)\.\.(?:/|$))(?!.*\\).+$",- r"^[a-f0-9]{64}$",- r"^(?!.*(?:^|/)\.\.(?:/|$))(?!.*\\)artifacts/.+$",- r"^csf_[a-f0-9]{24}$",- r"^occ_[a-f0-9]{24}$",- r"^[a-z0-9][a-z0-9._/-]*$",- r"^codex-security/v1:sha256:[a-f0-9]{64}$",- r"^findings/([a-z0-9][a-z0-9._-]*)/\1\.md$",-}EXPORT_PATHS = {"csv": "exports/findings.csv","json": "exports/findings.json",@@ -106,9 +81,7 @@ def _loads_json(value: str | bytes) -> Any:def _read_json(path: Path) -> dict[str, Any]:try:- with path.open("rb") as handle:- raw = _read_bounded_json_document(handle, str(path), SCHEMA_DOCUMENT_MAX_BYTES)- payload = _loads_json(raw.decode("utf-8"))+ payload = _loads_json(path.read_text(encoding="utf-8"))_require_safe_json_value(payload, str(path))except FileNotFoundError as exc:raise ContractError(f"missing required contract artifact: {path}") from exc@@ -157,72 +130,21 @@ def _json_bytes(payload: Any) -> bytes:def _contract_json_bytes(relative_path: str, payload: Any) -> bytes:_require_safe_json_value(payload, relative_path)- encoded = _json_bytes(payload)- maximum = CONTRACT_DOCUMENT_MAX_BYTES.get(relative_path)- if maximum is not None and len(encoded) > maximum:- raise ContractError(f"{relative_path}: JSON document exceeds the {maximum}-byte limit")- return encoded---def _read_bounded_json_document(handle: BinaryIO, context: str, maximum: int) -> bytes:- if os.fstat(handle.fileno()).st_size > maximum:- raise ContractError(f"{context}: JSON document exceeds the {maximum}-byte limit")- chunks: list[bytes] = []- length = 0- while length <= maximum:- chunk = handle.read(min(JSON_DOCUMENT_READ_CHUNK_SIZE, maximum + 1 - length))- if not chunk:- break- length += len(chunk)- if length > maximum:- raise ContractError(f"{context}: JSON document exceeds the {maximum}-byte limit")- chunks.append(chunk)- result = b"".join(chunks)- _require_json_nesting(result, context)- return result---def _require_json_nesting(value: bytes, context: str) -> None:- depth = 0- in_string = False- escaped = False- for character in value:- if in_string:- if escaped:- escaped = False- elif character == ord("\\"):- escaped = True- elif character == ord('"'):- in_string = False- continue- if character == ord('"'):- in_string = True- elif character in (ord("{"), ord("[")):- depth += 1- if depth > MAX_JSON_DEPTH + 1:- raise ContractError(- f"{context}: JSON document exceeds the {MAX_JSON_DEPTH}-level nesting limit"- )- elif character in (ord("}"), ord("]")):- depth -= 1+ return _json_bytes(payload)def _require_safe_json_value(value: Any, context: str, *, validate_strings: bool = True) -> None:- def visit(item: Any, location: str, depth: int) -> None:- if depth > MAX_JSON_DEPTH:- raise ContractError(- f"{location}: JSON document exceeds the {MAX_JSON_DEPTH}-level nesting limit"- )+ def visit(item: Any, location: str) -> None:if isinstance(item, dict):for key, child in item.items():if not isinstance(key, str):raise ContractError(f"{location}: expected string JSON property names")if validate_strings:_require_safe_json_string(key, location)- visit(child, f"{location}.<property>", depth + 1)+ visit(child, f"{location}.<property>")elif isinstance(item, list):for index, child in enumerate(item):- visit(child, f"{location}[{index}]", depth + 1)+ visit(child, f"{location}[{index}]")elif isinstance(item, str) and validate_strings:_require_safe_json_string(item, location)elif isinstance(item, int) and not isinstance(item, bool):@@ -238,7 +160,7 @@ def _require_safe_json_value(value: Any, context: str, *, validate_strings: boolf"{location}: unsafe integer-valued JSON numbers are not supported")- visit(value, context, 0)+ visit(value, context)def _require_safe_json_string(value: str, context: str) -> None:@@ -505,12 +427,7 @@ def _read_scan_local_json_bytes(try:with os.fdopen(descriptor, "rb") as handle:descriptor = -1- maximum = CONTRACT_DOCUMENT_MAX_BYTES.get(relative_path)- raw = (- handle.read()- if maximum is None- else _read_bounded_json_document(handle, context, maximum)- )+ raw = handle.read()try:payload = _loads_json(raw.decode("utf-8"))except (UnicodeDecodeError, ValueError) as exc:@@ -786,7 +703,6 @@ def _recover_unsealed_findings(warnings: list[str],) -> list[str]:schema = _read_json(schema_dir / "findings.schema.json")- _require_safe_schema(schema, "findings.schema.json")properties = _require_dict(schema, "properties", "findings.schema")finding_array = _require_dict(properties, "findings", "findings.schema.properties")finding_schema = _require_dict(finding_array, "items", "findings.schema.properties.findings")@@ -937,7 +853,6 @@ def _recover_unsealed_coverage(discarded_findings: list[str],) -> None:schema = _read_json(schema_dir / "coverage.schema.json")- _require_safe_schema(schema, "coverage.schema.json")properties = _require_dict(schema, "properties", "coverage.schema")completeness = coverage.get("completeness")partial = completeness not in ("complete", "partial", "unknown")@@ -1536,91 +1451,9 @@ def _validate_schema_node(value: Any, schema: dict[str, Any], context: str) -> Ndef validate_against_schema(payload: dict[str, Any], schema_path: Path) -> None:schema = _read_json(schema_path)- _require_safe_schema(schema, schema_path.name)_validate_schema_node(payload, schema, schema_path.stem)-def _require_safe_schema(schema: dict[str, Any], context: str) -> None:- pending: list[tuple[Any, bool]] = [(schema, True)]- nodes = 0- applicator_edges = 0- unsupported_keywords = {- "$async",- "$ref",- "$dynamicRef",- "$recursiveRef",- "prefixItems",- "patternProperties",- "propertyNames",- "dependentSchemas",- "dependencies",- "uniqueItems",- }- while pending:- value, is_schema = pending.pop()- nodes += 1- if nodes > MAX_SCHEMA_NODES:- raise ContractError(- f"{context}: JSON Schema exceeds the {MAX_SCHEMA_NODES}-node complexity limit"- )- if isinstance(value, list):- if len(value) > MAX_SCHEMA_COLLECTION_ENTRIES:- raise ContractError(- f"{context}: JSON Schema exceeds the "- f"{MAX_SCHEMA_COLLECTION_ENTRIES}-entry collection limit"- )- pending.extend((child, is_schema) for child in value)- continue- if not isinstance(value, dict):- continue- if len(value) > MAX_SCHEMA_COLLECTION_ENTRIES:- raise ContractError(- f"{context}: JSON Schema exceeds the "- f"{MAX_SCHEMA_COLLECTION_ENTRIES}-entry collection limit"- )- for keyword, child in value.items():- if not is_schema:- pending.append((child, False))- continue- if keyword in unsupported_keywords:- raise ContractError(f"{context}: unsupported JSON Schema keyword")- edges = 0- if keyword in {"allOf", "anyOf", "oneOf"} and isinstance(child, list):- edges = len(child)- elif keyword in {- "if",- "then",- "else",- "not",- "items",- "contains",- "additionalProperties",- "unevaluatedProperties",- "unevaluatedItems",- } and isinstance(child, (dict, bool)):- edges = 1- elif keyword in {"properties", "$defs", "definitions"} and isinstance(child, dict):- edges = len(child)- applicator_edges += edges- if applicator_edges > MAX_SCHEMA_APPLICATOR_EDGES:- raise ContractError(- f"{context}: JSON Schema exceeds the "- f"{MAX_SCHEMA_APPLICATOR_EDGES}-edge applicator limit"- )- if keyword == "pattern" and isinstance(child, str):- if child not in SAFE_SCHEMA_PATTERNS:- raise ContractError(f"{context}: unsupported JSON Schema pattern")- if keyword in {"properties", "$defs", "definitions"} and isinstance(child, dict):- pending.extend((child_schema, True) for child_schema in child.values())- continue- pending.append(- (- child,- keyword not in {"const", "enum", "default", "examples", "dependentRequired"},- )- )--def _validate_canonical_schemas_before_projection(manifest: dict[str, Any],findings: dict[str, Any],@@ -1666,10 +1499,7 @@ def _utf16_code_units(value: str) -> Iterator[int]:def _github_line_hashes(handle: TextIO,requested_lines: set[int] | None = None,- source_read_budget: list[int] | None = None,-) -> dict[int, str] | None:- if source_read_budget is not None and source_read_budget[0] <= 0:- return None+) -> dict[int, str]:window = [0] * GITHUB_HASH_BLOCK_SIZEline_numbers = [-1] * GITHUB_HASH_BLOCK_SIZEhash_counts: dict[str, int] = {}@@ -1680,7 +1510,6 @@ def _github_line_hashes(line_number = 0line_start = Trueprevious_was_cr = False- source_bytes = 0def output_hash() -> None:nonlocal index@@ -1698,11 +1527,11 @@ def _github_line_hashes(hash_raw = (GITHUB_HASH_MOD * hash_raw + current - first_mod * beginning) & GITHUB_HASH_MASKindex = (index + 1) % GITHUB_HASH_BLOCK_SIZE- def process_character(current: int) -> bool:+ def process_character(current: int) -> None:nonlocal line_number, line_start, previous_was_crif current in {ord(" "), ord("\t")} or (previous_was_cr and current == ord("\n")):previous_was_cr = False- return True+ returnif current == ord("\r"):current = ord("\n")previous_was_cr = True@@ -1713,28 +1542,15 @@ def _github_line_hashes(if line_start:line_start = Falseline_number += 1- if line_number > GITHUB_HASH_MAX_LINES:- return Falseline_numbers[index] = line_numberif current == ord("\n"):line_start = Trueupdate_hash(current)- return Truewhile chunk := handle.read(SOURCE_READ_CHUNK_SIZE):- chunk_bytes = len(chunk.encode("utf-8", errors="replace"))- source_bytes += chunk_bytes- if source_bytes > SOURCE_READ_MAX_BYTES:- return None- if source_read_budget is not None:- source_read_budget[0] -= chunk_bytes- if source_read_budget[0] < 0:- return Nonefor code_unit in _utf16_code_units(chunk):- if not process_character(code_unit):- return None- if not process_character(GITHUB_HASH_EOF):- return None+ process_character(code_unit)+ process_character(GITHUB_HASH_EOF)for _ in range(GITHUB_HASH_BLOCK_SIZE):if line_numbers[index] != -1:output_hash()@@ -1762,14 +1578,13 @@ def _github_line_hashes_for_source(source_root: Path,relative_path: str,requested_lines: set[int] | None = None,- source_read_budget: list[int] | None = None,) -> dict[int, str] | None:handle = _open_source_file(source_root, relative_path)if handle is None:return Nonetry:with handle:- return _github_line_hashes(handle, requested_lines, source_read_budget)+ return _github_line_hashes(handle, requested_lines)except OSError:return None@@ -1849,15 +1664,8 @@ def _github_line_hash_cache()requested_lines_by_path.setdefault(relative_path, set()).add(primary_location["startLine"])line_hash_cache: dict[tuple[Path, int], str | None] = {}- source_read_budget = [SOURCE_READ_MAX_BYTES]for relative_path, requested_lines in requested_lines_by_path.items():- line_hashes = (- None- if source_read_budget[0] <= 0- else _github_line_hashes_for_source(- source_root, relative_path, requested_lines, source_read_budget- )- )+ line_hashes = _github_line_hashes_for_source(source_root, relative_path, requested_lines)source_path = source_root / relative_pathfor line_number in requested_lines:line_hash_cache[(source_path, line_number)] = (diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_source_excerpt.py b/sdk/typescript/_bundled_plugin/scripts/workbench_source_excerpt.pyindex 30c12033e..cea25e23f 100644--- a/sdk/typescript/_bundled_plugin/scripts/workbench_source_excerpt.py+++ b/sdk/typescript/_bundled_plugin/scripts/workbench_source_excerpt.py@@ -10,11 +10,10 @@ from typing import Any# Some plugin hosts launch Python with safe-path isolation enabled.sys.path.insert(0, str(Path(__file__).resolve().parent))-from workbench_target import clean_worktree_content_digest, git_bytes, git_output+from workbench_target import clean_worktree_content_digest, git_bytesCONTEXT_LINES = 3MAX_BYTES = 16_000-MAX_FILE_BYTES = 1_048_576MAX_LINES = 60@@ -70,9 +69,6 @@ def scanned_source_text(scan: sqlite3.Row, target: Path, path: str) -> str | Nonif snapshot_digest is not None and snapshot_digest != clean_worktree_content_digest():return Noneobject_name = f"{revision}:{path}"- size = git_output(target, "cat-file", "-s", object_name)- if size is None or not size.isdigit() or int(size) > MAX_FILE_BYTES:- return Nonecontent = git_bytes(target, "cat-file", "blob", object_name)return content.decode("utf-8", errors="replace") if content is not None else Nonediff --git a/sdk/typescript/tests-ts/plugin-report-limits.test.ts b/sdk/typescript/tests-ts/plugin-report-limits.test.tsnew file mode 100644index 000000000..72b45bc6f--- /dev/null+++ b/sdk/typescript/tests-ts/plugin-report-limits.test.ts@@ -0,0 +1,42 @@+import { join } from "node:path";+import { describe, expect, test } from "bun:test";+import { PLUGIN_ROOT } from "./plugin-root.js";++describe("bundled scan report and source limits", () => {+ test("accepts large reports, schemas, source files, and late source lines", () => {+ const python = Bun.which("python3") ?? Bun.which("python");+ expect(python).not.toBeNull();+ const program = [+ "import io, json, pathlib, sys, tempfile",+ "sys.path.insert(0, sys.argv[1])",+ "import finalize_scan_contract as finalizer",+ "import workbench_source_excerpt as excerpts",+ "document = finalizer._contract_json_bytes('scan-manifest.json', {'metadata': 'x' * (16 * 1024 * 1024)})",+ "nested = 0",+ "for _ in range(258): nested = [nested]",+ "finalizer._require_safe_json_value(nested, 'nested')",+ "with tempfile.TemporaryDirectory() as directory:",+ " schema = pathlib.Path(directory) / 'large.schema.json'",+ " schema.write_text(json.dumps({'type': 'object', 'description': 'x' * (4 * 1024 * 1024), 'allOf': [{'type': 'object'}] * 129}))",+ " finalizer.validate_against_schema({'safe': True}, schema)",+ " source = b'x' * (1024 * 1024 + 1)",+ " excerpts.git_bytes = lambda *args: source",+ " target = pathlib.Path(directory).resolve()",+ " excerpt = excerpts.scanned_source_text({'target_revision': 'deadbeef', 'target_snapshot_digest': None}, target, 'large.py')",+ " hashes = finalizer._github_line_hashes(io.StringIO('line\\n' * 100001), {100001})",+ " print(json.dumps({'documentBytes': len(document), 'sourceBytes': len(excerpt), 'lateSourceLine': 100001 in hashes, 'unsafePathRejected': excerpts.safe_source_path(target, '../outside') is None}))",+ ].join("\n");+ const result = Bun.spawnSync(+ [python!, "-I", "-B", "-c", program, join(PLUGIN_ROOT, "scripts")],+ { stdout: "pipe", stderr: "pipe" },+ );++ expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);+ expect(JSON.parse(new TextDecoder().decode(result.stdout))).toMatchObject({+ documentBytes: expect.any(Number),+ sourceBytes: 1024 * 1024 + 1,+ lateSourceLine: true,+ unsafePathRejected: true,+ });+ });+});