Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(preview): preserve bounded source and finding details (#1275)

mldangelo-oai66b336119f21
diff --git a/plugins/codex-security/scripts/finding_preview.py b/plugins/codex-security/scripts/finding_preview.py
index f511e8a78..3ac978eb3 100644
--- a/plugins/codex-security/scripts/finding_preview.py
+++ b/plugins/codex-security/scripts/finding_preview.py
@@ -19,7 +19,9 @@ from workbench_constants import (
FINDING_CODE_EVIDENCE_SNIPPET_BYTES,
FINDING_DETAILS_PREVIEW_BYTES,
FINDING_EVIDENCE_EXCERPT_BYTES,
+ FINDING_LEVEL_BYTES,
FINDING_ROOT_CAUSE_PREVIEW_BYTES,
+ FINDING_SUMMARY_BYTES,
FINDING_VALIDATION_PREVIEW_BYTES,
)
@@ -177,6 +179,37 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
for key in ("remediationTests", "preventiveControls")
if key in prepared and isinstance(prepared[key], list)
}
+ for key in ("severity", "confidence"):
+ if key in prepared:
+ prepared[key] = bounded_finding_section(
+ prepared[key],
+ FINDING_SUMMARY_BYTES,
+ ("level",),
+ ((("level",), FINDING_LEVEL_BYTES),),
+ )
+ if "taxonomy" in prepared:
+ prepared["taxonomy"] = bounded_finding_section(
+ prepared["taxonomy"], FINDING_SUMMARY_BYTES, ("cwe",), ()
+ )
+ metadata_keys = (
+ "ruleId",
+ "status",
+ "detectedAt",
+ "identity",
+ "taxonomy",
+ "severity",
+ "confidence",
+ )
+ metadata = {
+ key: bounded_json_value(prepared[key], [FINDING_SUMMARY_BYTES])
+ for key in metadata_keys
+ if key in prepared
+ }
+ metadata = dict(sorted(metadata.items(), key=lambda item: json_size(item[1])))
+ metadata_budget = FINDING_SUMMARY_BYTES - 2 - sum(json_size(key) + 2 for key in metadata)
+ for index, (key, item) in enumerate(metadata.items()):
+ metadata[key] = bounded_json_value(item, [metadata_budget // (len(metadata) - index)])
+ metadata_budget -= json_size(metadata[key])
diagnostics = (
"rootCause",
"root_cause",
@@ -188,20 +221,15 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
core_keys = (
"writeup",
*diagnostics,
- "confidence",
- "detectedAt",
- "identity",
"provenance",
- "ruleId",
- "severity",
- "status",
- "taxonomy",
"evidence",
"evidenceExcerpt",
)
core = {key: prepared[key] for key in core_keys if key in prepared}
extras = {
- key: item for key, item in prepared.items() if key not in core and key not in guidance
+ key: item
+ for key, item in prepared.items()
+ if key not in core and key not in guidance and key not in metadata_keys
}
complete_guidance = {key: items[:1] for key, items in guidance.items()}
minimum_guidance = {
@@ -216,6 +244,8 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
if selected_guidance
else 0
)
+ if metadata:
+ reserved += json_size(metadata) - 1
if reserved >= FINDING_DETAILS_PREVIEW_BYTES:
continue
projected_core = bounded_json_value(
@@ -227,7 +257,7 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
break
ordered_guidance = dict(sorted(guidance.items(), key=lambda entry: bool(entry[1])))
bounded = bounded_json_value(
- {**projected_core, **ordered_guidance, **extras},
+ {**metadata, **projected_core, **ordered_guidance, **extras},
[FINDING_DETAILS_PREVIEW_BYTES],
max_depth=5,
)
diff --git a/plugins/codex-security/scripts/generate_in_scope_files.py b/plugins/codex-security/scripts/generate_in_scope_files.py
index 8144abaeb..f36e98fc5 100644
--- a/plugins/codex-security/scripts/generate_in_scope_files.py
+++ b/plugins/codex-security/scripts/generate_in_scope_files.py
@@ -195,10 +195,9 @@ def generate_diff_in_scope_files(
"""Reuse the existing diff selection without generating previews or duplicate worklists."""
from generate_rank_input import git_changed_paths, path_is_diff_excluded
from rank_preview import (
- DEFAULT_PREVIEW_BYTES,
TEXT_CODE_EXTENSIONS,
+ is_binary_file,
is_binary_sample,
- preview_for,
)
rows: list[bytes] = []
@@ -240,11 +239,7 @@ def generate_diff_in_scope_files(
)
if is_binary_sample(contents):
continue
- elif (
- path.is_symlink()
- or not path.is_file()
- or preview_for(path, DEFAULT_PREVIEW_BYTES)[1]
- ):
+ elif path.is_symlink() or not path.is_file() or is_binary_file(path):
continue
relative_path = relative.as_posix()
if "\n" in relative_path or "\r" in relative_path:
diff --git a/plugins/codex-security/scripts/generate_rank_input.py b/plugins/codex-security/scripts/generate_rank_input.py
index b589cb061..f6c5a4b33 100644
--- a/plugins/codex-security/scripts/generate_rank_input.py
+++ b/plugins/codex-security/scripts/generate_rank_input.py
@@ -40,7 +40,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
from generate_in_scope_files import windows_stream_component
from rank_preview import (
DEFAULT_PREVIEW_BYTES,
+ DEFAULT_PREVIEW_READ_BYTES,
TEXT_CODE_EXTENSIONS,
+ is_binary_sample,
preview_for,
preview_for_bytes,
)
@@ -701,7 +703,11 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
raise SystemExit(
f"Unable to read committed diff blob: {args.head}:{rel.as_posix()}"
)
- preview, is_binary = preview_for_bytes(rel, content, args.preview_bytes)
+ if is_binary_sample(content):
+ continue
+ preview, is_binary = preview_for_bytes(
+ rel, content[:DEFAULT_PREVIEW_READ_BYTES], args.preview_bytes
+ )
if is_binary:
continue
elif status != "D" and not path.is_symlink() and path.is_file():
diff --git a/plugins/codex-security/scripts/rank_preview.py b/plugins/codex-security/scripts/rank_preview.py
index 4ab2d3f02..25772b209 100644
--- a/plugins/codex-security/scripts/rank_preview.py
+++ b/plugins/codex-security/scripts/rank_preview.py
@@ -110,7 +110,7 @@ PHP_HEREDOC_RE = re.compile(r"<<<\s*['\"]?([A-Za-z_]\w*)['\"]?")
def _decode_source(data: bytes) -> str:
- encoding = "utf-16" if data.startswith(_UTF16_BOMS) else "utf-8"
+ encoding = "utf-16" if data.startswith(_UTF16_BOMS) else "utf-8-sig"
return data.decode(encoding, errors="ignore")
@@ -120,6 +120,14 @@ def is_binary_sample(data: bytes) -> bool:
return b"\0" in data
+def is_binary_file(path: Path) -> bool:
+ try:
+ with path.open("rb") as source:
+ return is_binary_sample(source.read(DEFAULT_PREVIEW_READ_BYTES))
+ except OSError:
+ return True
+
+
def compact_preview_line(line: str) -> str:
return " ".join(line.split())
@@ -235,40 +243,7 @@ def python_outline(text: str) -> list[str]:
return outline
-def javascript_regex_end(text: str, start: int) -> int | None:
- if start + 1 >= len(text) or text[start + 1] in {"/", "*"}:
- return None
- previous = start - 1
- while previous >= 0 and text[previous] in " \t\r":
- previous -= 1
- if previous >= 0 and text[previous] not in "=(:,[!&|?{};\n":
- prefix = text[max(0, previous - 8) : previous + 1]
- if not re.search(r"\b(?:case|return|throw)$", prefix):
- return None
-
- index = start + 1
- in_character_class = False
- while index < len(text):
- char = text[index]
- if char == "\n":
- return None
- if char == "\\" and index + 1 < len(text):
- index += 2
- continue
- if char == "[":
- in_character_class = True
- elif char == "]":
- in_character_class = False
- elif char == "/" and not in_character_class:
- index += 1
- while index < len(text) and text[index].isalpha():
- index += 1
- return index
- index += 1
- return None
-
-
-def mask_c_style_source(text: str, suffix: str) -> str:
+def mask_c_style_source(text: str, suffix: str) -> str | None:
masked: list[str] = []
index = 0
block_comment_depth = 0
@@ -408,12 +383,6 @@ def mask_c_style_source(text: str, suffix: str) -> str:
heredoc_terminator = heredoc_match.group(1)
index += len(token)
continue
- if suffix in JAVASCRIPT_EXTENSIONS and char == "/":
- regex_end = javascript_regex_end(text, index)
- if regex_end is not None:
- masked.extend(" " * (regex_end - index))
- index = regex_end
- continue
if char == "/" and next_char == "/":
masked.extend((" ", " "))
in_line_comment = True
@@ -429,6 +398,9 @@ def mask_c_style_source(text: str, suffix: str) -> str:
in_line_comment = True
index += 1
continue
+ if suffix in JAVASCRIPT_EXTENSIONS and char == "/":
+ # Regex literals and division need grammar context; sample the original source.
+ return None
if char in {'"', "'", "`"}:
quote = char
masked.append(" ")
@@ -693,7 +665,10 @@ def strip_leading_annotations(original: str, masked: str) -> tuple[str, str, lis
def brace_language_outline(text: str, suffix: str) -> list[str]:
original_lines = text.splitlines()
- masked_lines = mask_c_style_source(text, suffix).splitlines()
+ masked = mask_c_style_source(text, suffix)
+ if masked is None:
+ return []
+ masked_lines = masked.splitlines()
outline: dict[str, None] = {}
type_stack: list[tuple[str, int]] = []
function_depths: list[int] = []
@@ -944,7 +919,7 @@ def simple_language_outline(text: str, suffix: str) -> list[str]:
def json_outline(text: str) -> list[str]:
try:
parsed = json.loads(text)
- except (json.JSONDecodeError, RecursionError, MemoryError):
+ except (ValueError, RecursionError, MemoryError):
return []
if not isinstance(parsed, dict):
return []
@@ -995,7 +970,11 @@ def preview_for_bytes(path: Path, data: bytes, preview_bytes: int) -> tuple[str,
text = _decode_source(data)
outline = structural_outline(path, text)
preview_lines = select_preview_lines(outline or text.splitlines())
- return fit_preview_lines(preview_lines, preview_bytes), False
+ try:
+ preview = fit_preview_lines(preview_lines, preview_bytes)
+ except UnicodeEncodeError:
+ preview = fit_preview_lines(select_preview_lines(text.splitlines()), preview_bytes)
+ return preview, False
if __name__ == "__main__":
diff --git a/plugins/codex-security/tests/test_finding_preview.py b/plugins/codex-security/tests/test_finding_preview.py
index 4b094b49d..18efdee79 100644
--- a/plugins/codex-security/tests/test_finding_preview.py
+++ b/plugins/codex-security/tests/test_finding_preview.py
@@ -4,9 +4,120 @@ import json
import runpy
from pathlib import Path
+import pytest
+
FINDING_PREVIEW_SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "finding_preview.py"
+def test_compact_finding_fields_survive_verbose_code_evidence() -> None:
+ preview = runpy.run_path(str(FINDING_PREVIEW_SCRIPT))
+ compact = {
+ "severity": {"level": "high", "rationale": "Synthetic impact"},
+ "confidence": {"level": "high", "rationale": "Synthetic validation"},
+ "ruleId": "example-rule",
+ "status": "validated",
+ "identity": {"anchor": "example-finding"},
+ }
+ finding = {
+ **compact,
+ "codeEvidence": [
+ {
+ "id": "example-source",
+ "code": "synthetic code " * 1000,
+ "explanation": "synthetic explanation " * 1000,
+ }
+ ],
+ }
+
+ bounded = preview["bounded_finding_details"](finding)
+
+ assert {key: bounded[key] for key in compact} == compact
+ assert bounded["codeEvidence"]
+ assert (
+ len(json.dumps(bounded, separators=(",", ":")).encode())
+ <= preview["FINDING_DETAILS_PREVIEW_BYTES"]
+ )
+
+
+ ("metadata", "value"),
+ [
+ ("severity", {"rationale": "x" * 20_000, "level": "high"}),
+ ("confidence", {"rationale": "x" * 20_000, "level": "high"}),
+ ("identity", {"anchor": "x" * 20_000}),
+ ("ruleId", "x" * 20_000),
+ ("taxonomy", {"category": "x" * 1_800, "cwe": ["CWE-79"]}),
+ ("taxonomy", {"category": "x" * 20_000, "cwe": ["CWE-79"]}),
+ ],
+ ids=["severity", "confidence", "identity", "rule", "taxonomy-short", "taxonomy-long"],
+)
+def test_verbose_finding_metadata_retains_diagnostics_and_report(
+ metadata: str, value: object
+) -> None:
+ preview = runpy.run_path(str(FINDING_PREVIEW_SCRIPT))
+ finding = {
+ "identity": {"anchor": "example-finding"},
+ "status": "validated",
+ "writeup": {"reportPath": "findings/example/example.md"},
+ "rootCause": {"summary": "Synthetic root cause"},
+ "validation": {"summary": "Synthetic validation"},
+ "attackPath": {"summary": "Synthetic path"},
+ "codeEvidence": [{"id": "source", "code": "example()"}],
+ metadata: value,
+ }
+
+ bounded = preview["bounded_finding_details"](finding)
+
+ assert {key: bounded[key] for key in finding if key != metadata} == {
+ key: value for key, value in finding.items() if key != metadata
+ }
+ assert bounded[metadata]
+ if metadata == "taxonomy":
+ assert bounded[metadata]["cwe"] == value["cwe"]
+ if metadata in {"severity", "confidence"}:
+ assert bounded[metadata]["level"] == "high"
+ assert bounded[metadata]["rationale"]
+ assert (
+ len(json.dumps(bounded, separators=(",", ":")).encode())
+ <= preview["FINDING_DETAILS_PREVIEW_BYTES"]
+ )
+
+
[email protected]("length", [2_000, 20_000])
[email protected]("extra_metadata", [False, True])
+def test_multiple_verbose_metadata_sections_each_retain_a_preview(
+ length: int, extra_metadata: bool
+) -> None:
+ preview = runpy.run_path(str(FINDING_PREVIEW_SCRIPT))
+ finding = {
+ "severity": {"level": "high", "rationale": "s" * length},
+ "confidence": {"level": "high", "rationale": "c" * length},
+ "status": "validated",
+ "writeup": {"reportPath": "findings/example/example.md"},
+ "validation": {"summary": "Synthetic validation"},
+ }
+ if extra_metadata:
+ finding.update(
+ identity={"anchor": "i" * length},
+ ruleId="r" * length,
+ taxonomy={"category": "t" * length},
+ )
+
+ bounded = preview["bounded_finding_details"](finding)
+
+ assert set(bounded) == set(finding)
+ for key in ("severity", "confidence"):
+ assert bounded[key]["level"] == "high"
+ assert bounded[key]["rationale"]
+ assert bounded["status"] == finding["status"]
+ assert bounded["writeup"] == finding["writeup"]
+ assert bounded["validation"] == finding["validation"]
+ assert (
+ len(json.dumps(bounded, separators=(",", ":")).encode())
+ <= preview["FINDING_DETAILS_PREVIEW_BYTES"]
+ )
+
+
def test_nested_attack_path_stays_within_its_preview_budget() -> None:
preview = runpy.run_path(str(FINDING_PREVIEW_SCRIPT))
attack_path = {
diff --git a/plugins/codex-security/tests/test_generate_in_scope_files.py b/plugins/codex-security/tests/test_generate_in_scope_files.py
index 749f5d399..e0adceb91 100644
--- a/plugins/codex-security/tests/test_generate_in_scope_files.py
+++ b/plugins/codex-security/tests/test_generate_in_scope_files.py
@@ -531,6 +531,31 @@ def test_diff_inventory_keeps_changed_and_deleted_source_files(tmp_path: Path) -
]
[email protected]("mode", ["revisions", "local-patch"])
+def test_diff_inventory_only_classifies_source_bytes(tmp_path: Path, mode: str) -> None:
+ repository = make_repository(tmp_path)
+ git(repository, "add", ".")
+ git(repository, "commit", "-qm", "base")
+ base = git(repository, "rev-parse", "HEAD")
+ write_file(repository, "app/surrogate.json", b'{"\\ud800":1}')
+ write_file(repository, "app/integer.json", ('{"value":' + "1" * 4301 + "}").encode())
+ write_file(repository, "app/binary.py", b"x" * 4096 + b"\0")
+ arguments = ["--diff-base", base, "--diff-mode", mode]
+ if mode == "revisions":
+ git(repository, "add", ".")
+ git(repository, "commit", "-qm", "change")
+ arguments.extend(["--diff-head", "HEAD"])
+ output = tmp_path / "in_scope_files.txt"
+
+ result = run_inventory(repository, ".", output, arguments=arguments)
+
+ assert result.returncode == 0, result.stderr
+ assert output.read_text(encoding="utf-8").splitlines() == [
+ "app/integer.json",
+ "app/surrogate.json",
+ ]
+
+
@pytest.mark.parametrize("mode", ["revisions", "local-patch"])
def test_diff_inventory_includes_changed_terraform(tmp_path: Path, mode: str) -> None:
repository = make_repository(tmp_path)
diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.py
index d740a04b4..1c82a5a9d 100644
--- a/plugins/codex-security/tests/test_generate_rank_input.py
+++ b/plugins/codex-security/tests/test_generate_rank_input.py
@@ -192,25 +192,25 @@ void after() {}
assert rows[0]["preview"] == "function before\nfunction after"
-def test_make_repo_rank_input_keeps_python_with_ast_recursion(tmp_path: Path) -> None:
[email protected]("mode", ["repo", "revisions"])
+def test_rank_input_keeps_large_generated_python(tmp_path: Path, mode: str) -> None:
repo = tmp_path / "repo"
(repo / "src").mkdir(parents=True)
- source = "value = " + " + ".join(["x"] * 10000) + "\n"
+ initialize_repo(repo)
+ git(repo, "commit", "--allow-empty", "-qm", "base")
+ base = git(repo, "rev-parse", "HEAD")
+ source = "value = " + " + ".join(["x"] * 150000) + "\n"
(repo / "src" / "generated.py").write_text(source, encoding="utf-8")
(repo / "src" / "normal.py").write_text("value = 1\n", encoding="utf-8")
output = tmp_path / "rank_input.jsonl"
- run_cli(
- "make-repo-rank-input",
- "--repo",
- str(repo),
- "--scope",
- "src",
- "--preview-bytes",
- "128",
- "--out",
- str(output),
- )
+ if mode == "repo":
+ arguments = ["make-repo-rank-input", "--repo", str(repo), "--scope", "src"]
+ else:
+ git(repo, "add", ".")
+ git(repo, "commit", "-qm", "change")
+ arguments = ["make-diff-rank-input", "--repo", str(repo), "--base", base, "--head", "HEAD"]
+ run_cli(*arguments, "--preview-bytes", "128", "--out", str(output))
rows = read_jsonl(output)
assert [row["path"] for row in rows] == ["src/generated.py", "src/normal.py"]
@@ -221,6 +221,94 @@ def test_make_repo_rank_input_keeps_python_with_ast_recursion(tmp_path: Path) ->
assert rows[1]["preview"] == "value = 1"
[email protected]("mode", ["repo", "revisions", "local-patch"])
+ "source",
+ ['{"\\ud800":1}', '{"value":' + "1" * 4301 + "}"],
+ ids=["surrogate", "integer-limit"],
+)
+def test_rank_input_samples_json_without_a_renderable_outline(
+ tmp_path: Path, mode: str, source: str
+) -> None:
+ repo = tmp_path / "repo"
+ repo.mkdir()
+ initialize_repo(repo)
+ git(repo, "commit", "--allow-empty", "-qm", "base")
+ base = git(repo, "rev-parse", "HEAD")
+ (repo / "data.json").write_text(source, encoding="utf-8")
+ output = tmp_path / "rank.jsonl"
+ if mode == "repo":
+ arguments = ["make-repo-rank-input", "--repo", str(repo)]
+ else:
+ arguments = ["make-diff-rank-input", "--repo", str(repo), "--mode", mode, "--base", base]
+ if mode == "revisions":
+ git(repo, "add", ".")
+ git(repo, "commit", "-qm", "change")
+ arguments.extend(["--head", "HEAD"])
+
+ run_cli(*arguments, "--preview-bytes", "128", "--out", str(output))
+
+ rows = read_jsonl(output)
+ assert len(rows) == 1
+ assert rows[0]["path"] == "data.json"
+ assert rows[0]["preview"] == source[:128]
+
+
+def test_revision_previews_use_the_worktree_read_budget(tmp_path: Path) -> None:
+ repo = tmp_path / "repo"
+ repo.mkdir()
+ initialize_repo(repo)
+ git(repo, "commit", "--allow-empty", "-qm", "base")
+ base = git(repo, "rev-parse", "HEAD")
+ (repo / "large.py").write_text(
+ "def visible():\n pass\n" + "# comment\n" * 10000 + "def outside():\n pass\n",
+ encoding="utf-8",
+ )
+ git(repo, "add", ".")
+ git(repo, "commit", "-qm", "change")
+ output = tmp_path / "rank.jsonl"
+
+ run_cli(
+ "make-diff-rank-input",
+ "--repo",
+ str(repo),
+ "--base",
+ base,
+ "--head",
+ "HEAD",
+ "--out",
+ str(output),
+ )
+
+ assert read_jsonl(output)[0]["preview"] == "function visible()"
+
+
+def test_revision_rank_input_classifies_bytes_beyond_the_preview_sample(tmp_path: Path) -> None:
+ repo = tmp_path / "repo"
+ repo.mkdir()
+ initialize_repo(repo)
+ git(repo, "commit", "--allow-empty", "-qm", "base")
+ base = git(repo, "rev-parse", "HEAD")
+ (repo / "binary.py").write_bytes(b"# comment\n" * 10_000 + b"\0")
+ git(repo, "add", ".")
+ git(repo, "commit", "-qm", "change")
+ output = tmp_path / "rank.jsonl"
+
+ run_cli(
+ "make-diff-rank-input",
+ "--repo",
+ str(repo),
+ "--base",
+ base,
+ "--head",
+ "HEAD",
+ "--out",
+ str(output),
+ )
+
+ assert read_jsonl(output) == []
+
+
@pytest.mark.parametrize("scope", [".", "src", "src/large.py", "explicit", "overlap", "diff"])
def test_rank_input_bounds_large_text_reads(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, scope: str
diff --git a/plugins/codex-security/tests/test_rank_preview.py b/plugins/codex-security/tests/test_rank_preview.py
index ce5479ffb..fb24d771a 100644
--- a/plugins/codex-security/tests/test_rank_preview.py
+++ b/plugins/codex-security/tests/test_rank_preview.py
@@ -8,7 +8,7 @@ import pytest
PLUGIN_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PLUGIN_ROOT / "scripts"))
-from rank_preview import DEFAULT_PREVIEW_BYTES, preview_for, preview_for_bytes
+from rank_preview import DEFAULT_PREVIEW_BYTES, mask_c_style_source, preview_for, preview_for_bytes
@pytest.fixture(
@@ -578,16 +578,123 @@ def test_kotlin_companion_object_lists_factory_method(tmp_path: Path) -> None:
assert "method Service.visible" in preview
-def test_javascript_regex_literal_does_not_change_declaration_depth(tmp_path: Path) -> None:
- source = r"""class Service {
- pattern = /\{/;
- visible() {}
-}
-"""
+ "statement",
+ [
+ "yield /if(enabled)/;",
+ "yield /foo|if(enabled)/;",
+ "yield /foo:if(enabled)/;",
+ "yield /* pattern */ /else/.test(input) ? { ratio: count / total } : null;",
+ "yield /while(enabled)/;",
+ "const matcher = () => /if(enabled)/;",
+ "if (enabled) /{/.test(input);",
+ "if (enabled) {} else /{/.test(input);",
+ "return object.if(enabled) / { valueOf() { return count / total; } };",
+ "return value /* units */ / { valueOf() { return count / total; } };",
+ ],
+)
+def test_javascript_ambiguous_syntax_preserves_source(tmp_path: Path, statement: str) -> None:
+ source = f"function* patterns() {{ {statement} }} // note\nfunction authorize() {{}}"
- preview = generate_preview(tmp_path, "service.ts", source)
+ assert generate_preview(tmp_path, "example.js", source) == source
- assert "method Service.visible" in preview
+
+def test_javascript_yield_identifier_preserves_division_source(tmp_path: Path) -> None:
+ source = (
+ "function ratio() { var yield = 12; return yield / divisor; } // note\n"
+ "function authorize() {}"
+ )
+
+ assert generate_preview(tmp_path, "example.js", source) == source
+
+
[email protected]("operator", ["++", "--"])
+def test_typescript_postfix_assertion_preserves_division_source(
+ tmp_path: Path, operator: str
+) -> None:
+ source = (
+ "class Service {\n"
+ f"calculate() {{ return count{operator}! /* known value */ / "
+ "(() => { return count / total; })(); }\n"
+ "authorize() {}\n}"
+ )
+
+ assert generate_preview(tmp_path, "example.ts", source) == source
+
+
+ "suffix", [".cjs", ".cts", ".js", ".jsx", ".mjs", ".mts", ".ts", ".tsx", ".vue"]
+)
+def test_javascript_family_uses_source_for_slash_expressions(tmp_path: Path, suffix: str) -> None:
+ source = "function ratio() { return count / total; }\nfunction authorize() {}"
+
+ assert generate_preview(tmp_path, f"example{suffix}", source) == source
+
+
+ "body",
+ [
+ "return 1;",
+ 'return "a/b";',
+ "return 'a/b';",
+ "return `a/b`;",
+ "/* /if(enabled)/ */ return 1;",
+ "// /if(enabled)/\nreturn 1;",
+ ],
+)
+def test_javascript_simple_and_quoted_sources_keep_outlines(tmp_path: Path, body: str) -> None:
+ source = f"class Service {{\ncheck() {{ {body} }}\nauthorize() {{}}\n}}"
+
+ assert generate_preview(tmp_path, "example.js", source).splitlines() == [
+ "class Service",
+ "method Service.check",
+ "method Service.authorize",
+ ]
+
+
+def test_javascript_source_sample_respects_preview_budget(tmp_path: Path) -> None:
+ source = (
+ "function first() { return count / total; }\n"
+ + ("// " + "é" * 128 + "\n") * 100
+ + "function last() {}\n"
+ )
+ path = tmp_path / "example.js"
+ path.write_text(source, encoding="utf-8")
+
+ preview, binary = preview_for(path, DEFAULT_PREVIEW_BYTES)
+
+ assert binary is False
+ assert len(preview.encode("utf-8")) <= DEFAULT_PREVIEW_BYTES
+ assert "function first()" in preview
+ assert "function last()" in preview
+ assert "..." in preview
+
+
+def test_javascript_slash_fallback_does_not_scan_regex_suffixes() -> None:
+ class CountedSource(str):
+ reads = 0
+
+ def __getitem__(self, key):
+ self.reads += 1
+ return super().__getitem__(key)
+
+ reads = []
+ for repetitions in (1000, 2000):
+ source = CountedSource(",/[" * repetitions)
+ assert mask_c_style_source(source, ".js") is None
+ reads.append(source.reads)
+
+ assert reads[1] < 3 * reads[0]
+
+
+def test_python_preview_ignores_utf8_bom(tmp_path: Path) -> None:
+ source = b"def first():\n pass\n\ndef second():\n pass\n"
+ path = tmp_path / "example.py"
+ path.write_bytes(b"\xef\xbb\xbf" + source)
+
+ expected = preview_for_bytes(path, source, DEFAULT_PREVIEW_BYTES)
+ assert expected[0].splitlines() == ["function first()", "function second()"]
+ assert preview_for(path, DEFAULT_PREVIEW_BYTES) == expected
def test_javascript_preview_lists_class_field_arrow_handlers(tmp_path: Path) -> None: