OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
refactor(plugin): replace language parsers with source previews (#1391)
mldangelo-oaieb73cc0fa64f
diff --git a/plugins/codex-security/references/scan-contract.md b/plugins/codex-security/references/scan-contract.mdindex a24422196..06eef1789 100644--- a/plugins/codex-security/references/scan-contract.md+++ b/plugins/codex-security/references/scan-contract.md@@ -138,7 +138,7 @@ For a whole-repository Deep scan, keep `inventoryStrategy` as `repository`; repe| Inventory strategy | Meaning || ------------------ | --------------------------------------------------------- |-| `repository` | Repository-wide tracked source-like file inventory |+| `repository` | Repository-wide file inventory || `scoped_path` | Repository inventory constrained to requested paths || `diff` | Files selected from the reviewed Git change set || `directory` | Deterministic non-Git directory inventory |diff --git a/plugins/codex-security/scripts/generate_in_scope_files.py b/plugins/codex-security/scripts/generate_in_scope_files.pyindex 32873c8fa..3f26ff84f 100644--- a/plugins/codex-security/scripts/generate_in_scope_files.py+++ b/plugins/codex-security/scripts/generate_in_scope_files.py@@ -157,35 +157,6 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:return write_inventory(output, sorted(set(rows)))-def committed_changed_paths(repository: Path, base: str, head: str) -> list[tuple[Path, str]]:- result = git_command(- repository,- "diff",- "--ignore-submodules=all",- "--raw",- "-z",- "--diff-filter=ACMRD",- f"{base}..{head}",- text=False,- )- result.check_returncode()- fields = result.stdout.split(b"\0")- changed: list[tuple[Path, str]] = []- index = 0- while index < len(fields) - 1:- metadata = fields[index].split()- status = chr(metadata[-1][0])- index += 1- if status in {"C", "R"}:- index += 1- path = os.fsdecode(fields[index])- index += 1- selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]- if selected_mode != b"120000":- changed.append((repository / path, status))- return changed--def generate_diff_in_scope_files(repository: Path,base: str,@@ -194,13 +165,19 @@ def generate_diff_in_scope_files(output: Path,) -> int:"""Reuse the existing diff selection without generating previews or duplicate worklists."""- from generate_rank_input import git_changed_paths, path_is_diff_excluded+ from generate_rank_input import git_changed_paths, path_is_diff_excluded, run_git_changed_pathsfrom rank_preview import is_binary_filerows: list[bytes] = []try:changed = (- committed_changed_paths(repository, base, head)+ [+ (path, status)+ for path, status, selected_mode in run_git_changed_paths(+ repository, [f"{base}..{head}"]+ )+ if selected_mode != b"120000"+ ]if mode == "revisions"else git_changed_paths(repository, base, head, mode))diff --git a/plugins/codex-security/scripts/generate_rank_input.py b/plugins/codex-security/scripts/generate_rank_input.pyindex 2eaead3f0..09ccb0d32 100644--- a/plugins/codex-security/scripts/generate_rank_input.py+++ b/plugins/codex-security/scripts/generate_rank_input.py@@ -10,7 +10,7 @@ This script stays deliberately model-free:local working-tree patches.Candidate selection uses the existing path exclusions and binary detection,-not a language-extension allowlist. Unknown formats use sampled text previews.+not a language-extension allowlist. All formats use bounded source previews."""from __future__ import annotations@@ -153,7 +153,7 @@ def parse_args() -> argparse.Namespace:diff = subparsers.add_parser("make-diff-rank-input",- help="Create rank_input.jsonl from Git changed source-like files.",+ help="Create rank_input.jsonl from Git changed text files.",)diff.add_argument("--repo", required=True, help="Repository root.")diff.add_argument("--base", required=True, help="Git diff base revision.")@@ -388,12 +388,13 @@ def make_repo_scope_input(args: argparse.Namespace) -> None:print(f"Wrote {len(rows)} scoped paths to {output}")-def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:+def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str, bytes]]:+ """Return changed paths, statuses, and modes from the selected side of each change."""result = git_command(repo,"diff","--ignore-submodules=all",- "--name-status",+ "--raw","-z","--diff-filter=ACMRD",*diff_args,@@ -404,22 +405,26 @@ def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path,if fields and not fields[-1]:fields.pop()- changed: list[tuple[Path, str]] = []+ changed: list[tuple[Path, str, bytes]] = []index = 0while index < len(fields):- status = chr(fields[index][0])+ metadata = fields[index].split()+ status = chr(metadata[-1][0])index += 1if status in {"C", "R"}:index += 1path = repo / os.fsdecode(fields[index])index += 1- changed.append((path, status))+ selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]+ changed.append((path, status, selected_mode))return changeddef git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple[Path, str]]:if mode == "revisions":- return run_git_changed_paths(repo, [f"{base}..{head}"])+ return [+ (path, status) for path, status, _ in run_git_changed_paths(repo, [f"{base}..{head}"])+ ]if mode == "local-patch":unstaged = run_git_changed_paths(repo, [base])staged = run_git_changed_paths(repo, ["--cached", base])@@ -432,8 +437,8 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tupletext=False,)untracked.check_returncode()- combined = dict(staged)- combined.update(unstaged)+ combined = {path: status for path, status, _ in staged}+ combined.update((path, status) for path, status, _ in unstaged)combined.update((repo / os.fsdecode(relative), "A")for relative in untracked.stdout.split(b"\0")@@ -478,7 +483,7 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:if is_binary:continueif status != "D":- preview, _ = preview_for_bytes(rel, content, args.preview_bytes)+ preview, _ = preview_for_bytes(content, args.preview_bytes)elif not path.is_symlink() and path.is_file():try:path.resolve(strict=True).relative_to(repo)diff --git a/plugins/codex-security/scripts/rank_preview.py b/plugins/codex-security/scripts/rank_preview.pyindex f88072528..f095484bc 100644--- a/plugins/codex-security/scripts/rank_preview.py+++ b/plugins/codex-security/scripts/rank_preview.py@@ -1,11 +1,8 @@-"""Build deterministic structural or sampled source previews for rank inputs."""+"""Build bounded source previews for rank inputs regardless of language."""from __future__ import annotationsimport argparse-import ast-import json-import refrom bisect import bisect_rightfrom pathlib import Path@@ -15,39 +12,6 @@ PREVIEW_HEAD_LINES = 12PREVIEW_SAMPLE_LINES = 10_UTF16_BOMS = (b"\xff\xfe", b"\xfe\xff")-JAVASCRIPT_EXTENSIONS = {".cjs", ".cts", ".js", ".jsx", ".mjs", ".mts", ".ts", ".tsx", ".vue"}-JAVA_LIKE_EXTENSIONS = {- ".c",- ".cc",- ".cpp",- ".cs",- ".cxx",- ".h",- ".hh",- ".hpp",- ".hxx",- ".java",- ".mm",-}-BRACE_LANGUAGE_EXTENSIONS = {- *JAVASCRIPT_EXTENSIONS,- *JAVA_LIKE_EXTENSIONS,- ".dart",- ".go",- ".kt",- ".kts",- ".php",- ".rs",- ".scala",- ".swift",-}-NESTED_BLOCK_COMMENT_EXTENSIONS = {".kt", ".kts", ".rs", ".scala", ".swift"}-CPP_RAW_STRING_RE = re.compile(r'(?<![\w\x80-\U0010ffff])(?:u8|u|U|L)?R"([^\s()\\]{0,16})\(')-CSHARP_RAW_STRING_RE = re.compile(r'"{3,}')-RUST_RAW_STRING_RE = re.compile(r'(?:br|r)(#{0,16})"')-RUST_LIFETIME_RE = re.compile(r"'[A-Za-z_][A-Za-z0-9_]*")-PHP_HEREDOC_RE = re.compile(r"<<<\s*['\"]?([A-Za-z_]\w*)['\"]?")-def _decode_source(data: bytes) -> str:encoding = "utf-16" if data.startswith(_UTF16_BOMS) else "utf-8-sig"@@ -77,10 +41,6 @@ def is_binary_file(path: Path) -> bool:return False-def compact_preview_line(line: str) -> str:- return " ".join(line.split())--def truncate_utf8(text: str, max_bytes: int) -> str:if max_bytes <= 0:return ""@@ -91,9 +51,9 @@ def truncate_utf8(text: str, max_bytes: int) -> str:def select_preview_lines(lines: list[str]) -> list[str]:- compact = [value for line in lines if (value := compact_preview_line(line))]- head = compact[:PREVIEW_HEAD_LINES]- remainder = compact[PREVIEW_HEAD_LINES:]+ nonblank = [line for line in lines if line.strip()]+ head = nonblank[:PREVIEW_HEAD_LINES]+ remainder = nonblank[PREVIEW_HEAD_LINES:]if len(remainder) <= PREVIEW_SAMPLE_LINES:return [*head, *remainder]@@ -115,7 +75,7 @@ def fit_preview_lines(lines: list[str], max_bytes: int) -> str:def render(line_bytes: int) -> str:return "\n".join(- line if line == "..." else truncate_utf8(line, line_bytes).rstrip() for line in lines+ line if line == "..." else truncate_utf8(line, line_bytes) for line in lines)high = max(len(line.encode("utf-8")) for line in lines)@@ -123,773 +83,9 @@ def fit_preview_lines(lines: list[str], max_bytes: int) -> str:range(high + 1), max_bytes, key=lambda size: len(render(size).encode("utf-8")))best = render(limit - 1) if limit else ""- return best or truncate_utf8(full_preview, max_bytes)---def python_decorators(node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef) -> str:- values: list[str] = []- for decorator in node.decorator_list:- try:- rendered = compact_preview_line(ast.unparse(decorator))- except (AttributeError, RecursionError, ValueError):- continue- if rendered:- values.append(f"@{rendered}")- return " ".join(values)---def python_arguments(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str:- arguments = [argument.arg for argument in (*node.args.posonlyargs, *node.args.args)]- if node.args.vararg is not None:- arguments.append(f"*{node.args.vararg.arg}")- arguments.extend(argument.arg for argument in node.args.kwonlyargs)- if node.args.kwarg is not None:- arguments.append(f"**{node.args.kwarg.arg}")- return ", ".join(arguments)---def python_outline(text: str) -> list[str]:- try:- tree = ast.parse(text)- except (SyntaxError, ValueError, TypeError, MemoryError, RecursionError):- return []-- outline: list[str] = []- for node in tree.body:- if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):- prefix = python_decorators(node)- kind = "async function" if isinstance(node, ast.AsyncFunctionDef) else "function"- declaration = f"{kind} {node.name}({python_arguments(node)})"- outline.append(f"{prefix} {declaration}".strip())- elif isinstance(node, ast.ClassDef):- prefix = python_decorators(node)- bases: list[str] = []- for base in node.bases:- try:- bases.append(compact_preview_line(ast.unparse(base)))- except (AttributeError, RecursionError, ValueError):- continue- suffix = f"({', '.join(bases)})" if bases else ""- outline.append(f"{prefix} class {node.name}{suffix}".strip())- for member in node.body:- if not isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)):- continue- member_prefix = python_decorators(member)- kind = "async method" if isinstance(member, ast.AsyncFunctionDef) else "method"- declaration = f"{kind} {node.name}.{member.name}"- outline.append(f"{member_prefix} {declaration}".strip())- return outline---def mask_c_style_source(text: str, suffix: str) -> str | None:- masked: list[str] = []- index = 0- block_comment_depth = 0- in_line_comment = False- quote = ""- raw_terminator = ""- heredoc_terminator = ""- while index < len(text):- char = text[index]- next_char = text[index + 1] if index + 1 < len(text) else ""- if char == "\n":- masked.append(char)- in_line_comment = False- if quote and quote not in {"`", '"""', "'''", '@"'}:- quote = ""- index += 1- continue- if in_line_comment:- masked.append(" ")- index += 1- continue- if block_comment_depth:- if char == "/" and next_char == "*" and suffix in NESTED_BLOCK_COMMENT_EXTENSIONS:- masked.extend((" ", " "))- block_comment_depth += 1- index += 2- elif char == "*" and next_char == "/":- masked.extend((" ", " "))- block_comment_depth -= 1- index += 2- else:- masked.append(" ")- index += 1- continue- if heredoc_terminator:- if index == 0 or text[index - 1] == "\n":- # PHP identifiers allow every non-ASCII byte, including non-word characters.- terminator_match = re.compile(- rf"[ \t]*{heredoc_terminator}(?![A-Za-z0-9_\x80-\U0010ffff])"- ).match(text, index)- if terminator_match:- masked.extend(" " * (terminator_match.end() - index))- index = terminator_match.end()- heredoc_terminator = ""- continue- masked.append(" ")- index += 1- continue- if raw_terminator:- if text.startswith(raw_terminator, index):- masked.extend(" " * len(raw_terminator))- index += len(raw_terminator)- raw_terminator = ""- else:- masked.append(" ")- index += 1- continue- if quote in {'"""', "'''"}:- if text.startswith(quote, index):- masked.extend(" " * len(quote))- index += len(quote)- quote = ""- else:- masked.append(" ")- index += 1- continue- if quote == '@"':- masked.append(" ")- if char == '"' and next_char == '"':- masked.append(" ")- index += 2- else:- if char == '"':- quote = ""- index += 1- continue- if quote:- if char == "\\" and next_char:- masked.append(" ")- if next_char == "\n":- index += 1- else:- masked.append(" ")- index += 2- else:- masked.append(" ")- if char == quote:- quote = ""- index += 1- continue- if suffix in {".cc", ".cpp", ".cxx", ".h", ".hh", ".hpp", ".hxx", ".mm"}:- raw_match = CPP_RAW_STRING_RE.match(text, index)- if raw_match:- token = raw_match.group(0)- masked.extend(" " * len(token))- raw_terminator = f'){raw_match.group(1)}"'- index += len(token)- continue- if suffix == ".rs":- raw_match = RUST_RAW_STRING_RE.match(text, index)- if raw_match:- token = raw_match.group(0)- masked.extend(" " * len(token))- raw_terminator = f'"{raw_match.group(1)}'- index += len(token)- continue- if char == "'":- lifetime_match = RUST_LIFETIME_RE.match(text, index)- if lifetime_match:- token = lifetime_match.group(0)- following = text[index + len(token) : index + len(token) + 1]- if following != "'":- masked.extend(token)- index += len(token)- continue- if suffix == ".cs" and char == "@" and next_char == '"':- masked.extend((" ", " "))- quote = '@"'- index += 2- continue- if suffix == ".cs":- raw_match = CSHARP_RAW_STRING_RE.match(text, index)- if raw_match:- token = raw_match.group(0)- masked.extend(" " * len(token))- raw_terminator = token- index += len(token)- continue- if suffix == ".go" and char == "`":- masked.append(" ")- raw_terminator = "`"- index += 1- continue- triple_quote = text[index : index + 3]- if triple_quote in {'"""', "'''"}:- masked.extend((" ", " ", " "))- quote = triple_quote- index += 3- continue- if suffix == ".php" and text.startswith("<<<", index):- heredoc_match = PHP_HEREDOC_RE.match(text, index)- if heredoc_match:- token = heredoc_match.group(0)- masked.extend(" " * len(token))- heredoc_terminator = heredoc_match.group(1)- index += len(token)- continue- if char == "/" and next_char == "/":- masked.extend((" ", " "))- in_line_comment = True- index += 2- continue- if char == "/" and next_char == "*":- masked.extend((" ", " "))- block_comment_depth = 1- index += 2- continue- if char == "#" and suffix == ".php" and next_char != "[":- masked.append(" ")- in_line_comment = True- index += 1- continue- if suffix in JAVASCRIPT_EXTENSIONS and char == "/":- # Regex literals and division need grammar context; sample the original source.- return None- if char in {'"', "'", "`"}:- quote = char- masked.append(" ")- index += 1- continue- masked.append(char)- index += 1- return "".join(masked)---def match_type_declaration(line: str, suffix: str) -> tuple[str, str] | None:- match: re.Match[str] | None = None- if suffix in JAVASCRIPT_EXTENSIONS:- match = re.match(- r"^(?:(?:export|default|declare|abstract)\s+)*(class|interface|enum|namespace)\s+"- r"([A-Za-z_$][\w$]*)",- line,- )- elif suffix == ".java":- match = re.search(r"\b(class|interface|enum|record|@interface)\s+([A-Za-z_$][\w$]*)", line)- elif suffix == ".cs":- match = re.search(- r"\b(class|interface|struct|enum|record(?:\s+(?:class|struct))?)\s+"- r"(@?[A-Za-z_]\w*)",- line,- )- elif suffix == ".php":- match = re.match(- r"^(?:(?:abstract|final|readonly)\s+)*(class|interface|trait|enum)\s+"- r"([A-Za-z_]\w*)",- line,- )- elif suffix == ".go":- match = re.match(r"^type\s+([A-Za-z_]\w*)\s+(struct|interface)\b", line)- if match:- return match.group(2), match.group(1)- elif suffix in {".kt", ".kts"}:- companion_match = re.search(r"\bcompanion\s+object(?:\s+([A-Za-z_]\w*))?", line)- if companion_match:- return "object", companion_match.group(1) or "Companion"- match = re.search(- r"\b((?:(?:data|enum|sealed|annotation|value)\s+)?(?:class|interface|object))\s+"- r"([A-Za-z_]\w*)",- line,- )- elif suffix == ".scala":- match = re.match(- r"^(?:(?:case|sealed|abstract)\s+)*(class|trait|object|enum)\s+([A-Za-z_]\w*)", line- )- elif suffix == ".rs":- match = re.search(r"\b(struct|enum|trait|union)\s+([A-Za-z_]\w*)", line)- if not match:- impl_match = re.search(- r"\bimpl(?:\s*<[^>{}]*>)?\s+(?:[\w:<>]+\s+for\s+)?([A-Za-z_]\w*)",- line,- )- if impl_match:- return "impl", impl_match.group(1)- elif suffix == ".swift":- match = re.search(r"\b(class|struct|enum|protocol|actor|extension)\s+([A-Za-z_]\w*)", line)- elif suffix == ".dart":- match = re.search(r"\b(class|mixin|enum|extension)\s+([A-Za-z_]\w*)", line)- elif suffix in {".c", ".cc", ".cpp", ".cxx", ".h", ".hh", ".hpp", ".hxx", ".mm"}:- match = re.search(- r"\b(class|struct|union|enum(?:\s+class)?)\s+([A-Za-z_]\w*)",- line,- )- if not match:- return None- return match.group(1), match.group(2)---CONTROL_NAMES = {- "assert",- "catch",- "for",- "foreach",- "if",- "lock",- "new",- "return",- "sizeof",- "static_assert",- "switch",- "synchronized",- "throw",- "typeof",- "while",-}---def match_java_like_function(- line: str, suffix: str, type_name: str | None-) -> tuple[str, str] | None:- paren = line.find("(")- if paren < 0:- return None- before = line[:paren].rstrip()- if not before or "=" in before or before.startswith("#"):- return None- before = re.sub(r"<[^<>]*>$", "", before).rstrip()- name_match = re.search(- r"(~?[A-Za-z_$@][\w$@]*(?:(?:::|\.)~?[A-Za-z_$@][\w$@]*)?)$",- before,- )- if not name_match:- return None- qualified_name = name_match.group(1)- name = re.split(r"::|\.", qualified_name)[-1].lstrip("~")- if name in CONTROL_NAMES:- return None- prefix = before[: name_match.start()].strip()- if not prefix and name != (type_name or ""):- return None- if suffix in {- ".c",- ".cc",- ".cpp",- ".cxx",- ".h",- ".hh",- ".hpp",- ".hxx",- ".mm",- } and prefix.startswith(("typedef", "using")):- return None- if (- suffix in {".c", ".cc", ".cpp", ".cxx", ".h", ".hh", ".hpp", ".hxx", ".mm"}- and type_name is None- and line.endswith(";")- and not re.search(- r"(?:\b(?:auto|bool|char|consteval|constexpr|double|extern|float|inline|int|long|"- r"short|signed|static|unsigned|void)\b|[*&:]|<)",- prefix,- )- ):- return None- return ("method" if type_name else "function"), qualified_name---def match_function_declaration(- line: str, suffix: str, type_name: str | None-) -> tuple[str, str] | None:- if suffix in JAVASCRIPT_EXTENSIONS:- function_match = re.match(- r"^(?:(?:export|default|declare)\s+)*(async\s+)?function\s*\*?\s*"- r"([A-Za-z_$][\w$]*)\s*(?:<[^>{}]*>)?\s*\(",- line,- )- if function_match:- kind = "async function" if function_match.group(1) else "function"- return kind, function_match.group(2)- arrow_match = re.match(- r"^(?:(?:export|declare)\s+)*(?:const|let|var)\s+([A-Za-z_$][\w$]*)"- r"\s*(?::[^=]+)?=\s*(?:async\s+)?(?:function\b|\([^)]*\)\s*(?::[^=]+)?=>|"- r"[A-Za-z_$][\w$]*\s*=>)",- line,- )- if arrow_match:- return "function", arrow_match.group(1)- if type_name:- field_arrow_match = re.match(- r"^(?:(?:public|private|protected|static|abstract|override|readonly|declare|"- r"accessor)\s+)*(#?[A-Za-z_$][\w$]*)(?:[?!])?\s*(?::[^=]+)?=\s*"- r"(?:async\s+)?(?:\([^)]*\)\s*(?::[^=]+)?=>|[A-Za-z_$][\w$]*\s*=>)",- line,- )- if field_arrow_match:- return "method", field_arrow_match.group(1)- method_match = re.match(- r"^(?:(?:public|private|protected|static|abstract|async|override|readonly|"- r"declare|get|set)\s+)*(#?[A-Za-z_$][\w$]*|constructor)\s*"- r"(?:<[^>{}]*>)?\s*\(",- line,- )- if method_match and method_match.group(1) not in CONTROL_NAMES:- return "method", method_match.group(1)- return None- if suffix == ".php":- match = re.match(- r"^(?:(?:public|protected|private|static|final|abstract|readonly)\s+)*"- r"function\s*&?\s*([A-Za-z_]\w*)\s*\(",- line,- )- if match:- return ("method" if type_name else "function"), match.group(1)- return None- if suffix == ".go":- match = re.match(- r"^func\s+(?:\(([^)]*)\)\s*)?([A-Za-z_]\w*)\s*(?:\[[^]]+\]\s*)?\(",- line,- )- if match:- receiver = match.group(1)- name = match.group(2)- if receiver:- receiver_match = re.search(r"\*?([A-Za-z_]\w*)(?:\[[^]]*])?\s*$", receiver)- if receiver_match:- return "method", f"{receiver_match.group(1)}.{name}"- return "method", name- return ("method" if type_name else "function"), name- return None- if suffix in {".kt", ".kts"}:- match = re.search(r"\bfun\s+(?:<[^>{}]*>\s*)?(?:[\w.<>?]+\.)?([A-Za-z_]\w*)\s*\(", line)- if match:- return ("method" if type_name else "function"), match.group(1)- return None- if suffix == ".scala":- match = re.search(r"\bdef\s+([A-Za-z_]\w*)\b", line)- if match:- return ("method" if type_name else "function"), match.group(1)- return None- if suffix == ".rs":- match = re.search(- r"\b(?:async\s+)?(?:unsafe\s+)?(?:extern\s+\"[^\"]+\"\s+)?(?:const\s+)?"- r"fn\s+([A-Za-z_]\w*)",- line,- )- if match:- return ("method" if type_name else "function"), match.group(1)- return None- if suffix == ".swift":- match = re.search(r"\bfunc\s+([A-Za-z_]\w*)\s*(?:<[^>{}]*>)?\s*\(", line)- if match:- return ("method" if type_name else "function"), match.group(1)- if type_name and re.search(r"\binit\s*\(", line):- return "method", "init"- return None- if suffix == ".dart":- return match_java_like_function(line, suffix, type_name)- if suffix in JAVA_LIKE_EXTENSIONS:- return match_java_like_function(line, suffix, type_name)- return None---def is_annotation_line(line: str) -> bool:- return bool(- re.match(r"^@[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*(?:\(.*\))?$", line)- or re.match(r"^\[[A-Za-z_][^]]*]$", line)- or re.match(r"^#\[[A-Za-z_][^]]*]$", line)- )---LEADING_ANNOTATION_RE = re.compile(- r"^(?:@[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*(?:\([^)]*\))?|"- r"\[[A-Za-z_][^]]*]|#\[[A-Za-z_][^]]*])\s*"-)---def strip_leading_annotations(original: str, masked: str) -> tuple[str, str, list[str]]:- annotations: list[str] = []- if masked.startswith("@interface "):- return original, masked, annotations- while masked_match := LEADING_ANNOTATION_RE.match(masked):- original_match = LEADING_ANNOTATION_RE.match(original)- if original_match:- annotations.append(compact_preview_line(original_match.group(0)))- original = original[original_match.end() :].lstrip()- masked = masked[masked_match.end() :].lstrip()- return original, masked, annotations---def brace_language_outline(text: str, suffix: str) -> list[str]:- original_lines = text.splitlines()- masked = mask_c_style_source(text, suffix)- if masked is None:- return []- masked_lines = masked.splitlines()- outline: dict[str, None] = {}- type_stack: list[tuple[str, int]] = []- function_depths: list[int] = []- blocked_depths: list[int] = []- pending_type: str | None = None- pending_function = False- pending_annotations: list[str] = []- depth = 0-- def add(value: str) -> None:- decorated = f"{' '.join(pending_annotations[-2:])} {value}".strip()- outline[decorated] = None-- def next_lines_open_body(line_index: int) -> bool:- inspected = 0- for future_index in range(line_index + 1, len(masked_lines)):- future = masked_lines[future_index]- future_line = compact_preview_line(future)- if not future_line:- continue- inspected += 1- if "{" in future_line:- return True- if "}" in future_line or future_line.endswith(";") or inspected >= 6:- return False- return False-- for line_index, (original, masked) in enumerate(- zip(original_lines, masked_lines, strict=False)- ):- while function_depths and depth < function_depths[-1]:- function_depths.pop()- while type_stack and depth < type_stack[-1][1]:- type_stack.pop()- while blocked_depths and depth < blocked_depths[-1]:- blocked_depths.pop()-- masked_line = compact_preview_line(masked)- original_line = compact_preview_line(original)- opens = masked.count("{")- closes = masked.count("}")-- opened_pending_scope = False- if pending_type and "{" in masked:- type_stack.append((pending_type, depth + 1))- pending_type = None- opened_pending_scope = True- if pending_function and "{" in masked:- function_depths.append(depth + 1)- pending_function = False- opened_pending_scope = True-- if (- masked_line- and original_line- and is_annotation_line(original_line)- and not function_depths- and not blocked_depths- ):- pending_annotations.append(original_line)- elif masked_line and not function_depths and not blocked_depths:- _, masked_line, inline_annotations = strip_leading_annotations(- original_line, masked_line- )- pending_annotations.extend(inline_annotations)- current_type = type_stack[-1][0] if type_stack else None- direct_type_body = not type_stack or depth == type_stack[-1][1]- type_match = match_type_declaration(masked_line, suffix) if direct_type_body else None- if type_match:- kind, name = type_match- if (- current_type- and suffix in {".kt", ".kts"}- and kind == "object"- and name == "Companion"- ):- name = f"{current_type}.{name}"- add(f"{kind} {name}")- pending_annotations.clear()- if "{" in masked:- type_stack.append((name, depth + 1))- elif not masked_line.endswith(";") and next_lines_open_body(line_index):- pending_type = name- elif direct_type_body:- function_match = match_function_declaration(masked_line, suffix, current_type)- if function_match:- kind, name = function_match- qualified_name = (- f"{current_type}.{name}"- if current_type and "." not in name and "::" not in name- else name- )- add(f"{kind} {qualified_name}")- pending_annotations.clear()- if "{" in masked:- function_depths.append(depth + 1)- elif (- not masked_line.endswith(";")- and "=>" not in masked_line- and "=" not in masked_line- and next_lines_open_body(line_index)- ):- pending_function = True- else:- pending_annotations.clear()- is_transparent_container = bool(- re.match(- r"^(?:(?:export|inline)\s+)?namespace\b|"- r"^(?:pub\s+)?mod\b|^(?:declare\s+)?module\b|"- r"^(?:unsafe\s+)?extern\b",- masked_line,- )- )- if opens and not opened_pending_scope and not is_transparent_container:- blocked_depths.append(depth + 1)-- depth = max(0, depth + opens - closes)- while function_depths and depth < function_depths[-1]:- function_depths.pop()- while type_stack and depth < type_stack[-1][1]:- type_stack.pop()- while blocked_depths and depth < blocked_depths[-1]:- blocked_depths.pop()- return list(outline)---def ruby_outline(text: str) -> list[str]:- outline: dict[str, None] = {}- type_stack: list[tuple[int, str]] = []- function_indents: list[int] = []-- for raw_line in text.splitlines():- stripped = raw_line.strip()- if not stripped or stripped.startswith("#"):- continue- indent = len(raw_line.expandtabs(4)) - len(raw_line.lstrip().expandtabs(4))- while function_indents and indent <= function_indents[-1]:- function_indents.pop()- while type_stack and indent <= type_stack[-1][0]:- type_stack.pop()-- type_match = re.match(r"^(class|module)\s+([A-Z]\w*(?:::[A-Z]\w*)*)", stripped)- if type_match and not function_indents:- kind, name = type_match.groups()- outline[f"{kind} {name}"] = None- type_stack.append((indent, name))- continue-- function_match = re.match(r"^def\s+(?:self\.)?([^\s(]+)", stripped)- if function_match and not function_indents:- name = function_match.group(1)- if type_stack:- outline[f"method {type_stack[-1][1]}.{name}"] = None- else:- outline[f"function {name}"] = None- function_indents.append(indent)- return list(outline)---def simple_language_outline(text: str, suffix: str) -> list[str]:- outline: dict[str, None] = {}-- for raw_line in text.splitlines():- line = compact_preview_line(raw_line)- if not line:- continue- match: re.Match[str] | None = None- if suffix == ".py":- match = re.match(r"^(async\s+)?def\s+([A-Za-z_]\w*)\s*\(", line)- if match:- outline[- f"{'async function' if match.group(1) else 'function'} {match.group(2)}"- ] = None- continue- match = re.match(r"^class\s+([A-Za-z_]\w*)", line)- if match:- outline[f"class {match.group(1)}"] = None- elif suffix in {".ex", ".exs"}:- match = re.match(r"^defmodule\s+([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", line)- if match:- outline[f"module {match.group(1)}"] = None- continue- match = re.match(r"^(?:def|defp|defmacro)\s+([A-Za-z_]\w*[!?]?)", line)- if match:- outline[f"function {match.group(1)}"] = None- elif suffix == ".clj":- match = re.match(- r"^\((defn|defmacro|defprotocol|defrecord|deftype|defmulti|defmethod)\s+([^\s)]+)",- line,- )- if match:- outline[f"{match.group(1)} {match.group(2)}"] = None- elif suffix == ".lua":- match = re.match(r"^(?:local\s+)?function\s+([A-Za-z_]\w*(?:[.:][A-Za-z_]\w*)*)", line)- if match:- outline[f"function {match.group(1)}"] = None- elif suffix == ".sh":- match = re.match(r"^(?:function\s+)?([A-Za-z_]\w*)\s*\(\s*\)\s*\{?", line)- if match:- outline[f"function {match.group(1)}"] = None- elif suffix == ".hs":- match = re.match(r"^(data|newtype|type|class)\s+([A-Z]\w*)", line)- if match:- outline[f"{match.group(1)} {match.group(2)}"] = None- continue- match = re.match(r"^([a-z_]\w*)\s*::", line)- if match:- outline[f"function {match.group(1)}"] = None- elif suffix == ".proto":- match = re.match(r"^(message|service|enum)\s+([A-Za-z_]\w*)", line)- if match:- outline[f"{match.group(1)} {match.group(2)}"] = None- continue- match = re.match(r"^rpc\s+([A-Za-z_]\w*)\s*\(", line)- if match:- outline[f"rpc {match.group(1)}"] = None- elif suffix == ".graphql":- match = re.match(- r"^(type|interface|input|enum|union|scalar|directive)\s+([A-Za-z_]\w*)",- line,- )- if match:- outline[f"{match.group(1)} {match.group(2)}"] = None- continue- match = re.match(r"^(query|mutation|subscription)\s+([A-Za-z_]\w*)", line)- if match:- outline[f"{match.group(1)} {match.group(2)}"] = None- elif suffix == ".sql":- match = re.match(- r"^CREATE\s+(?:OR\s+REPLACE\s+)?(FUNCTION|PROCEDURE|TABLE|VIEW|TRIGGER)\s+"- r"([^\s(;]+)",- line,- flags=re.IGNORECASE,- )- if match:- outline[f"{match.group(1).lower()} {match.group(2)}"] = None- elif suffix in {".yaml", ".yml"}:- match = re.match(r"^([A-Za-z0-9_.-]+):(?:\s|$)", raw_line)- if match:- outline[f"key {match.group(1)}"] = None- elif suffix in {".toml", ".cfg"}:- match = re.match(r"^\[([^]]+)]$", line)- if match:- outline[f"section {match.group(1)}"] = None- return list(outline)---def json_outline(text: str) -> list[str]:- try:- parsed = json.loads(text)- except (ValueError, RecursionError, MemoryError):- return []- if not isinstance(parsed, dict):- return []- outline: list[str] = []- for key, value in parsed.items():- if isinstance(value, dict) and value:- children = ", ".join(str(child) for child in value)- outline.append(f"key {key} [{children}]")- else:- outline.append(f"key {key}")- return outline---def structural_outline(path: Path, text: str) -> list[str]:- suffix = path.suffix.lower()- if suffix == ".py":- return python_outline(text) or simple_language_outline(text, suffix)- if suffix == ".rb":- return ruby_outline(text)- if suffix in BRACE_LANGUAGE_EXTENSIONS:- return brace_language_outline(text, suffix)- if suffix == ".json":- return json_outline(text)- return simple_language_outline(text, suffix)+ if best.strip() not in {"", "..."}:+ return best+ return truncate_utf8(full_preview, max_bytes)def preview_for(@@ -907,24 +103,27 @@ def preview_for(data = sample + remainingexcept OSError:return "", True- return preview_for_bytes(path, data, preview_bytes)+ return preview_for_bytes(data, preview_bytes)-def preview_for_bytes(path: Path, data: bytes, preview_bytes: int) -> tuple[str, bool]:+def preview_for_bytes(data: bytes, preview_bytes: int) -> tuple[str, bool]:if is_binary_sample(data):return "", True- text = _decode_source(data)- outline = structural_outline(path, text)- preview_lines = select_preview_lines(outline or text.splitlines())- try:- preview = fit_preview_lines(preview_lines, preview_bytes)- except UnicodeEncodeError:- preview = fit_preview_lines(select_preview_lines(text.splitlines()), preview_bytes)- return preview, False+ lines = _decode_source(data).replace("\r\n", "\n").replace("\r", "\n").split("\n")+ start, end = 0, len(lines)+ while start < end and not lines[start].strip():+ start += 1+ while end > start and not lines[end - 1].strip():+ end -= 1+ lines = lines[start:end]+ text = "\n".join(lines)+ if len(text.encode("utf-8")) <= preview_bytes:+ return text, False+ return fit_preview_lines(select_preview_lines(lines), preview_bytes), Falseif __name__ == "__main__":parser = argparse.ArgumentParser(- description="Build structural or sampled previews for Codex Security rank inputs."+ description="Build bounded source previews for Codex Security rank inputs.")parser.parse_args()diff --git a/plugins/codex-security/skills/finding-discovery/SKILL.md b/plugins/codex-security/skills/finding-discovery/SKILL.mdindex d0794ff3c..1ee25a3f6 100644--- a/plugins/codex-security/skills/finding-discovery/SKILL.md+++ b/plugins/codex-security/skills/finding-discovery/SKILL.md@@ -33,7 +33,7 @@ When a running diff scan already supplies its file inventory through `list_codexFor a targeted code diff without an existing compact inventory:- Read `../security-scan/references/scan-artifacts-and-ledger.md`.-- Generate `rank_input.jsonl` deterministically from changed source-like files with `<python_command> <plugin_dir>/scripts/generate_rank_input.py make-diff-rank-input --repo <repo_root> --base <base> --mode revisions --head <head> --out <discovery_dir>/rank_input.jsonl` for PR, commit, and branch diffs, or `<python_command> <plugin_dir>/scripts/generate_rank_input.py make-diff-rank-input --repo <repo_root> --base <base> --mode local-patch --out <discovery_dir>/rank_input.jsonl` for a local patch.+- Generate `rank_input.jsonl` deterministically from changed text files regardless of extension, subject to the existing path exclusions, with `<python_command> <plugin_dir>/scripts/generate_rank_input.py make-diff-rank-input --repo <repo_root> --base <base> --mode revisions --head <head> --out <discovery_dir>/rank_input.jsonl` for PR, commit, and branch diffs, or `<python_command> <plugin_dir>/scripts/generate_rank_input.py make-diff-rank-input --repo <repo_root> --base <base> --mode local-patch --out <discovery_dir>/rank_input.jsonl` for a local patch.- Copy every diff row into `deep_review_input.jsonl` with `<plugin_dir>/scripts/launch_codex_security_mcp --helper copy-deep-review-input --rank-input <discovery_dir>/rank_input.jsonl --out <discovery_dir>/deep_review_input.jsonl`. On Windows, use the [PowerShell copy command](../security-scan/references/scan-artifacts-and-ledger.md#windows-worklist-copy) to preserve literal paths. Diff scans do not rank or drop changed files before deep review.- Add directly supporting files required to understand the changed security behavior only when repository evidence shows they are needed. Do not use them to broaden into unrelated repository-wide enumeration.- Deep-review every file in `deep_review_input.jsonl` using the shared scoped file-review rules.diff --git a/plugins/codex-security/skills/security-diff-scan/SKILL.md b/plugins/codex-security/skills/security-diff-scan/SKILL.mdindex 4fee03dd1..ca63a9fce 100644--- a/plugins/codex-security/skills/security-diff-scan/SKILL.md+++ b/plugins/codex-security/skills/security-diff-scan/SKILL.md@@ -34,7 +34,7 @@ For terminal scans without a `scanId`, generate the changed-file list with:<python_command> <plugin_dir>/scripts/generate_in_scope_files.py --repo <repo_root> --scope . --diff-base <base> --diff-head <head> --diff-mode <revisions|local-patch> --out <discovery_dir>/in_scope_files.txt```-The terminal inventory includes changed text files regardless of extension, subject to the existing path exclusions. Deleted files are classified from the base revision. Extensions select preview formats, not scan eligibility.+The inventory includes changed text files regardless of extension, subject to the existing path exclusions. Deleted files are classified from the base revision.Record candidates with `<plugin_dir>/scripts/launch_codex_security_mcp --helper normalize-candidates --input <candidate-source> --out <discovery_dir>/candidate_ledger.jsonl --repo-root <repo_root> --in-scope-files <discovery_dir>/in_scope_files.txt --allow-missing-in-scope`. Use the `.cmd` launcher on Windows. Add validation and attack-path decisions to that same file. Following `../../references/final-report.md`, assemble unsealed `scan-manifest.json`, `findings.json`, and `coverage.json` before running `finalize_scan_contract.py --scan-dir <scan_dir> --source-root <repo_root>`.diff --git a/plugins/codex-security/tests/source_cases.py b/plugins/codex-security/tests/source_cases.pyindex d92a261cf..8febfccc3 100644--- a/plugins/codex-security/tests/source_cases.py+++ b/plugins/codex-security/tests/source_cases.py@@ -1,7 +1,7 @@"""Shared source examples for inventory and ranking integration tests.-Add source-selection and preview regressions here so every inventory and ranking-mode uses the same examples, including source with unfamiliar or no extensions.+Keep these examples representative: selection and previews do not depend on+language-specific support. Every inventory and ranking mode uses the same cases."""from __future__ import annotations@@ -13,92 +13,24 @@ class SourceCase(NamedTuple):path: strbefore: strafter: str- # Sampled previews preserve the changed text; structural previews override it.- preview: str | None = NoneSOURCE_CASES = (SourceCase("scripts/entrypoint", "exec service --before", "exec service --after"),SourceCase("src/handler.unlisted", "render(before)", "render(after)"),SourceCase(- "src/raw.cpp",- "void before() {}",- 'void before() {}\nconst char* text = R"tag("{)tag";\nvoid after() {}',- "function before\nfunction after",+ "src/service.py",+ "def handle():\n return before",+ "def handle():\n return after",),SourceCase(- "src/raw.go",- "package sample\n\nfunc Before() {}",- "package sample\n\nfunc Before() {}\n\nconst Root = `C:\\`\n\nfunc After() {}",- "function Before\nfunction After",- ),- *(- SourceCase(- f"src/{name}.cs",- "class Before {}",- f"class Before {{}}\nclass Service {{\n string Text = {literal};\n"- " void Visible() {}\n}\nclass After {}",- "class Before\nclass Service\nmethod Service.Visible\nclass After",- )- for name, literal in (- ("three_quotes", '"""\n { "a": "b" }\n """'),- ("four_quotes", '""""\n { "a": """ }\n """"'),- ("five_quotes", '""""" contains """" and { """""'),- )- ),- SourceCase(- "infra/main.tf",- 'variable "enabled" { default = false }',- 'variable "enabled" { default = true }',+ "infra/policy.yaml",+ "access:\n enabled: false",+ "access:\n enabled: true",),SourceCase(- "ios/ViewController.m",- 'NSString *greeting = @"hello";',- 'NSString *greeting = @"goodbye";',- ),- SourceCase(- "ios/Bridge.mm",- "@implementation Bridge\n@end",- "@implementation Bridge\n@end\n// Changed.",- ),- SourceCase(- "ios/ViewController.h",- "@interface ViewController : NSObject\n@end",- "@interface ViewController : NSObject\n@end\n// Changed.",- ),- *(- SourceCase(- f"include/{name}",- "inline int before() { return 1; }",- "inline int after() { return 2; }",- "function after",- )- for name in ("base.h", "base.hpp", "lower.hh", "lower.hxx", "upper.HH", "upper.HXX")- ),- SourceCase("contracts/Vault.sol", "uint256 public limit = 1;", "uint256 public limit = 2;"),- SourceCase(- "contracts/Vault.vy",- "stored: public(uint256)",- "stored: public(uint256)\nowner: public(address)",- ),- SourceCase("src/routes/+page.svelte", "<p>before</p>", "<p>after</p>"),- *(- SourceCase(f"src/{filename}", content, content.replace("count = 0", "count = 1"))- for filename, content in (- (- "Counter.svelte",- '<script lang="ts">\nlet count = 0;\n</script>\n<button>{count}</button>',- ),- ("routes/profile.ejs", "<% const count = 0; %>\n<p><%= count %></p>"),- ("profile.EJS", "<% const count = 0; %>\n<p><%= count %></p>"),- ("show.html.erb", "<% count = 0 %>\n<p><%= count %></p>"),- ("card.phtml", "<?php $count = 0; ?>\n<p><?= $count ?></p>"),- ("index.jsp", "<% int count = 0; %>\n<p><%= count %></p>"),- ("list.jspx", "<jsp:scriptlet>int count = 0;</jsp:scriptlet>\n<p>${count}</p>"),- ("Index.cshtml", "@{ var count = 0; }\n<p>@count</p>"),- ("Counter.razor", "<p>@count</p>\n@code { int count = 0; }"),- ("Default.ASPX", "<% var count = 0; %>\n<p><%= count %></p>"),- ("Header.ascx", "<% var count = 0; %>\n<p><%= count %></p>"),- )+ "src/view.ejs",+ "<% const count = 0; %>\n<p><%= count %></p>",+ "<% const count = 1; %>\n<p><%= count %></p>",),)diff --git a/plugins/codex-security/tests/test_generate_in_scope_files.py b/plugins/codex-security/tests/test_generate_in_scope_files.pyindex 4ea5ea86a..1e4625ece 100644--- a/plugins/codex-security/tests/test_generate_in_scope_files.py+++ b/plugins/codex-security/tests/test_generate_in_scope_files.py@@ -558,8 +558,7 @@ def test_diff_inventory_only_classifies_source_bytes(tmp_path: Path, mode: str)git(repository, "add", ".")git(repository, "commit", "-qm", "base")base = git(repository, "rev-parse", "HEAD")- write_file(repository, "app/surrogate.json", b'{"\\ud800":1}')- write_file(repository, "app/integer.json", ('{"value":' + "1" * 4301 + "}").encode())+ write_file(repository, "app/text.unlisted", b"x" * (64 * 1024 + 1) + b"text")write_file(repository, "app/binary.py", b"x" * (64 * 1024 + 1) + b"\0")write_file(repository, "app/binary.unlisted", b"x" * (64 * 1024 + 1) + b"\0")arguments = ["--diff-base", base, "--diff-mode", mode]@@ -572,10 +571,7 @@ def test_diff_inventory_only_classifies_source_bytes(tmp_path: Path, mode: str)result = run_inventory(repository, ".", output, arguments=arguments)assert result.returncode == 0, result.stderr- assert output.read_text(encoding="utf-8").splitlines() == [- "app/integer.json",- "app/surrogate.json",- ]+ assert output.read_text(encoding="utf-8").splitlines() == ["app/text.unlisted"]@pytest.mark.parametrize("mode", ["revisions", "staged", "unstaged"])@@ -605,44 +601,6 @@ def test_diff_inventory_includes_source_cases(tmp_path: Path, mode: str) -> None)-def test_diff_inventory_keeps_every_javascript_module_extension(tmp_path: Path) -> None:- repository = make_repository(tmp_path)- git(repository, "add", ".")- git(repository, "commit", "-qm", "base")- base = git(repository, "rev-parse", "HEAD")-- for name in (- "app/loader.cjs",- "app/loader.mjs",- "app/loader.js",- "app/types.cts",- "app/types.mts",- "app/types.ts",- ):- write_file(repository, name, b"export const handler = 1;\n")- git(repository, "add", ".")- git(repository, "commit", "-qm", "change")- head = git(repository, "rev-parse", "HEAD")- output = tmp_path / "in_scope_files.txt"-- result = run_inventory(- repository,- ".",- output,- arguments=["--diff-base", base, "--diff-head", head],- )-- assert result.returncode == 0, result.stderr- assert output.read_text(encoding="utf-8").splitlines() == [- "app/loader.cjs",- "app/loader.js",- "app/loader.mjs",- "app/types.cts",- "app/types.mts",- "app/types.ts",- ]--def test_diff_inventory_combines_staged_and_unstaged_changes(tmp_path: Path) -> None:repository = make_repository(tmp_path)git(repository, "add", ".")diff --git a/plugins/codex-security/tests/test_generate_rank_input.py b/plugins/codex-security/tests/test_generate_rank_input.pyindex 3baea5ac9..47eb468c1 100644--- a/plugins/codex-security/tests/test_generate_rank_input.py+++ b/plugins/codex-security/tests/test_generate_rank_input.py@@ -112,39 +112,6 @@ def test_rank_input_keeps_large_generated_python(tmp_path: Path, mode: str) -> Nassert rows[1]["preview"] == "value = 1"[email protected]("mode", ["repo", "revisions", "local-patch"])- "source",- ['{"\\ud800":1}', '{"value":' + "1" * 4301 + "}"],- ids=["surrogate", "integer-limit"],-)-def test_rank_input_samples_json_without_a_renderable_outline(- tmp_path: Path, mode: str, source: str-) -> None:- repo = tmp_path / "repo"- repo.mkdir()- initialize_repo(repo)- git(repo, "commit", "--allow-empty", "-qm", "base")- base = git(repo, "rev-parse", "HEAD")- (repo / "data.json").write_text(source, encoding="utf-8")- output = tmp_path / "rank.jsonl"- if mode == "repo":- arguments = ["make-repo-rank-input", "--repo", str(repo)]- else:- arguments = ["make-diff-rank-input", "--repo", str(repo), "--mode", mode, "--base", base]- if mode == "revisions":- git(repo, "add", ".")- git(repo, "commit", "-qm", "change")- arguments.extend(["--head", "HEAD"])-- run_cli(*arguments, "--preview-bytes", "128", "--out", str(output))-- rows = read_jsonl(output)- assert len(rows) == 1- assert rows[0]["path"] == "data.json"- assert rows[0]["preview"] == source[:128]--def test_revision_previews_use_the_worktree_read_budget(tmp_path: Path) -> None:repo = tmp_path / "repo"repo.mkdir()@@ -171,7 +138,10 @@ def test_revision_previews_use_the_worktree_read_budget(tmp_path: Path) -> None:str(output),)- assert read_jsonl(output)[0]["preview"] == "function visible()"+ preview = read_jsonl(output)[0]["preview"]+ assert preview.startswith("def visible():\n pass\n# comment")+ assert len(preview.encode("utf-8")) <= 1024+ assert "outside" not in previewdef test_revision_rank_input_classifies_bytes_beyond_the_preview_sample(tmp_path: Path) -> None:@@ -207,7 +177,11 @@ def test_rank_input_streams_classification_and_bounds_previews(repo = tmp_path / "repo"(repo / "src").mkdir(parents=True)source = repo / "src" / "large.py"- source.write_bytes(b"def visible():\n pass\n" + b"# source comment\n" * (128 * 1024))+ source.write_bytes(+ b"def visible():\n pass\n"+ + b"# source comment\n" * (128 * 1024)+ + b"def outside():\n pass\n"+ )output = tmp_path / "rank_input.jsonl"arguments = ["make-repo-rank-input", "--repo", str(repo), "--out", str(output)]if scope in {"explicit", "overlap"}:@@ -251,7 +225,10 @@ def test_rank_input_streams_classification_and_bounds_previews(rows = read_jsonl(output)assert len(rows) == 1assert rows[0]["path"] == "src/large.py"- assert rows[0]["preview"] == "function visible()"+ preview = rows[0]["preview"]+ assert preview.startswith("def visible():\n pass\n# source comment")+ assert len(preview.encode("utf-8")) <= 1024+ assert "outside" not in preview@pytest.mark.parametrize("mode", ["repo", "revisions", "staged", "unstaged"])@@ -296,7 +273,7 @@ def test_rank_input_includes_source_cases(tmp_path: Path, mode: str) -> None:{"path": case.path,"area": "." if mode == "repo" else "diff",- "preview": case.after if case.preview is None else case.preview,+ "preview": case.after,}for case in sorted(SOURCE_CASES, key=lambda case: case.path)]diff --git a/plugins/codex-security/tests/test_rank_preview.py b/plugins/codex-security/tests/test_rank_preview.pyindex 1e92237bb..61ab58b3c 100644--- a/plugins/codex-security/tests/test_rank_preview.py+++ b/plugins/codex-security/tests/test_rank_preview.py@@ -8,7 +8,7 @@ import pytestPLUGIN_ROOT = Path(__file__).resolve().parents[1]sys.path.insert(0, str(PLUGIN_ROOT / "scripts"))-from rank_preview import DEFAULT_PREVIEW_BYTES, mask_c_style_source, preview_for, preview_for_bytes+from rank_preview import DEFAULT_PREVIEW_BYTES, preview_for, preview_for_bytes@pytest.fixture(@@ -30,8 +30,8 @@ def test_preview_for_decodes_bom_marked_utf16(expected = (source.strip(), False)assert preview_for(path, DEFAULT_PREVIEW_BYTES) == expected- assert preview_for_bytes(path, data, DEFAULT_PREVIEW_BYTES) == expected- assert preview_for_bytes(path, source.encode("utf-8"), DEFAULT_PREVIEW_BYTES) == expected+ assert preview_for_bytes(data, DEFAULT_PREVIEW_BYTES) == expected+ assert preview_for_bytes(source.encode("utf-8"), DEFAULT_PREVIEW_BYTES) == expected@pytest.mark.parametrize(@@ -49,7 +49,7 @@ def test_preview_for_rejects_binary_source_bytes(tmp_path: Path, data: bytes) ->path.write_bytes(data)assert preview_for(path, DEFAULT_PREVIEW_BYTES) == ("", True)- assert preview_for_bytes(path, data, DEFAULT_PREVIEW_BYTES) == ("", True)+ assert preview_for_bytes(data, DEFAULT_PREVIEW_BYTES) == ("", True)def test_preview_for_handles_utf16_surrogate_at_sample_boundary(@@ -62,7 +62,7 @@ def test_preview_for_handles_utf16_surrogate_at_sample_boundary(path.write_bytes(data)assert preview_for(path, 8192) == (source.strip(), False)- assert preview_for_bytes(path, data, 8192) == (source.strip(), False)+ assert preview_for_bytes(data, 8192) == (source.strip(), False)def test_preview_for_bounds_utf16_source_at_incomplete_character(@@ -86,7 +86,7 @@ def test_preview_for_ignores_incomplete_utf16_tail(path.write_bytes(data)assert preview_for(path, DEFAULT_PREVIEW_BYTES) == (source, False)- assert preview_for_bytes(path, data, DEFAULT_PREVIEW_BYTES) == (source, False)+ assert preview_for_bytes(data, DEFAULT_PREVIEW_BYTES) == (source, False)def test_preview_for_does_not_fully_read_a_large_binary(tmp_path: Path) -> None:@@ -150,7 +150,7 @@ def test_preview_for_rejects_nonpositive_read_limit(max_read_bytes: int) -> None@pytest.mark.parametrize(("filename", "prefix", "suffix", "expected"),[- ("source.py", b"def visible():\n value = (", b"1)\n", "function visible"),+ ("source.py", b"def visible():\n value = (", b"1)\n", "def visible():\n value = ("),("source.css", b"body { color: red; }\n\xf0\x9f", b"\x98\x80", "body { color: red; }"),("source.css", b"body { color: red; }\n", b"\0binary", "body { color: red; }"),],@@ -161,8 +161,7 @@ def test_preview_for_uses_only_the_bounded_prefix(source = tmp_path / filenamesource.write_bytes(prefix + suffix)- for _ in range(2):- assert preview_for(source, 128, max_read_bytes=len(prefix)) == (expected, False)+ assert preview_for(source, 128, max_read_bytes=len(prefix)) == (expected, False)def generate_preview(@@ -179,480 +178,62 @@ def generate_preview(return preview-def test_python_preview_lists_top_level_symbols_and_direct_methods(tmp_path: Path) -> None:- imports = "\n".join(f"import package_{index}" for index in range(30))- source = f"""{imports}-[email protected]("/login")-async def login(request, token=None):- def nested_helper():- return None- return nested_helper()--class UserService(BaseService):- @classmethod- def create(cls, config):- return cls(config)-- async def refresh(self):- return None-"""-- preview = generate_preview(tmp_path, "service.py", source)-- assert "@router.post('/login') async function login(request, token)" in preview- assert "class UserService(BaseService)" in preview- assert "@classmethod method UserService.create" in preview- assert "async method UserService.refresh" in preview- assert "package_" not in preview- assert "nested_helper" not in preview+def test_small_source_is_complete(tmp_path: Path) -> None:+ source = (+ "\n\n# Source context\n"+ + "\n".join(f"setting_{index} = {index}" for index in range(24))+ + '\n\ndef handle(request):\n text = "two spaces"\n'+ + ' return subprocess.run(request.args["cmd"], shell=True)\n\n'+ )+ assert len(source.strip().encode("utf-8")) < DEFAULT_PREVIEW_BYTES+ preview = generate_preview(tmp_path, "entrypoint", source)- ("filename", "source", "expected"),- [- (- "controller.js",- """if (enabled) {- function conditionalOnly() {}-}-export class UserController {- async login(request) {- function nestedHelper() {}- return request;- }-}-export const verifyToken = (token) => token;-""",- ("class UserController", "method UserController.login", "function verifyToken"),- ),- (- "store.ts",- """export interface TokenStore {- get(id: string): Token;-}-export function loadStore(): TokenStore { throw new Error(); }-""",- ("interface TokenStore", "method TokenStore.get", "function loadStore"),- ),- (- "UserService.java",- """public class UserService {- @PostMapping("/login") public User login(String name) { return null; }-}-""",- ("class UserService", "method UserService.login"),- ),- (- "UserController.cs",- """public class UserController {- [HttpPost("login")] public T Login<T>(User user) { return default(T); }-}-""",- ("class UserController", "method UserController.Login"),- ),- (- "user_service.rb",- """class FirstService- def call(user)- true- end-end-class SecondService- def call(user)- true- end-end-""",- (- "class FirstService",- "method FirstService.call",- "class SecondService",- "method SecondService.call",- ),- ),- (- "UserService.php",- """<?php-final class UserService {- # A comment brace must not change declaration depth: {- public function authenticate($user) { return true; }-}-""",- ("class UserService", "method UserService.authenticate"),- ),- (- "user_service.go",- """package service-type UserService struct {}-func (service *UserService) Authenticate(user User) bool { return true }-""",- ("struct UserService", "method UserService.Authenticate"),- ),- (- "user_service.cpp",- """Widget widget(options);-class UserService {-public:- bool authenticate(const User& user) { return true; }-};-bool verify_token(const Token& token) { return true; }-""",- ("class UserService", "method UserService.authenticate", "function verify_token"),- ),- (- "UserService.kt",- """class UserService {- fun authenticate(user: User): Boolean { return true }-}-""",- ("class UserService", "method UserService.authenticate"),- ),- (- "UserService.scala",- """class UserService {- def authenticate(user: User): Boolean = { true }-}-""",- ("class UserService", "method UserService.authenticate"),- ),- (- "user_service.rs",- """pub fn borrow<'a>() {}-struct UserService {}-impl UserService {- pub fn authenticate(&self, user: User) -> bool { true }-}-""",- (- "function borrow",- "struct UserService",- "impl UserService",- "method UserService.authenticate",- ),- ),- (- "UserService.swift",- """final class UserService {- func authenticate(user: User) -> Bool { return true }-}-""",- ("class UserService", "method UserService.authenticate"),- ),- (- "user_service.ex",- """defmodule UserService do- def authenticate(user), do: true-end-""",- ("module UserService", "function authenticate"),- ),- (- "service.clj",- """(defprotocol UserService- (authenticate [service user]))-(defn verify-token [token] true)-""",- ("defprotocol UserService", "defn verify-token"),- ),- (- "service.proto",- """service UserService {- rpc Authenticate (User) returns (Result);-}-message User {}-""",- ("service UserService", "rpc Authenticate", "message User"),- ),- (- "schema.graphql",- """type User { id: ID! }-type Query { user(id: ID!): User }-query CurrentUser { user(id: "me") { id } }-""",- ("type User", "type Query", "query CurrentUser"),- ),- (- "schema.sql",- """CREATE TABLE users (id BIGINT PRIMARY KEY);-CREATE PROCEDURE rotate_tokens() BEGIN SELECT 1; END;-""",- ("table users", "procedure rotate_tokens"),- ),- (- "service.json",- '{"service":{"host":"localhost","port":443},"enabled":true}',- ("key service [host, port]", "key enabled"),- ),- ],-)-def test_enterprise_language_previews_list_declarations(- tmp_path: Path, filename: str, source: str, expected: tuple[str, ...]-) -> None:- preview = generate_preview(tmp_path, filename, source, preview_bytes=4096)-- for declaration in expected:- assert declaration in preview- assert "nestedHelper" not in preview- assert "conditionalOnly" not in preview- assert "function widget" not in preview--[email protected]("suffix", [".h", ".hpp", ".hh", ".hxx", ".HH", ".HXX"])-def test_cpp_headers_use_structural_previews(tmp_path: Path, suffix: str) -> None:- source = """typedef void Callback();-Widget widget(options);-template <typename T>-class Box {-public:- Box() {}- T get() const { return value; }-};-inline int answer() { return 42; }-"""- path = tmp_path / f"box{suffix}"- preview = generate_preview(tmp_path, path.name, source)-- assert "class Box" in preview- assert "method Box.Box" in preview- assert "method Box.get" in preview- assert "function answer" in preview- assert "function Callback" not in preview- assert "function widget" not in preview- assert preview_for_bytes(path, source.encode("utf-8"), DEFAULT_PREVIEW_BYTES) == (+ assert preview == source.strip()+ assert preview_for_bytes(source.encode("utf-8"), DEFAULT_PREVIEW_BYTES) == (preview,False,)- "suffix", [".cc", ".cpp", ".cxx", ".h", ".hh", ".hpp", ".hxx", ".mm", ".CPP"]-)-def test_cpp_raw_string_does_not_hide_following_declarations(tmp_path: Path, suffix: str) -> None:- source = """void before() {}-const char* text = R"tag("{)tag";-class Service {-public:- const char* value() { return R"("{)"; }- void visible() {}-};-void after() {}-"""-- preview = generate_preview(tmp_path, f"sample{suffix}", source)-- assert preview.splitlines() == [- "function before",- "class Service",- "method Service.value",- "method Service.visible",- "function after",- ]--[email protected]("prefix", ["", "u8", "u", "U", "L"])[email protected]("delimiter", ["", "tag", "abcdefghijklmnop"])-def test_cpp_raw_string_body_is_not_code(tmp_path: Path, prefix: str, delimiter: str) -> None:- source = f'''void before() {{}}-const auto text = {prefix}R"{delimiter}(-void fake() {{}}-" {{ /*-){delimiter}";-void after() {{}}-'''-- preview = generate_preview(tmp_path, "sample.cpp", source)-- assert preview.splitlines() == ["function before", "function after"]--[email protected]("macro", ["ERROR", "FORMAT_u8R", "x\u0301R"])-def test_cpp_macro_before_string_is_not_a_raw_string(tmp_path: Path, macro: str) -> None:- source = f"""#define {macro} "error: "-void before() {{}}-const char* text = {macro}"(connection failed";-void after() {{}}-"""-- preview = generate_preview(tmp_path, "sample.cpp", source)-- assert preview.splitlines() == ["function before", "function after"]---def test_expression_bodied_function_does_not_consume_next_type_body(tmp_path: Path) -> None:- source = """fun answer(): Int = 42-class Service {- fun run() {}-}-"""-- preview = generate_preview(tmp_path, "Service.kt", source)-- assert "function answer" in preview- assert "class Service" in preview- assert "method Service.run" in preview---def test_protocol_requirement_does_not_consume_following_function(tmp_path: Path) -> None:- source = """protocol Required {- func run()-}-func visible() {}-"""-- preview = generate_preview(tmp_path, "Required.swift", source)-- assert "method Required.run" in preview- assert "function visible" in preview---def test_multiline_string_and_commented_annotation_do_not_change_depth(tmp_path: Path) -> None:- source = '''public class Visible {- String template = """- { notABlock }- """;- /*- * @Route("/not-a-route")- */- public void run() {}-}-'''-- preview = generate_preview(tmp_path, "Visible.java", source)-- assert "class Visible" in preview- assert "method Visible.run" in preview- assert "Route" not in preview--- ("filename", "source", "expected"),- [- (- "api.rs",- """mod api {- pub fn handle() {}-}-""",- "function handle",- ),- (- "api.ts",- """declare module "api" {- export function handle(): void;-}-""",- "function handle",- ),- (- "api.cpp",- """extern "C" {- void handle() {}-}-""",- "function handle",- ),- ],-)-def test_namespace_like_containers_expose_declarations(- tmp_path: Path, filename: str, source: str, expected: str-) -> None:- preview = generate_preview(tmp_path, filename, source)-- assert expected in preview---def test_kotlin_companion_object_lists_factory_method(tmp_path: Path) -> None:- source = """class Service {- companion object {- fun create() {}- }- fun visible() {}-}-"""-- preview = generate_preview(tmp_path, "Service.kt", source)-- assert "object Service.Companion" in preview- assert "method Service.Companion.create" in preview- assert "method Service.visible" in preview--- "statement",- [- "yield /if(enabled)/;",- "yield /foo|if(enabled)/;",- "yield /foo:if(enabled)/;",- "yield /* pattern */ /else/.test(input) ? { ratio: count / total } : null;",- "yield /while(enabled)/;",- "const matcher = () => /if(enabled)/;",- "if (enabled) /{/.test(input);",- "if (enabled) {} else /{/.test(input);",- "return object.if(enabled) / { valueOf() { return count / total; } };",- "return value /* units */ / { valueOf() { return count / total; } };",- ],-)-def test_javascript_ambiguous_syntax_preserves_source(tmp_path: Path, statement: str) -> None:- source = f"function* patterns() {{ {statement} }} // note\nfunction authorize() {{}}"-- assert generate_preview(tmp_path, "example.js", source) == source-[email protected]("large", [False, True], ids=["complete", "sampled"])+def test_source_preview_preserves_retained_whitespace(tmp_path: Path, large: bool) -> None:+ retained = ' value = "two spaces\tand\u0085a separator" \n return value '+ source = "\n \t\n" + retained+ if large:+ source += "\n# " + "x" * DEFAULT_PREVIEW_BYTES+ source += "\n \t\n"-def test_javascript_yield_identifier_preserves_division_source(tmp_path: Path) -> None:- source = (- "function ratio() { var yield = 12; return yield / divisor; } // note\n"- "function authorize() {}"- )+ preview = generate_preview(tmp_path, "source.py", source)- assert generate_preview(tmp_path, "example.js", source) == source+ if large:+ assert preview.startswith(retained + "\n")+ assert len(preview.encode("utf-8")) <= DEFAULT_PREVIEW_BYTES+ else:+ assert preview == retained[email protected]("operator", ["++", "--"])-def test_typescript_postfix_assertion_preserves_division_source(- tmp_path: Path, operator: str-) -> None:- source = (- "class Service {\n"- f"calculate() {{ return count{operator}! /* known value */ / "- "(() => { return count / total; })(); }\n"- "authorize() {}\n}"- )[email protected]("newline", ["\n", "\r\n", "\r"], ids=["lf", "crlf", "cr"])+def test_source_preview_normalizes_newlines(tmp_path: Path, newline: str) -> None:+ expected = 'def handle():\n text = "two spaces"\n\n return text'+ data = (expected.replace("\n", newline) + newline).encode("utf-8")+ path = tmp_path / "source.py"+ path.write_bytes(data)- assert generate_preview(tmp_path, "example.ts", source) == source+ assert preview_for(path, DEFAULT_PREVIEW_BYTES) == (expected, False)+ assert preview_for_bytes(data, DEFAULT_PREVIEW_BYTES) == (expected, False)- "suffix", [".cjs", ".cts", ".js", ".jsx", ".mjs", ".mts", ".ts", ".tsx", ".vue"]-)-def test_javascript_family_uses_source_for_slash_expressions(tmp_path: Path, suffix: str) -> None:- source = "function ratio() { return count / total; }\nfunction authorize() {}"[email protected]("preview_bytes", [0, -1])+def test_nonpositive_preview_budget_returns_no_source(preview_bytes: int) -> None:+ assert preview_for_bytes(b"value = 1\n", preview_bytes) == ("", False)- assert generate_preview(tmp_path, f"example{suffix}", source) == source+def test_empty_source_preview(tmp_path: Path) -> None:+ assert generate_preview(tmp_path, "empty.py", "\n \t\n") == ""- "body",- [- "return 1;",- 'return "a/b";',- "return 'a/b';",- "return `a/b`;",- "/* /if(enabled)/ */ return 1;",- "// /if(enabled)/\nreturn 1;",- ],-)-def test_javascript_simple_and_quoted_sources_keep_outlines(tmp_path: Path, body: str) -> None:- source = f"class Service {{\ncheck() {{ {body} }}\nauthorize() {{}}\n}}"-- assert generate_preview(tmp_path, "example.js", source).splitlines() == [- "class Service",- "method Service.check",- "method Service.authorize",- ]--def test_javascript_source_sample_respects_preview_budget(tmp_path: Path) -> None:+def test_large_source_preview_preserves_head_and_tail(tmp_path: Path) -> None:source = ("function first() { return count / total; }\n"+ ("// " + "é" * 128 + "\n") * 100@@ -670,166 +251,20 @@ def test_javascript_source_sample_respects_preview_budget(tmp_path: Path) -> Nonassert "..." in preview-def test_javascript_slash_fallback_does_not_scan_regex_suffixes() -> None:- class CountedSource(str):- reads = 0-- def __getitem__(self, key):- self.reads += 1- return super().__getitem__(key)-- reads = []- for repetitions in (1000, 2000):- source = CountedSource(",/[" * repetitions)- assert mask_c_style_source(source, ".js") is None- reads.append(source.reads)-- assert reads[1] < 3 * reads[0]---def test_python_preview_ignores_utf8_bom(tmp_path: Path) -> None:+def test_source_preview_ignores_utf8_bom(tmp_path: Path) -> None:source = b"def first():\n pass\n\ndef second():\n pass\n"path = tmp_path / "example.py"path.write_bytes(b"\xef\xbb\xbf" + source)- expected = preview_for_bytes(path, source, DEFAULT_PREVIEW_BYTES)- assert expected[0].splitlines() == ["function first()", "function second()"]+ expected = preview_for_bytes(source, DEFAULT_PREVIEW_BYTES)+ assert expected == (source.decode("utf-8").strip(), False)assert preview_for(path, DEFAULT_PREVIEW_BYTES) == expected-def test_javascript_preview_lists_class_field_arrow_handlers(tmp_path: Path) -> None:- source = """export class Controller {- login = async (request: Request) => request;- public refresh: Handler = (request) => request;- protected validate = value => Boolean(value);-}-"""-- preview = generate_preview(tmp_path, "controller.ts", source)-- assert "method Controller.login" in preview- assert "method Controller.refresh" in preview- assert "method Controller.validate" in preview--[email protected]("opener", ["TXT", '"TXT"', "'TXT'"])[email protected]("indent", ["", " "])-def test_php_heredoc_terminator_can_continue_expression(- tmp_path: Path, opener: str, indent: str-) -> None:- source = f"""<?php-function before() {{}}--$values = [<<<{opener}-{indent}hello-{indent}TXT,-];--function after() {{}}-"""-- preview = generate_preview(tmp_path, "sample.php", source)-- assert "function before" in preview- assert "function after" in preview--[email protected]("opener", ["TXT", '"TXT"', "'TXT'"])-def test_php_heredoc_terminator_preserves_following_brace(tmp_path: Path, opener: str) -> None:- source = f"""<?php-class Service {{- public function template() {{- return <<<{opener}-hello-TXT; }}- public function visible() {{}}-}}-"""-- preview = generate_preview(tmp_path, "Service.php", source)-- assert "method Service.template" in preview- assert "method Service.visible" in preview--[email protected]("opener", ["TXT", "'TXT'"])[email protected]("suffix", ["_more", "2", "😀", "\u0301"])-def test_php_heredoc_label_prefix_stays_in_body(tmp_path: Path, opener: str, suffix: str) -> None:- source = f"""<?php-function before() {{}}-$value = <<<{opener}-TXT{suffix} {{-TXT;-function after() {{}}-"""-- preview = generate_preview(tmp_path, "sample.php", source)-- assert "function before" in preview- assert "function after" in preview---def test_php_heredoc_does_not_hide_following_method(tmp_path: Path) -> None:- source = """<?php-class Service {- public function template() {- $value = <<<TXT-{-TXT;- }- public function visible() {}-}-"""-- preview = generate_preview(tmp_path, "Service.php", source)-- assert "method Service.template" in preview- assert "method Service.visible" in preview---def test_malformed_python_uses_sampled_source_fallback(tmp_path: Path) -> None:- source = """import package-broken = (-first_runtime_line()-second_runtime_line()-"""-- preview = generate_preview(tmp_path, "broken.py", source)-- assert preview.splitlines() == [- "import package",- "broken = (",- "first_runtime_line()",- "second_runtime_line()",- ]--- "prefix", ["", "def visible():\n pass\n"], ids=["sampled-source", "simple-outline"]-)-def test_python_preview_falls_back_on_ast_recursion(tmp_path: Path, prefix: str) -> None:- source = prefix + "value = 1\n"- path = tmp_path / "generated.py"- data = source.encode("utf-8")- path.write_bytes(data)-- with patch("rank_preview.ast.parse", side_effect=RecursionError):- preview, is_binary = preview_for(path, DEFAULT_PREVIEW_BYTES)- assert preview_for_bytes(path, data, DEFAULT_PREVIEW_BYTES) == (preview, False)-- assert not is_binary- assert preview- assert len(preview.encode("utf-8")) <= DEFAULT_PREVIEW_BYTES- if prefix:- assert preview == "function visible"- else:- assert source.startswith(preview)---def test_fallback_preview_uses_head_and_evenly_sampled_nonblank_lines(tmp_path: Path) -> None:+def test_large_source_preview_samples_nonblank_lines(tmp_path: Path) -> None:source = "\n\n".join(f"line_{index:02d} {{ color: red; }}" for index in range(40))- preview = generate_preview(tmp_path, "styles.css", source)+ preview = generate_preview(tmp_path, "styles.css", source, preview_bytes=700)assert preview.splitlines() == [*(f"line_{index:02d} {{ color: red; }}" for index in range(12)),@@ -841,15 +276,6 @@ def test_fallback_preview_uses_head_and_evenly_sampled_nonblank_lines(tmp_path:]-def test_fallback_preview_omits_marker_when_no_lines_are_skipped(tmp_path: Path) -> None:- source = "\n".join(f"line_{index:02d}" for index in range(22))-- preview = generate_preview(tmp_path, "styles.css", source, preview_bytes=4096)-- assert preview.splitlines() == [f"line_{index:02d}" for index in range(22)]- assert "..." not in preview--def test_preview_byte_budget_preserves_sampled_tail_and_valid_unicode(tmp_path: Path) -> None:source = "\n".join(f"line_{index:02d} {'😀' * 20}" for index in range(40))@@ -864,3 +290,16 @@ def test_literal_elision_line_respects_tiny_byte_budget(tmp_path: Path) -> None:preview = generate_preview(tmp_path, "styles.css", "...", preview_bytes=2)assert preview == ".."++[email protected](("prefix", "preview_bytes"), [("line", 25), ("😀", 30), (" line", 80)])+def test_tiny_preview_budget_retains_source(+ tmp_path: Path, prefix: str, preview_bytes: int+) -> None:+ source = "\n".join(f"{prefix}_{index:02d}" for index in range(40))++ preview = generate_preview(tmp_path, "source.txt", source, preview_bytes=preview_bytes)++ assert preview.startswith(f"{prefix}_00")+ assert source.startswith(preview)+ assert len(preview.encode("utf-8")) <= preview_bytesdiff --git a/sdk/typescript/src/custom-validation-prompt.ts b/sdk/typescript/src/custom-validation-prompt.tsindex 38f45cf3f..e796fe4ac 100644--- a/sdk/typescript/src/custom-validation-prompt.ts+++ b/sdk/typescript/src/custom-validation-prompt.ts@@ -13,7 +13,7 @@ const SOURCES = {"skills/security-scan/SKILL.md":"104e2f93fc965c34e91970f517a89e330d5b29dbb243dcea5d95d83f14d135cf","skills/security-diff-scan/SKILL.md":- "926290b5f5cee1a30f4c8bb49682fb883fba6f47b1bc73a9a31b9561d0011c8e",+ "653df6b1127141fb35bd0d6eea4ce5e2e764f5f5cede91b6fc7f3b3eb3a65874",} as const;const DISABLED_TOOLS = [