Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

refactor(plugin): split workbench publication helpers (#721)

soyeon-oai445397db5b11
diff --git a/plugins/codex-security/plugin-files.json b/plugins/codex-security/plugin-files.json
index 7d77201ce..2d0004e21 100644
--- a/plugins/codex-security/plugin-files.json
+++ b/plugins/codex-security/plugin-files.json
@@ -70,6 +70,7 @@
"scripts/workbench_findings.py",
"scripts/workbench_native_indexes.py",
"scripts/workbench_progress.py",
+ "scripts/workbench_publication.py",
"scripts/workbench_remediation.py",
"scripts/workbench_saved_results.py",
"scripts/workbench_scan_history.py",
diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py
index 04f5a33bd..e82dc36f4 100644
--- a/plugins/codex-security/scripts/finalize_scan_contract.py
+++ b/plugins/codex-security/scripts/finalize_scan_contract.py
@@ -250,9 +250,7 @@ def _require_safe_relative_path(value: str, context: str, *, allow_dot: bool = F
return normalized
-def _require_portable_relative_path(
- value: str, context: str, *, allow_dot: bool = False
-) -> str:
+def _require_portable_relative_path(value: str, context: str, *, allow_dot: bool = False) -> str:
normalized = _require_safe_relative_path(value, context, allow_dot=allow_dot)
if any(_windows_unsafe_path_component(part) for part in value.split("/")):
raise ContractError(f"{context}: expected a safe scan-relative POSIX path")
@@ -288,10 +286,7 @@ def _validate_scan_local_output_path(scan_dir: Path, path: Path, relative_path:
resolved_parent.relative_to(scan_dir)
except (OSError, RuntimeError, ValueError) as exc:
raise ContractError(f"{relative_path}: expected a path inside the scan directory") from exc
- if (
- os.path.normcase(resolved_parent) != os.path.normcase(path.parent)
- or path.is_symlink()
- ):
+ if os.path.normcase(resolved_parent) != os.path.normcase(path.parent) or path.is_symlink():
raise ContractError(
f"{relative_path}: expected a non-symlink path inside the scan directory"
)
@@ -341,10 +336,7 @@ def _open_verified_scan_directory(
try:
expected = scan_dir.lstat()
observed_identity = (expected.st_dev, expected.st_ino)
- if (
- expected_root_identity is not None
- and observed_identity != expected_root_identity
- ):
+ if expected_root_identity is not None and observed_identity != expected_root_identity:
raise ContractError("scan directory: changed after artifact restoration setup")
canonical = _require_scan_directory(scan_dir)
descriptor = os.open(
@@ -352,14 +344,11 @@ def _open_verified_scan_directory(
os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0),
)
except OSError as exc:
- raise ContractError(
- "scan directory: expected an existing non-symlink directory"
- ) from exc
+ raise ContractError("scan directory: expected an existing non-symlink directory") from exc
opened = os.fstat(descriptor)
opened_identity = (opened.st_dev, opened.st_ino)
if opened_identity != observed_identity or (
- expected_root_identity is not None
- and opened_identity != expected_root_identity
+ expected_root_identity is not None and opened_identity != expected_root_identity
):
os.close(descriptor)
raise ContractError("scan directory: changed while it was being opened")
@@ -372,9 +361,7 @@ def scan_root_identity(scan_dir: Path) -> tuple[Path, tuple[int, int]]:
scan_dir = _require_scan_directory(scan_dir)
if not _descriptor_relative_writes_available():
if not _is_windows():
- raise ContractError(
- "scan-local output requires descriptor-relative file operations"
- )
+ raise ContractError("scan-local output requires descriptor-relative file operations")
return _windows_scan_local_files().scan_root_identity(scan_dir)
descriptor = _open_verified_scan_directory(scan_dir)
try:
@@ -584,9 +571,7 @@ def write_scan_local_bytes(
try:
existing_fd = os.open(
parts[-1],
- os.O_RDONLY
- | getattr(os, "O_NOFOLLOW", 0)
- | getattr(os, "O_NONBLOCK", 0),
+ os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0),
dir_fd=parent_fd,
)
except OSError as exc:
@@ -596,20 +581,13 @@ def write_scan_local_bytes(
try:
opened = os.fstat(existing_fd)
if not stat.S_ISREG(opened.st_mode):
- raise ContractError(
- f"{relative_path}: expected a regular non-symlink file"
- )
+ raise ContractError(f"{relative_path}: expected a regular non-symlink file")
if (opened.st_dev, opened.st_ino) != (
metadata.st_dev,
metadata.st_ino,
):
- raise ContractError(
- f"{relative_path}: changed while it was being opened"
- )
- if (
- expected_root_identity is not None
- and opened.st_size == len(payload)
- ):
+ raise ContractError(f"{relative_path}: changed while it was being opened")
+ if expected_root_identity is not None and opened.st_size == len(payload):
try:
with os.fdopen(existing_fd, "rb") as handle:
existing_fd = -1
@@ -857,9 +835,9 @@ def _recover_unsealed_findings(
try:
if not isinstance(finding, dict):
raise ContractError(f"{context}: expected an object")
- compatible_findings = _legacy_sealed_findings_for_validation(
- {"findings": [finding]}
- )["findings"]
+ compatible_findings = _legacy_sealed_findings_for_validation({"findings": [finding]})[
+ "findings"
+ ]
compatible_finding = compatible_findings[0]
normalized_legacy_details = compatible_finding != finding
finding = compatible_finding
@@ -870,9 +848,7 @@ def _recover_unsealed_findings(
]
if "instance" in identity:
fields.append((identity, "instance", f"{context}.identity", "instance"))
- normalized_fields = (
- ["legacy finding details"] if normalized_legacy_details else []
- )
+ normalized_fields = ["legacy finding details"] if normalized_legacy_details else []
for parent, field, field_context, label in fields:
value = _require_str(parent, field, field_context)
if SLUG_RE.fullmatch(value):
@@ -1649,10 +1625,7 @@ def _validate_schema_node(
raise ContractError(f"{context}: array has too many items")
if schema.get("uniqueItems") is True:
for index, item in enumerate(value):
- if any(
- _schema_values_equal(item, candidate)
- for candidate in value[:index]
- ):
+ if any(_schema_values_equal(item, candidate) for candidate in value[:index]):
raise ContractError(f"{context}: array items must be unique")
contains = schema.get("contains")
if isinstance(contains, dict):
@@ -1671,9 +1644,7 @@ def _validate_schema_node(
item_schema = schema.get("items")
if isinstance(item_schema, dict):
for index, item in enumerate(value):
- _validate_schema_node(
- item, item_schema, f"{context}[{index}]", root_schema
- )
+ _validate_schema_node(item, item_schema, f"{context}[{index}]", root_schema)
if isinstance(value, dict):
for item_schema in schema.get("allOf", []):
_validate_schema_node(value, item_schema, context, root_schema)
@@ -1697,9 +1668,7 @@ def _validate_schema_node(
for key, item in value.items():
item_schema = properties.get(key)
if isinstance(item_schema, dict):
- _validate_schema_node(
- item, item_schema, f"{context}.{key}", root_schema
- )
+ _validate_schema_node(item, item_schema, f"{context}.{key}", root_schema)
elif additional_properties is False:
raise ContractError(f"{context}.{key}: unexpected schema property")
elif isinstance(additional_properties, dict):
@@ -1716,22 +1685,16 @@ def validate_against_schema(payload: dict[str, Any], schema_path: Path) -> None:
_validate_schema_node(payload, schema, schema_path.stem, schema)
-def _filter_unknown_legacy_evidence_refs(
- section: dict[str, Any], evidence_ids: set[str]
-) -> None:
+def _filter_unknown_legacy_evidence_refs(section: dict[str, Any], evidence_ids: set[str]) -> None:
for refs_field in ("evidenceRefs", "evidence_refs"):
refs = section.get(refs_field)
if isinstance(refs, list):
section[refs_field] = [
- ref
- for ref in refs
- if isinstance(ref, str) and ref.strip() and ref in evidence_ids
+ ref for ref in refs if isinstance(ref, str) and ref.strip() and ref in evidence_ids
]
-def _normalize_legacy_string_list_fields(
- section: dict[str, Any], fields: tuple[str, ...]
-) -> None:
+def _normalize_legacy_string_list_fields(section: dict[str, Any], fields: tuple[str, ...]) -> None:
for field in fields:
if field not in section:
continue
@@ -1739,9 +1702,7 @@ def _normalize_legacy_string_list_fields(
if isinstance(value, str):
normalized = [value] if value.strip() else []
elif isinstance(value, list):
- normalized = [
- item for item in value if isinstance(item, str) and item.strip()
- ]
+ normalized = [item for item in value if isinstance(item, str) and item.strip()]
else:
normalized = []
if normalized:
@@ -1754,9 +1715,7 @@ def _remove_unsupported_legacy_scalar_fields(
section: dict[str, Any], fields: tuple[str, ...]
) -> None:
for field in fields:
- if field in section and (
- not isinstance(section[field], str) or section[field] == ""
- ):
+ if field in section and (not isinstance(section[field], str) or section[field] == ""):
section.pop(field)
@@ -1773,9 +1732,7 @@ def _legacy_sealed_findings_for_validation(findings: dict[str, Any]) -> dict[str
canonical_evidence_ids = {
evidence["id"]
for evidence in canonical_evidence
- if isinstance(evidence, dict)
- and isinstance(evidence.get("id"), str)
- and evidence["id"]
+ if isinstance(evidence, dict) and isinstance(evidence.get("id"), str) and evidence["id"]
}
legacy_evidence = finding.get("code_evidence")
if isinstance(legacy_evidence, list):
@@ -1807,9 +1764,7 @@ def _legacy_sealed_findings_for_validation(findings: dict[str, Any]) -> dict[str
evidence_ids = canonical_evidence_ids | {
evidence["id"]
for evidence in compatible_legacy_evidence
- if isinstance(evidence, dict)
- and isinstance(evidence.get("id"), str)
- and evidence["id"]
+ if isinstance(evidence, dict) and isinstance(evidence.get("id"), str) and evidence["id"]
}
for section_name, list_fields in (
("rootCause", ("evidenceRefs", "evidence_refs")),
@@ -1891,12 +1846,8 @@ def _legacy_sealed_findings_for_validation(findings: dict[str, Any]) -> dict[str
for field in ("impact", "likelihood"):
detail = attack_path.get(field)
if isinstance(detail, dict):
- _remove_unsupported_legacy_scalar_fields(
- detail, ("level", "rationale", "why")
- )
- elif detail is not None and (
- not isinstance(detail, str) or detail == ""
- ):
+ _remove_unsupported_legacy_scalar_fields(detail, ("level", "rationale", "why"))
+ elif detail is not None and (not isinstance(detail, str) or detail == ""):
attack_path.pop(field)
return compatible
@@ -2805,9 +2756,7 @@ def _prepare_scan_finalization(
_validate_sealed_coverage_receipts(scan, coverage)
_validate_manifest(manifest)
validate_against_schema(manifest, schema_dir / "scan-manifest.schema.json")
- validate_against_schema(
- findings_for_validation, schema_dir / "findings.schema.json"
- )
+ validate_against_schema(findings_for_validation, schema_dir / "findings.schema.json")
validate_against_schema(coverage, schema_dir / "coverage.schema.json")
report_markdown_bytes = _generate_report_projection(manifest, findings, coverage)
_validate_report_output_paths(scan_dir)
diff --git a/plugins/codex-security/scripts/generate_in_scope_files.py b/plugins/codex-security/scripts/generate_in_scope_files.py
index 2a400d183..eff29decf 100644
--- a/plugins/codex-security/scripts/generate_in_scope_files.py
+++ b/plugins/codex-security/scripts/generate_in_scope_files.py
@@ -21,11 +21,7 @@ def windows_stream_component(path: Path) -> str | None:
if os.name != "nt":
return None
return next(
- (
- component
- for component in path.parts
- if component != path.anchor and ":" in component
- ),
+ (component for component in path.parts if component != path.anchor and ":" in component),
None,
)
@@ -49,9 +45,7 @@ def resolve_scope(repository: Path, value: str) -> str:
requested = Path(value).expanduser()
stream = windows_stream_component(requested)
if stream is not None:
- raise InventoryError(
- f"--scope: NTFS alternate data streams are not supported: {stream}"
- )
+ raise InventoryError(f"--scope: NTFS alternate data streams are not supported: {stream}")
scope = requested if requested.is_absolute() else repository / requested
try:
resolved = scope.resolve(strict=True)
diff --git a/plugins/codex-security/scripts/generate_rank_input.py b/plugins/codex-security/scripts/generate_rank_input.py
index 8c4dd3722..221e3d7c8 100644
--- a/plugins/codex-security/scripts/generate_rank_input.py
+++ b/plugins/codex-security/scripts/generate_rank_input.py
@@ -299,11 +299,7 @@ def windows_stream_component(path: Path) -> str | None:
if os.name != "nt":
return None
return next(
- (
- component
- for component in path.parts
- if component != path.anchor and ":" in component
- ),
+ (component for component in path.parts if component != path.anchor and ":" in component),
None,
)
diff --git a/plugins/codex-security/scripts/report_projection.py b/plugins/codex-security/scripts/report_projection.py
index 281eb7d1c..697dda8bd 100644
--- a/plugins/codex-security/scripts/report_projection.py
+++ b/plugins/codex-security/scripts/report_projection.py
@@ -333,13 +333,13 @@ def merged_root_cause(value: dict[str, Any]) -> tuple[str | None, Any]:
"language",
):
continue
- if field in text_fields and (
- not isinstance(item, str) or not item.strip()
- ):
+ if field in text_fields and (not isinstance(item, str) or not item.strip()):
continue
current = merged.get(field)
- if field not in merged or current in (None, "", [], {}) or (
- isinstance(current, str) and not current.strip()
+ if (
+ field not in merged
+ or current in (None, "", [], {})
+ or (isinstance(current, str) and not current.strip())
):
merged[field] = item
@@ -363,9 +363,7 @@ def merged_root_cause(value: dict[str, Any]) -> tuple[str | None, Any]:
item
for detail in details
for field in ("codeEvidence", "code_evidence")
- for item in (
- detail.get(field, []) if isinstance(detail.get(field), list) else []
- )
+ for item in (detail.get(field, []) if isinstance(detail.get(field), list) else [])
]
if embedded_evidence:
merged["codeEvidence"] = embedded_evidence
@@ -541,9 +539,7 @@ def _finding_section(number: int, finding: dict[str, Any]) -> list[str]:
(
section[key]
for section in dataflow_sections
- if key in section
- and isinstance(section[key], str)
- and section[key].strip()
+ if key in section and isinstance(section[key], str) and section[key].strip()
),
None,
)
@@ -652,9 +648,7 @@ def _finding_section(number: int, finding: dict[str, Any]) -> list[str]:
if validation.get("method"):
lines.extend(["", f"Validation method: {_text(validation['method'], 'not recorded')}"])
if validation_outcomes:
- lines.extend(
- ["", *(f"- **{label}:** {value}" for label, value in validation_outcomes)]
- )
+ lines.extend(["", *(f"- **{label}:** {value}" for label, value in validation_outcomes)])
lines.extend(_code_evidence_lines(validation_code_evidence))
if validation_assertions:
lines.extend(["", "Assertions:", *_bullets(validation_assertions, "None recorded.")])
diff --git a/plugins/codex-security/scripts/validate_scan_contract.py b/plugins/codex-security/scripts/validate_scan_contract.py
index abcf313fe..298b7eaf4 100644
--- a/plugins/codex-security/scripts/validate_scan_contract.py
+++ b/plugins/codex-security/scripts/validate_scan_contract.py
@@ -64,9 +64,7 @@ def validate_contract(scan_dir: Path) -> dict[str, Any]:
FINALIZER._validate_derived_finding_identities(manifest, findings)
FINALIZER._validate_coverage(manifest, coverage, scan_dir)
FINALIZER._validate_sealed_coverage_receipts(scan, coverage)
- FINALIZER.validate_against_schema(
- findings_for_validation, schema_dir / "findings.schema.json"
- )
+ FINALIZER.validate_against_schema(findings_for_validation, schema_dir / "findings.schema.json")
FINALIZER.validate_against_schema(coverage, schema_dir / "coverage.schema.json")
FINALIZER._require_scan_local_file(scan_dir, "report.md", "report.md")
return {
diff --git a/plugins/codex-security/scripts/windows_scan_local_files.py b/plugins/codex-security/scripts/windows_scan_local_files.py
index b89c6ef8b..0c5581740 100644
--- a/plugins/codex-security/scripts/windows_scan_local_files.py
+++ b/plugins/codex-security/scripts/windows_scan_local_files.py
@@ -410,13 +410,8 @@ def _locked_parent(
_require_windows()
parts = _validated_parts(relative_path)
root_path, observed_root_identity = _canonical_scan_directory(scan_dir)
- if (
- expected_root_identity is not None
- and observed_root_identity != expected_root_identity
- ):
- raise _invalid_path(
- scan_dir, "scan directory changed after artifact restoration setup"
- )
+ if expected_root_identity is not None and observed_root_identity != expected_root_identity:
+ raise _invalid_path(scan_dir, "scan directory changed after artifact restoration setup")
handles: list[_OwnedHandle] = []
try:
# Absolute-path Win32 calls remain safe only while every ancestor is
@@ -429,12 +424,9 @@ def _locked_parent(
current_root = root_path.lstat()
current_root_identity = (current_root.st_dev, current_root.st_ino)
if current_root_identity != observed_root_identity or (
- expected_root_identity is not None
- and current_root_identity != expected_root_identity
+ expected_root_identity is not None and current_root_identity != expected_root_identity
):
- raise _invalid_path(
- scan_dir, "scan directory changed while it was being opened"
- )
+ raise _invalid_path(scan_dir, "scan directory changed while it was being opened")
current_path = root_path
for component in parts[:-1]:
current_path /= component
@@ -596,9 +588,7 @@ def _existing_output_matches(path: Path, payload: bytes) -> bool:
raw_handle = handle.detach()
try:
assert _msvcrt is not None
- descriptor = _msvcrt.open_osfhandle(
- raw_handle, os.O_RDONLY | os.O_BINARY
- )
+ descriptor = _msvcrt.open_osfhandle(raw_handle, os.O_RDONLY | os.O_BINARY)
except BaseException:
_close_handle(raw_handle)
raise
diff --git a/plugins/codex-security/scripts/workbench_cli.py b/plugins/codex-security/scripts/workbench_cli.py
index 998a0c151..aed24fa69 100644
--- a/plugins/codex-security/scripts/workbench_cli.py
+++ b/plugins/codex-security/scripts/workbench_cli.py
@@ -247,15 +247,11 @@ def parse_args(description: str) -> argparse.Namespace:
preserve_scan.add_argument("--claim-token")
preserve_scan.add_argument("--coordinator-generation", type=positive_int)
- recovery_help = (
- "Validate and republish retained checkpoints for a failed, non-canceled scan."
- )
+ recovery_help = "Validate and republish retained checkpoints for a failed, non-canceled scan."
recover_scan = subparsers.add_parser(
"recover-scan-results", help=recovery_help, description=recovery_help
)
- recover_scan.add_argument(
- "--scan-id", required=True, help="ID of the stopped scan to recover."
- )
+ recover_scan.add_argument("--scan-id", required=True, help="ID of the stopped scan to recover.")
write_scan_draft = subparsers.add_parser("write-scan-draft")
write_scan_draft.add_argument("--scan-id", required=True)
diff --git a/plugins/codex-security/scripts/workbench_dashboard.py b/plugins/codex-security/scripts/workbench_dashboard.py
index 0e7b510b6..1c6e2e6b2 100644
--- a/plugins/codex-security/scripts/workbench_dashboard.py
+++ b/plugins/codex-security/scripts/workbench_dashboard.py
@@ -12,7 +12,6 @@ from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parent))
from workbench_findings import list_dedupe_groups
-
FINDING_RECORDS = """
SELECT findings.id, json_extract(details_json, '$.title') AS title,
COALESCE(repositories.ids, '[]') AS repositoryIds,
@@ -52,17 +51,24 @@ def detail(connection: sqlite3.Connection, view: str, selected: dict[str, Any])
result: dict[str, Any] = {"item": selected}
selected_id = selected["id"]
if view == "findings":
- result["finding"] = json.loads(connection.execute(
- "SELECT details_json FROM findings WHERE id = ?", (selected_id,),
- ).fetchone()[0])
+ result["finding"] = json.loads(
+ connection.execute(
+ "SELECT details_json FROM findings WHERE id = ?",
+ (selected_id,),
+ ).fetchone()[0]
+ )
result["groups"] = list_dedupe_groups(connection, selected_id)["groups"]
else:
result["group"] = {
- "groupId": selected_id, "createdAt": selected["createdAt"],
- "findingIds": [r[0] for r in connection.execute(
- "SELECT finding_id FROM finding_dedupe_group_members WHERE group_id = ? ORDER BY finding_id",
- (selected_id,),
- )],
+ "groupId": selected_id,
+ "createdAt": selected["createdAt"],
+ "findingIds": [
+ r[0]
+ for r in connection.execute(
+ "SELECT finding_id FROM finding_dedupe_group_members WHERE group_id = ? ORDER BY finding_id",
+ (selected_id,),
+ )
+ ],
}
return result
@@ -76,7 +82,11 @@ def dashboard(connection: sqlite3.Connection, query: dict[str, Any]) -> dict[str
if query.get("query"):
connection.create_function("casefold", 1, str.casefold, deterministic=True)
columns = ["id", "title", "repositoryIds"]
- clauses.append("(" + " OR ".join(f"instr(casefold(COALESCE({c}, '')), casefold(?)) > 0" for c in columns) + ")")
+ clauses.append(
+ "("
+ + " OR ".join(f"instr(casefold(COALESCE({c}, '')), casefold(?)) > 0" for c in columns)
+ + ")"
+ )
values.extend([query["query"]] * len(columns))
if query.get("repository"):
clauses.append("EXISTS (SELECT 1 FROM json_each(repositoryIds) WHERE value = ?)")
@@ -89,23 +99,36 @@ def dashboard(connection: sqlite3.Connection, query: dict[str, Any]) -> dict[str
SELECT DISTINCT repository_id AS id, repository_id AS label
FROM finding_repositories ORDER BY repository_id
""").fetchall()
- total = connection.execute(f"SELECT COUNT(*) FROM ({records}) {where}", values).fetchone()[0]
+ total = connection.execute(f"SELECT COUNT(*) FROM ({records}) {where}", values).fetchone()[
+ 0
+ ]
rows = connection.execute(
f"SELECT * FROM ({records}) {where} ORDER BY {order} LIMIT ? OFFSET ?",
(*values, query["limit"], query["offset"]),
).fetchall()
- selected = connection.execute(
- f"SELECT * FROM ({records}) WHERE id = ?", (query["id"],),
- ).fetchone() if query.get("id") else None
+ selected = (
+ connection.execute(
+ f"SELECT * FROM ({records}) WHERE id = ?",
+ (query["id"],),
+ ).fetchone()
+ if query.get("id")
+ else None
+ )
next_offset = query["offset"] + len(rows)
return {
"overview": {
- "findings": connection.execute("SELECT COUNT(*) FROM findings WHERE details_json IS NOT NULL").fetchone()[0],
- "groups": connection.execute("SELECT COUNT(*) FROM finding_dedupe_groups").fetchone()[0],
+ "findings": connection.execute(
+ "SELECT COUNT(*) FROM findings WHERE details_json IS NOT NULL"
+ ).fetchone()[0],
+ "groups": connection.execute(
+ "SELECT COUNT(*) FROM finding_dedupe_groups"
+ ).fetchone()[0],
},
"repositories": [dict(row) for row in repositories],
- "items": [item(row) for row in rows], "total": total,
- "limit": query["limit"], "offset": query["offset"],
+ "items": [item(row) for row in rows],
+ "total": total,
+ "limit": query["limit"],
+ "offset": query["offset"],
"nextOffset": next_offset if next_offset < total else None,
"detail": detail(connection, view, item(selected)) if selected is not None else None,
}
diff --git a/plugins/codex-security/scripts/workbench_db.py b/plugins/codex-security/scripts/workbench_db.py
index 49591f96a..b5227826f 100644
--- a/plugins/codex-security/scripts/workbench_db.py
+++ b/plugins/codex-security/scripts/workbench_db.py
@@ -4,10 +4,8 @@
from __future__ import annotations
import argparse
-import csv
import errno
import hashlib
-import io
import json
import os
import re
@@ -21,7 +19,6 @@ from contextlib import closing, contextmanager
from datetime import datetime, timedelta, timezone
from pathlib import Path, PurePosixPath
from typing import Any
-from urllib.parse import quote
try:
import fcntl as posix_file_lock
@@ -37,11 +34,14 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
import deep_scan_workbench as deep_scan
import workbench_native_indexes as native_indexes
import workbench_progress as progress
+import workbench_publication as publication
import workbench_remediation as remediation
import workbench_saved_results as saved_results
import workbench_scan_history as scan_history
import workbench_scan_usage as scan_usage
-from filesystem_identity import serialize_filesystem_identity as serialize_filesystem_identity
+from filesystem_identity import (
+ serialize_filesystem_identity as serialize_filesystem_identity,
+)
from filesystem_identity import (
stored_filesystem_identity_matches as stored_filesystem_identity_matches,
)
@@ -51,11 +51,9 @@ from finalize_scan_contract import (
RecoverableContractError,
_prepare_scan_finalization,
_write_prepared_scan_finalization,
- csv_cell,
finalize_scan,
finding_candidate_id,
open_scan_local_file_descriptor,
- write_sarif_projection,
write_scan_local_bytes,
)
from finding_preview import bounded_finding_details
@@ -80,8 +78,8 @@ from workbench_constants import (
PATCH_PREVIEW_BYTES,
SQLITE_RETRY_ATTEMPTS,
)
-from workbench_feedback import get_scan_feedback
from workbench_dashboard import dashboard
+from workbench_feedback import get_scan_feedback
from workbench_finding_index import index_findings
from workbench_finding_workflows import finding_workflow, register_workflow_scan
from workbench_findings import (
@@ -1023,7 +1021,10 @@ def _start_prompt_driven_scan(
).fetchone()
if existing is not None:
connection.commit()
- return {**scan_context(connection, existing["id"]), "startDisposition": "joined"}
+ return {
+ **scan_context(connection, existing["id"]),
+ "startDisposition": "joined",
+ }
target_root.mkdir(parents=True, exist_ok=True)
workspace_id = str(uuid.uuid4())
scan_id = str(uuid.uuid4())
@@ -1134,7 +1135,10 @@ def pin_legacy_manifest_digest(
def complete_scan(
- connection: sqlite3.Connection, args: argparse.Namespace, *, prepare_only: bool = False
+ connection: sqlite3.Connection,
+ args: argparse.Namespace,
+ *,
+ prepare_only: bool = False,
) -> dict[str, Any]:
scan_id = require_uuid(args.scan_id, "scan-id")
cost_json = None if prepare_only else parse_scan_cost(args.cost_json)
@@ -1347,8 +1351,7 @@ def budget_exhausted_draft(
existing_deferred = {
item.get("candidateId", item.get("id"))
for item in coverage["deferred"]
- if isinstance(item, dict)
- and isinstance(item.get("candidateId", item.get("id")), str)
+ if isinstance(item, dict) and isinstance(item.get("candidateId", item.get("id")), str)
}
existing_surfaces = {
item.get("id")
@@ -1551,9 +1554,7 @@ def complete_scan_locked(
wrote = True
manifest, findings, _ = _write_prepared_scan_finalization(prepared)
except ContractError as exc:
- if wrote or (
- scan["mode"] == "deep" and not isinstance(exc, RecoverableContractError)
- ):
+ if wrote or (scan["mode"] == "deep" and not isinstance(exc, RecoverableContractError)):
args = argparse.Namespace(claim_token=claim_token, cost_json=cost_json)
args.message, args.scan_id = str(exc), scan_id
fail_scan_locked(connection, args)
@@ -1959,7 +1960,14 @@ def set_finding_triage(connection: sqlite3.Connection, args: argparse.Namespace)
id, occurrence_id, status, close_reason, note, created_at
) VALUES (?, ?, ?, ?, ?, ?)
""",
- (str(uuid.uuid4()), occurrence["id"], args.status, close_reason, note, timestamp),
+ (
+ str(uuid.uuid4()),
+ occurrence["id"],
+ args.status,
+ close_reason,
+ note,
+ timestamp,
+ ),
)
connection.execute(
"""
@@ -2320,8 +2328,8 @@ def set_finding_remediation(
)
if current["pending_action_claim_token"] != action_token:
raise SystemExit("This remediation host request is owned by a different action token.")
- require_remediation_transition(current["state"], args.state)
- require_pending_remediation_action(current, args.state)
+ remediation.require_transition(current["state"], args.state)
+ remediation.require_pending_action(current, args.state)
patch_path = current["patch_path"]
if args.patch_path is not None:
requested_patch_path = require_scan_relative_file(scan, args.patch_path)
@@ -2422,493 +2430,15 @@ def set_finding_remediation(
return scan_context(connection, occurrence["scan_id"])
-def linear_publication_input(
- args: argparse.Namespace, *, recording: bool
-) -> tuple[dict[str, Any], dict[str, str], list[dict[str, str]]]:
- payload = read_json_object(Path(args.input_file))
- required = {"scanId", "scanDirectory", "destination", "findings"}
- if recording:
- required.add("publications")
- if set(payload) != required:
- raise SystemExit("Linear publication input contains unexpected or missing fields.")
-
- scan_id = payload["scanId"]
- scan_directory = payload["scanDirectory"]
- destination = payload["destination"]
- findings = payload["findings"]
- if not isinstance(scan_id, str) or not isinstance(scan_directory, str):
- raise SystemExit("Linear publication input must identify the exact completed scan.")
- if (
- not isinstance(destination, dict)
- or not {"type", "teamId"}.issubset(destination)
- or not set(destination).issubset({"type", "teamId", "projectId"})
- or destination.get("type") != "linear"
- or not isinstance(destination.get("teamId"), str)
- or not destination["teamId"].strip()
- or (
- "projectId" in destination
- and (
- not isinstance(destination["projectId"], str)
- or not destination["projectId"].strip()
- )
- )
- ):
- raise SystemExit("Linear publication input must identify the exact team and optional project.")
- if not isinstance(findings, list):
- raise SystemExit("Linear publication input must include the planned scan findings.")
-
- seen_finding_ids: set[str] = set()
- seen_occurrence_ids: set[str] = set()
- for finding in findings:
- if (
- not isinstance(finding, dict)
- or set(finding) != {"findingId", "occurrenceId"}
- or not isinstance(finding.get("findingId"), str)
- or not finding["findingId"].strip()
- or not isinstance(finding.get("occurrenceId"), str)
- or not finding["occurrenceId"].strip()
- ):
- raise SystemExit("Linear publication input contains an invalid finding identity.")
- if (
- finding["findingId"] in seen_finding_ids
- or finding["occurrenceId"] in seen_occurrence_ids
- ):
- raise SystemExit("Linear publication input repeats a finding or occurrence.")
- seen_finding_ids.add(finding["findingId"])
- seen_occurrence_ids.add(finding["occurrenceId"])
-
- return payload, destination, findings
-
-
-def verify_linear_publication_scan(
- connection: sqlite3.Connection,
- payload: dict[str, Any],
- findings: list[dict[str, str]],
-) -> sqlite3.Row:
- try:
- scan = require_scan(connection, payload["scanId"])
- except SystemExit as exc:
- raise SystemExit(
- "The completed scan is not present in the local Codex Security scan-history database. "
- "Use the state directory where the scan was completed."
- ) from exc
- if scan["id"] != payload["scanId"]:
- raise SystemExit("Linear publication must use the exact completed scan identifier.")
- if scan["status"] != "complete":
- raise SystemExit("Only completed scans can publish findings to Linear.")
-
- requested_directory = require_canonical_scan_directory(Path(payload["scanDirectory"]))
- recorded_directory = require_canonical_scan_directory(Path(scan["scan_dir"]))
- if os.path.normcase(requested_directory) != os.path.normcase(recorded_directory):
- raise SystemExit(
- "The selected scan directory does not match its local Codex Security scan history."
- )
- if "seal_manifest_digest" in scan.keys():
- require_recorded_manifest_digest(scan, recorded_directory)
-
- stored_findings = {
- row["id"]: row["finding_id"]
- for row in connection.execute(
- "SELECT id, finding_id FROM finding_occurrences WHERE scan_id = ?",
- (scan["id"],),
- )
- }
- for finding in findings:
- if stored_findings.get(finding["occurrenceId"]) != finding["findingId"]:
- raise SystemExit(
- "A selected finding or occurrence does not belong to the completed scan "
- "in local Codex Security scan history."
- )
- if len(stored_findings) != len(findings):
- raise SystemExit(
- "The completed scan findings do not exactly match local Codex Security scan history."
- )
- return scan
+_WORKBENCH_PUBLICATION_CONTEXT: publication.WorkbenchPublicationContext
def inspect_linear_publication(args: argparse.Namespace) -> dict[str, Any]:
- payload, destination, findings = linear_publication_input(args, recording=False)
- database_uri = f"file:{quote(str(database_path()), safe='')}?mode=ro"
- with closing(sqlite3.connect(database_uri, uri=True, timeout=5)) as connection:
- connection.row_factory = sqlite3.Row
- connection.execute("BEGIN")
- scan = verify_linear_publication_scan(connection, payload, findings)
- recorded: dict[str, dict[str, str]] = {}
- if connection.execute(
- "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'finding_publications'"
- ).fetchone():
- for row in connection.execute(
- """
- SELECT finding_id, occurrence_id, external_id, external_url
- FROM finding_publications
- WHERE scan_id = ? AND destination_type = ? AND team_id = ? AND project_id IS ?
- ORDER BY created_at, external_id
- """,
- (
- scan["id"],
- destination["type"],
- destination["teamId"],
- destination.get("projectId"),
- ),
- ):
- recorded.setdefault(
- row["occurrence_id"],
- {
- "findingId": row["finding_id"],
- "occurrenceId": row["occurrence_id"],
- "issueIdentifier": row["external_id"],
- **({"url": row["external_url"]} if row["external_url"] is not None else {}),
- },
- )
- return {
- "scanId": scan["id"],
- "destination": destination,
- "findingCount": len(findings),
- "recorded": [
- recorded[finding["occurrenceId"]]
- for finding in findings
- if finding["occurrenceId"] in recorded
- ],
- }
-
-
-def prepare_linear_publication(
- connection: sqlite3.Connection, args: argparse.Namespace
-) -> dict[str, Any]:
- payload, destination, findings = linear_publication_input(args, recording=False)
- connection.execute("BEGIN IMMEDIATE")
- try:
- scan = verify_linear_publication_scan(connection, payload, findings)
- result = {
- "scanId": scan["id"],
- "destination": destination,
- "findingCount": len(findings),
- }
- connection.commit()
- except BaseException:
- connection.rollback()
- raise
- return result
-
-
-def record_linear_publications(
- connection: sqlite3.Connection, args: argparse.Namespace
-) -> dict[str, Any]:
- payload, destination, findings = linear_publication_input(args, recording=True)
- publications = payload["publications"]
- if not isinstance(publications, list):
- raise SystemExit("Linear publication results must be an array.")
- planned = {finding["findingId"]: finding["occurrenceId"] for finding in findings}
- current: dict[str, dict[str, str]] = {}
- external_ids: set[str] = set()
- for publication in publications:
- if (
- not isinstance(publication, dict)
- or not {"findingId", "occurrenceId", "issueIdentifier"}.issubset(publication)
- or not set(publication).issubset(
- {"findingId", "occurrenceId", "issueIdentifier", "url"}
- )
- or not isinstance(publication.get("findingId"), str)
- or not isinstance(publication.get("occurrenceId"), str)
- or not isinstance(publication.get("issueIdentifier"), str)
- or not publication["issueIdentifier"].strip()
- or (
- "url" in publication
- and (
- not isinstance(publication["url"], str)
- or not publication["url"].strip()
- )
- )
- ):
- raise SystemExit("Linear publication results contain an invalid issue association.")
- finding_id = publication["findingId"]
- issue_identifier = publication["issueIdentifier"]
- if planned.get(finding_id) != publication["occurrenceId"]:
- raise SystemExit(
- "A created Linear issue does not match its planned finding and occurrence."
- )
- if finding_id in current or issue_identifier in external_ids:
- raise SystemExit("Linear publication results repeat a finding or issue identifier.")
- current[finding_id] = publication
- external_ids.add(issue_identifier)
-
- connection.execute("BEGIN IMMEDIATE")
- try:
- scan = verify_linear_publication_scan(connection, payload, findings)
- timestamp = now()
- for publication in publications:
- conflicting = connection.execute(
- """
- SELECT occurrence_id, external_url
- FROM finding_publications
- WHERE destination_type = ? AND team_id = ? AND project_id IS ?
- AND external_id = ?
- """,
- (
- destination["type"],
- destination["teamId"],
- destination.get("projectId"),
- publication["issueIdentifier"],
- ),
- ).fetchone()
- if conflicting is not None and conflicting["occurrence_id"] != publication[
- "occurrenceId"
- ]:
- raise SystemExit("This Linear issue is already associated with a different finding.")
- if (
- conflicting is not None
- and "url" in publication
- and conflicting["external_url"] != publication["url"]
- ):
- raise SystemExit("This Linear issue is already associated with a different URL.")
-
- connection.execute(
- """
- INSERT INTO finding_publications (
- scan_id, finding_id, occurrence_id, destination_type,
- team_id, project_id, external_id, external_url, created_at
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
- ON CONFLICT DO NOTHING
- """,
- (
- scan["id"],
- publication["findingId"],
- publication["occurrenceId"],
- destination["type"],
- destination["teamId"],
- destination.get("projectId"),
- publication["issueIdentifier"],
- publication.get("url"),
- timestamp,
- ),
- )
-
- created = []
- for finding in findings:
- publication = current.get(finding["findingId"])
- if publication is None:
- continue
- row = connection.execute(
- """
- SELECT finding_id, occurrence_id, external_id, external_url
- FROM finding_publications
- WHERE scan_id = ? AND occurrence_id = ? AND destination_type = ?
- AND team_id = ? AND project_id IS ? AND external_id = ?
- """,
- (
- scan["id"],
- publication["occurrenceId"],
- destination["type"],
- destination["teamId"],
- destination.get("projectId"),
- publication["issueIdentifier"],
- ),
- ).fetchone()
- if row is None:
- raise SystemExit("A created Linear issue could not be read from scan history.")
- created.append(
- {
- "findingId": row["finding_id"],
- "occurrenceId": row["occurrence_id"],
- "issueIdentifier": row["external_id"],
- **({"url": row["external_url"]} if row["external_url"] is not None else {}),
- }
- )
- result = {"scanId": scan["id"], "destination": destination, "created": created}
- connection.commit()
- except BaseException:
- connection.rollback()
- raise
- return result
+ return publication.inspect_linear_publication(_WORKBENCH_PUBLICATION_CONTEXT, args)
def export_findings(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]:
- scan = require_scan(connection, args.scan_id)
- if scan["status"] != "complete" and not (
- scan["status"] == "failed" and scan["seal_manifest_digest"]
- ):
- raise SystemExit(
- "Findings can be exported after the scan completes or preserves stopped results."
- )
- scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"]))
- require_recorded_manifest_digest(scan, scan_dir)
- verify_manifest_binding(scan, read_json_object(scan_dir / ARTIFACTS["manifest"]))
- try:
- manifest, _, _ = finalize_scan(
- scan_dir,
- expected_coverage_mode=expected_coverage_mode(scan),
- )
- except ContractError as exc:
- raise SystemExit(str(exc)) from exc
- verify_manifest_binding(scan, manifest)
- manifest_digest = published_manifest_digest(scan_dir, manifest)
- pin_legacy_manifest_digest(connection, scan["id"], manifest_digest)
- if args.format == "json":
- path = artifact_path(scan_dir, ARTIFACTS["findings"], required=True)
- elif args.format == "sarif":
- try:
- write_sarif_projection(scan_dir)
- except ContractError as exc:
- raise SystemExit(str(exc)) from exc
- path = artifact_path(scan_dir, "exports/results.sarif", required=True)
- else:
- path = write_csv_export(connection, scan)
- if path is None:
- raise SystemExit(f"Could not export Codex Security findings as {args.format.upper()}.")
- return {
- "export": {"format": args.format, "path": str(path)},
- "scan": scan_result(connection, scan),
- "workspace": workspace_state(connection, scan["workspace_id"]),
- }
-
-
-def write_csv_export(connection: sqlite3.Connection, scan: sqlite3.Row) -> Path:
- scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"]))
- output = io.StringIO(newline="")
- writer = csv.writer(output)
- deep_scan = scan["mode"] == "deep"
- candidate_ids_by_occurrence: dict[str, str] = {}
- if deep_scan:
- findings_document = read_json_object(scan_dir / ARTIFACTS["findings"])
- findings = findings_document.get("findings")
- if not isinstance(findings, list):
- raise SystemExit("findings.json must contain a findings array.")
- for finding in findings:
- if not isinstance(finding, dict):
- raise SystemExit("findings.json entries must be objects.")
- occurrence_id = finding.get("occurrenceId")
- candidate_id = finding_candidate_id(finding)
- if isinstance(occurrence_id, str) and isinstance(candidate_id, str):
- candidate_ids_by_occurrence[occurrence_id] = candidate_id
- columns = (
- "occurrence_id",
- "finding_id",
- *(("candidate_id",) if deep_scan else ()),
- "title",
- "summary",
- "severity",
- "confidence",
- "status",
- "close_reason",
- "note",
- "remediation",
- "path",
- "start_line",
- "end_line",
- )
- writer.writerow(columns)
- for row in finding_export_rows(connection, scan["id"]):
- writer.writerow(
- (
- csv_cell(row["occurrence_id"]),
- csv_cell(row["finding_id"]),
- *(
- (csv_cell(candidate_ids_by_occurrence.get(row["occurrence_id"])),)
- if deep_scan
- else ()
- ),
- csv_cell(row["title"]),
- csv_cell(row["summary"]),
- csv_cell(row["severity"]),
- csv_cell(row["confidence"]),
- csv_cell(row["status"]),
- csv_cell(row["close_reason"]),
- csv_cell(row["note"]),
- csv_cell(row["remediation"]),
- csv_cell(row["relative_path"]),
- row["start_line"],
- row["end_line"],
- )
- )
- try:
- write_scan_local_bytes(
- scan_dir,
- "exports/findings.csv",
- output.getvalue().encode("utf-8"),
- )
- except ContractError as exc:
- raise SystemExit(
- "exports: expected a regular directory inside the scan directory."
- ) from exc
- destination = scan_dir / "exports" / "findings.csv"
- path = available_artifact_path(scan_dir, destination)
- if path is None:
- raise SystemExit("findings.csv: expected a regular file inside the scan directory.")
- return path
-
-
-def finding_export_rows(connection: sqlite3.Connection, scan_id: str) -> sqlite3.Cursor:
- return connection.execute(
- """
- SELECT
- occurrences.id AS occurrence_id,
- occurrences.finding_id,
- occurrences.title,
- occurrences.summary,
- occurrences.severity,
- occurrences.confidence,
- occurrences.remediation,
- COALESCE(triage.status, 'open') AS status,
- triage.close_reason,
- triage.note,
- locations.relative_path,
- locations.start_line,
- locations.end_line
- FROM finding_occurrences AS occurrences
- LEFT JOIN finding_triage AS triage ON triage.occurrence_id = occurrences.id
- LEFT JOIN finding_locations AS locations
- ON locations.occurrence_id = occurrences.id
- AND locations.sort_order = (
- SELECT primary_location.sort_order
- FROM finding_locations AS primary_location
- WHERE primary_location.occurrence_id = occurrences.id
- ORDER BY
- CASE WHEN primary_location.role = 'root_control' THEN 0 ELSE 1 END,
- primary_location.sort_order
- LIMIT 1
- )
- WHERE occurrences.scan_id = ?
- ORDER BY occurrences.created_at, occurrences.id
- """,
- (scan_id,),
- )
-
-
-def require_remediation_transition(current: str, requested: str) -> None:
- allowed = {
- "requested": {"requested", "generated", "failed"},
- "generated": {"generated", "applied", "failed"},
- "applied": {"applied", "verifying", "failed"},
- "verifying": {"verifying", "verified", "failed"},
- "verified": {"verifying", "verified"},
- "failed": {"generated", "applied", "verifying", "verified", "failed"},
- }
- if requested not in allowed.get(current, set()):
- raise SystemExit(f"Finding remediation cannot move from {current} to {requested}.")
-
-
-def require_pending_remediation_action(current: sqlite3.Row, requested: str) -> None:
- pending_action = current["pending_action"]
- if pending_action is not None:
- allowed = {
- "generate": {"generated", "failed"},
- "apply": {"applied", "failed"},
- "verify": {"verifying", "verified", "failed"},
- }
- if requested not in allowed[pending_action]:
- raise SystemExit(
- f"Pending remediation action {pending_action} cannot record state {requested}."
- )
- return
- required_action = {
- ("requested", "generated"): "generate",
- ("generated", "applied"): "apply",
- ("applied", "verifying"): "verify",
- }.get((current["state"], requested))
- if required_action is not None:
- raise SystemExit(
- f"Request {required_action} before recording remediation state {requested}."
- )
+ return publication.export_findings(_WORKBENCH_PUBLICATION_CONTEXT, connection, args)
def require_reviewed_patch_applied(
@@ -3763,9 +3293,7 @@ def require_canonical_scan_directory(scan_dir: Path) -> Path:
# Re-check privacy so a shared parent cannot be used to substitute forged artifacts.
if os.name != "nt":
if stat.S_IMODE(metadata.st_mode) & 0o077:
- raise SystemExit(
- "Scan directory must not be accessible to other users (chmod 700)."
- )
+ raise SystemExit("Scan directory must not be accessible to other users (chmod 700).")
geteuid = getattr(os, "geteuid", None)
effective_uid = geteuid() if geteuid is not None else None
if effective_uid is not None and metadata.st_uid != effective_uid:
@@ -3775,9 +3303,7 @@ def require_canonical_scan_directory(scan_dir: Path) -> Path:
parent_metadata = parent.lstat()
except OSError as exc:
raise SystemExit("Scan output parent could not be inspected.") from exc
- if not stat.S_ISDIR(parent_metadata.st_mode) or stat.S_ISLNK(
- parent_metadata.st_mode
- ):
+ if not stat.S_ISDIR(parent_metadata.st_mode) or stat.S_ISLNK(parent_metadata.st_mode):
raise SystemExit("Scan output parent must be a non-symlink directory.")
if effective_uid is not None and parent_metadata.st_uid not in {
0,
@@ -3811,6 +3337,25 @@ def reject_non_finite_json(value: str) -> None:
raise ValueError(f"non-finite JSON number {value!r} is not supported")
+_WORKBENCH_PUBLICATION_CONTEXT = publication.WorkbenchPublicationContext(
+ ARTIFACTS=ARTIFACTS,
+ artifact_path=artifact_path,
+ available_artifact_path=available_artifact_path,
+ database_path=database_path,
+ expected_coverage_mode=expected_coverage_mode,
+ now=now,
+ pin_legacy_manifest_digest=pin_legacy_manifest_digest,
+ published_manifest_digest=published_manifest_digest,
+ read_json_object=read_json_object,
+ require_canonical_scan_directory=require_canonical_scan_directory,
+ require_recorded_manifest_digest=require_recorded_manifest_digest,
+ require_scan=require_scan,
+ scan_result=scan_result,
+ verify_manifest_binding=verify_manifest_binding,
+ workspace_state=workspace_state,
+)
+
+
_WORKBENCH_DB_CONTEXT = saved_results.WorkbenchDbContext(
ARTIFACTS=ARTIFACTS,
artifact_path=artifact_path,
@@ -4017,14 +3562,19 @@ def main() -> None:
result = release_finding_remediation_claim(connection, args)
elif args.command == "cancel-finding-remediation-request":
result = scan_context(
- connection, remediation.cancel_finding_remediation_request(connection, args)
+ connection,
+ remediation.cancel_finding_remediation_request(connection, args),
)
elif args.command == "set-finding-remediation":
result = set_finding_remediation(connection, args)
elif args.command == "prepare-linear-publication":
- result = prepare_linear_publication(connection, args)
+ result = publication.prepare_linear_publication(
+ _WORKBENCH_PUBLICATION_CONTEXT, connection, args
+ )
elif args.command == "record-linear-publications":
- result = record_linear_publications(connection, args)
+ result = publication.record_linear_publications(
+ _WORKBENCH_PUBLICATION_CONTEXT, connection, args
+ )
elif args.command == "export-findings":
result = export_findings(connection, args)
elif args.command == "database-info":
@@ -4035,7 +3585,9 @@ def main() -> None:
result = dashboard(connection, json.load(sys.stdin))
elif args.command == "store-findings":
payload = json.load(sys.stdin)
- result = store_findings(connection, payload["entries"], now(), payload.get("repositoryId"))
+ result = store_findings(
+ connection, payload["entries"], now(), payload.get("repositoryId")
+ )
elif args.command == "find-potential-duplicates":
result = find_potential_duplicates(connection, args.finding_id, args.repository_id)
elif args.command == "store-dedupe-groups":
diff --git a/plugins/codex-security/scripts/workbench_finding_workflows.py b/plugins/codex-security/scripts/workbench_finding_workflows.py
index dee35cbf2..907b9f534 100644
--- a/plugins/codex-security/scripts/workbench_finding_workflows.py
+++ b/plugins/codex-security/scripts/workbench_finding_workflows.py
@@ -3,8 +3,8 @@
from __future__ import annotations
import argparse
-import json
import hashlib
+import json
import sqlite3
import sys
from pathlib import Path
@@ -69,7 +69,9 @@ def save_workflow(connection: sqlite3.Connection, state: dict[str, Any], timesta
if "pendingWrite" in state["stages"]["dedupe"]:
results["dedupePendingWrite"] = state["stages"]["dedupe"]["pendingWrite"]
values.update(
- results_json=json.dumps(results, allow_nan=False), created_at=timestamp, updated_at=timestamp
+ results_json=json.dumps(results, allow_nan=False),
+ created_at=timestamp,
+ updated_at=timestamp,
)
updates = ", ".join(
f"{column} = excluded.{column}" for column in values if column not in {"id", "created_at"}
@@ -122,12 +124,16 @@ def finding_workflow(
return {"workflow": read_workflow(connection, workflow_id)}
if payload["action"] == "source":
target = Path(payload["repository"]).resolve(strict=True)
- return {"source": {
- "repository": str(target),
- "revision": git_revision(target),
- "refsDigest": hashlib.sha256((git_output(target, "show-ref") or "").encode()).hexdigest(),
- "content": directory_content_digest(target, include_ignored=True),
- }}
+ return {
+ "source": {
+ "repository": str(target),
+ "revision": git_revision(target),
+ "refsDigest": hashlib.sha256(
+ (git_output(target, "show-ref") or "").encode()
+ ).hexdigest(),
+ "content": directory_content_digest(target, include_ignored=True),
+ }
+ }
if payload["action"] == "get-review":
row = connection.execute(
"SELECT result_json FROM finding_workflow_reviews WHERE workflow_id = ? AND review_key = ?",
@@ -148,11 +154,23 @@ def finding_workflow(
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(workflow_id, review_key) DO NOTHING""",
(
- workflow_id, payload["key"], binding["version"], binding["codexVersion"],
- source["repository"], source["revision"], source["refsDigest"], source["content"],
- scope.get("repositoryId"), scope.get("allRepositories"), binding["model"],
- binding["effort"], binding.get("settingsDigest"), binding["promptDigest"],
- binding["contractDigest"], json.dumps(payload["result"], allow_nan=False), timestamp,
+ workflow_id,
+ payload["key"],
+ binding["version"],
+ binding["codexVersion"],
+ source["repository"],
+ source["revision"],
+ source["refsDigest"],
+ source["content"],
+ scope.get("repositoryId"),
+ scope.get("allRepositories"),
+ binding["model"],
+ binding["effort"],
+ binding.get("settingsDigest"),
+ binding["promptDigest"],
+ binding["contractDigest"],
+ json.dumps(payload["result"], allow_nan=False),
+ timestamp,
),
)
return {}
diff --git a/plugins/codex-security/scripts/workbench_findings.py b/plugins/codex-security/scripts/workbench_findings.py
index bcef8469b..5446ea3b2 100644
--- a/plugins/codex-security/scripts/workbench_findings.py
+++ b/plugins/codex-security/scripts/workbench_findings.py
@@ -157,9 +157,14 @@ def store_dedupe_groups(
for group in groups:
members = sorted(set(group))
# Membership, not input order, identifies a group on retries.
- group_id = "fdg_" + hashlib.sha256(
- json.dumps(members, ensure_ascii=True, separators=(",", ":")).encode("utf-8")
- ).hexdigest()
+ group_id = (
+ "fdg_"
+ + hashlib.sha256(
+ json.dumps(members, ensure_ascii=True, separators=(",", ":")).encode(
+ "utf-8"
+ )
+ ).hexdigest()
+ )
connection.execute(
"INSERT INTO finding_dedupe_groups (id, created_at) VALUES (?, ?) "
"ON CONFLICT(id) DO NOTHING",
diff --git a/plugins/codex-security/scripts/workbench_native_indexes.py b/plugins/codex-security/scripts/workbench_native_indexes.py
index 8ce2492bc..d3d458b3d 100644
--- a/plugins/codex-security/scripts/workbench_native_indexes.py
+++ b/plugins/codex-security/scripts/workbench_native_indexes.py
@@ -101,8 +101,7 @@ def _indexed_findings(connection: sqlite3.Connection) -> Iterator[dict[str, Any]
latest_scan_by_target = dict(
connection.execute(
- "SELECT target_id, id FROM scans "
- "WHERE status = 'complete' ORDER BY started_at, id"
+ "SELECT target_id, id FROM scans WHERE status = 'complete' ORDER BY started_at, id"
)
)
diff --git a/plugins/codex-security/scripts/workbench_publication.py b/plugins/codex-security/scripts/workbench_publication.py
new file mode 100644
index 000000000..fce303457
--- /dev/null
+++ b/plugins/codex-security/scripts/workbench_publication.py
@@ -0,0 +1,520 @@
+"""Validate, record, and export findings from completed scans."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import io
+import os
+import sqlite3
+from collections.abc import Callable
+from contextlib import closing
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote
+
+from finalize_scan_contract import (
+ ContractError,
+ csv_cell,
+ finalize_scan,
+ finding_candidate_id,
+ write_sarif_projection,
+ write_scan_local_bytes,
+)
+
+
+@dataclass(frozen=True)
+class WorkbenchPublicationContext:
+ ARTIFACTS: dict[str, str]
+ artifact_path: Callable[..., Path | None]
+ available_artifact_path: Callable[[Path, Path], Path | None]
+ database_path: Callable[[], Path]
+ expected_coverage_mode: Callable[[sqlite3.Row], str]
+ now: Callable[[], str]
+ pin_legacy_manifest_digest: Callable[[sqlite3.Connection, str, str], None]
+ published_manifest_digest: Callable[[Path, dict[str, Any]], str]
+ read_json_object: Callable[[Path], dict[str, Any]]
+ require_canonical_scan_directory: Callable[[Path], Path]
+ require_recorded_manifest_digest: Callable[[sqlite3.Row, Path], str]
+ require_scan: Callable[[sqlite3.Connection, str], sqlite3.Row]
+ scan_result: Callable[[sqlite3.Connection, sqlite3.Row], dict[str, Any]]
+ verify_manifest_binding: Callable[[sqlite3.Row, dict[str, Any]], None]
+ workspace_state: Callable[[sqlite3.Connection, str], dict[str, Any]]
+
+
+def linear_publication_input(
+ db: WorkbenchPublicationContext,
+ args: argparse.Namespace,
+ *,
+ recording: bool,
+) -> tuple[dict[str, Any], dict[str, str], list[dict[str, str]]]:
+ payload = db.read_json_object(Path(args.input_file))
+ required = {"scanId", "scanDirectory", "destination", "findings"}
+ if recording:
+ required.add("publications")
+ if set(payload) != required:
+ raise SystemExit("Linear publication input contains unexpected or missing fields.")
+
+ scan_id = payload["scanId"]
+ scan_directory = payload["scanDirectory"]
+ destination = payload["destination"]
+ findings = payload["findings"]
+ if not isinstance(scan_id, str) or not isinstance(scan_directory, str):
+ raise SystemExit("Linear publication input must identify the exact completed scan.")
+ if (
+ not isinstance(destination, dict)
+ or not {"type", "teamId"}.issubset(destination)
+ or not set(destination).issubset({"type", "teamId", "projectId"})
+ or destination.get("type") != "linear"
+ or not isinstance(destination.get("teamId"), str)
+ or not destination["teamId"].strip()
+ or (
+ "projectId" in destination
+ and (
+ not isinstance(destination["projectId"], str)
+ or not destination["projectId"].strip()
+ )
+ )
+ ):
+ raise SystemExit(
+ "Linear publication input must identify the exact team and optional project."
+ )
+ if not isinstance(findings, list):
+ raise SystemExit("Linear publication input must include the planned scan findings.")
+
+ seen_finding_ids: set[str] = set()
+ seen_occurrence_ids: set[str] = set()
+ for finding in findings:
+ if (
+ not isinstance(finding, dict)
+ or set(finding) != {"findingId", "occurrenceId"}
+ or not isinstance(finding.get("findingId"), str)
+ or not finding["findingId"].strip()
+ or not isinstance(finding.get("occurrenceId"), str)
+ or not finding["occurrenceId"].strip()
+ ):
+ raise SystemExit("Linear publication input contains an invalid finding identity.")
+ if (
+ finding["findingId"] in seen_finding_ids
+ or finding["occurrenceId"] in seen_occurrence_ids
+ ):
+ raise SystemExit("Linear publication input repeats a finding or occurrence.")
+ seen_finding_ids.add(finding["findingId"])
+ seen_occurrence_ids.add(finding["occurrenceId"])
+
+ return payload, destination, findings
+
+
+def verify_linear_publication_scan(
+ db: WorkbenchPublicationContext,
+ connection: sqlite3.Connection,
+ payload: dict[str, Any],
+ findings: list[dict[str, str]],
+) -> sqlite3.Row:
+ try:
+ scan = db.require_scan(connection, payload["scanId"])
+ except SystemExit as exc:
+ raise SystemExit(
+ "The completed scan is not present in the local Codex Security scan-history database. "
+ "Use the state directory where the scan was completed."
+ ) from exc
+ if scan["id"] != payload["scanId"]:
+ raise SystemExit("Linear publication must use the exact completed scan identifier.")
+ if scan["status"] != "complete":
+ raise SystemExit("Only completed scans can publish findings to Linear.")
+
+ requested_directory = db.require_canonical_scan_directory(Path(payload["scanDirectory"]))
+ recorded_directory = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
+ if os.path.normcase(requested_directory) != os.path.normcase(recorded_directory):
+ raise SystemExit(
+ "The selected scan directory does not match its local Codex Security scan history."
+ )
+ if "seal_manifest_digest" in scan.keys():
+ db.require_recorded_manifest_digest(scan, recorded_directory)
+
+ stored_findings = {
+ row["id"]: row["finding_id"]
+ for row in connection.execute(
+ "SELECT id, finding_id FROM finding_occurrences WHERE scan_id = ?",
+ (scan["id"],),
+ )
+ }
+ for finding in findings:
+ if stored_findings.get(finding["occurrenceId"]) != finding["findingId"]:
+ raise SystemExit(
+ "A selected finding or occurrence does not belong to the completed scan "
+ "in local Codex Security scan history."
+ )
+ if len(stored_findings) != len(findings):
+ raise SystemExit(
+ "The completed scan findings do not exactly match local Codex Security scan history."
+ )
+ return scan
+
+
+def inspect_linear_publication(
+ db: WorkbenchPublicationContext,
+ args: argparse.Namespace,
+) -> dict[str, Any]:
+ payload, destination, findings = linear_publication_input(db, args, recording=False)
+ database_uri = f"file:{quote(str(db.database_path()), safe='')}?mode=ro"
+ with closing(sqlite3.connect(database_uri, uri=True, timeout=5)) as connection:
+ connection.row_factory = sqlite3.Row
+ connection.execute("BEGIN")
+ scan = verify_linear_publication_scan(db, connection, payload, findings)
+ recorded: dict[str, dict[str, str]] = {}
+ if connection.execute(
+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'finding_publications'"
+ ).fetchone():
+ for row in connection.execute(
+ """
+ SELECT finding_id, occurrence_id, external_id, external_url
+ FROM finding_publications
+ WHERE scan_id = ? AND destination_type = ? AND team_id = ? AND project_id IS ?
+ ORDER BY created_at, external_id
+ """,
+ (
+ scan["id"],
+ destination["type"],
+ destination["teamId"],
+ destination.get("projectId"),
+ ),
+ ):
+ recorded.setdefault(
+ row["occurrence_id"],
+ {
+ "findingId": row["finding_id"],
+ "occurrenceId": row["occurrence_id"],
+ "issueIdentifier": row["external_id"],
+ **({"url": row["external_url"]} if row["external_url"] is not None else {}),
+ },
+ )
+ return {
+ "scanId": scan["id"],
+ "destination": destination,
+ "findingCount": len(findings),
+ "recorded": [
+ recorded[finding["occurrenceId"]]
+ for finding in findings
+ if finding["occurrenceId"] in recorded
+ ],
+ }
+
+
+def prepare_linear_publication(
+ db: WorkbenchPublicationContext,
+ connection: sqlite3.Connection,
+ args: argparse.Namespace,
+) -> dict[str, Any]:
+ payload, destination, findings = linear_publication_input(db, args, recording=False)
+ connection.execute("BEGIN IMMEDIATE")
+ try:
+ scan = verify_linear_publication_scan(db, connection, payload, findings)
+ result = {
+ "scanId": scan["id"],
+ "destination": destination,
+ "findingCount": len(findings),
+ }
+ connection.commit()
+ except BaseException:
+ connection.rollback()
+ raise
+ return result
+
+
+def record_linear_publications(
+ db: WorkbenchPublicationContext,
+ connection: sqlite3.Connection,
+ args: argparse.Namespace,
+) -> dict[str, Any]:
+ payload, destination, findings = linear_publication_input(db, args, recording=True)
+ publications = payload["publications"]
+ if not isinstance(publications, list):
+ raise SystemExit("Linear publication results must be an array.")
+ planned = {finding["findingId"]: finding["occurrenceId"] for finding in findings}
+ current: dict[str, dict[str, str]] = {}
+ external_ids: set[str] = set()
+ for publication in publications:
+ if (
+ not isinstance(publication, dict)
+ or not {"findingId", "occurrenceId", "issueIdentifier"}.issubset(publication)
+ or not set(publication).issubset(
+ {"findingId", "occurrenceId", "issueIdentifier", "url"}
+ )
+ or not isinstance(publication.get("findingId"), str)
+ or not isinstance(publication.get("occurrenceId"), str)
+ or not isinstance(publication.get("issueIdentifier"), str)
+ or not publication["issueIdentifier"].strip()
+ or (
+ "url" in publication
+ and (not isinstance(publication["url"], str) or not publication["url"].strip())
+ )
+ ):
+ raise SystemExit("Linear publication results contain an invalid issue association.")
+ finding_id = publication["findingId"]
+ issue_identifier = publication["issueIdentifier"]
+ if planned.get(finding_id) != publication["occurrenceId"]:
+ raise SystemExit(
+ "A created Linear issue does not match its planned finding and occurrence."
+ )
+ if finding_id in current or issue_identifier in external_ids:
+ raise SystemExit("Linear publication results repeat a finding or issue identifier.")
+ current[finding_id] = publication
+ external_ids.add(issue_identifier)
+
+ connection.execute("BEGIN IMMEDIATE")
+ try:
+ scan = verify_linear_publication_scan(db, connection, payload, findings)
+ timestamp = db.now()
+ for publication in publications:
+ conflicting = connection.execute(
+ """
+ SELECT occurrence_id, external_url
+ FROM finding_publications
+ WHERE destination_type = ? AND team_id = ? AND project_id IS ?
+ AND external_id = ?
+ """,
+ (
+ destination["type"],
+ destination["teamId"],
+ destination.get("projectId"),
+ publication["issueIdentifier"],
+ ),
+ ).fetchone()
+ if (
+ conflicting is not None
+ and conflicting["occurrence_id"] != publication["occurrenceId"]
+ ):
+ raise SystemExit(
+ "This Linear issue is already associated with a different finding."
+ )
+ if (
+ conflicting is not None
+ and "url" in publication
+ and conflicting["external_url"] != publication["url"]
+ ):
+ raise SystemExit("This Linear issue is already associated with a different URL.")
+
+ connection.execute(
+ """
+ INSERT INTO finding_publications (
+ scan_id, finding_id, occurrence_id, destination_type,
+ team_id, project_id, external_id, external_url, created_at
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT DO NOTHING
+ """,
+ (
+ scan["id"],
+ publication["findingId"],
+ publication["occurrenceId"],
+ destination["type"],
+ destination["teamId"],
+ destination.get("projectId"),
+ publication["issueIdentifier"],
+ publication.get("url"),
+ timestamp,
+ ),
+ )
+
+ created = []
+ for finding in findings:
+ publication = current.get(finding["findingId"])
+ if publication is None:
+ continue
+ row = connection.execute(
+ """
+ SELECT finding_id, occurrence_id, external_id, external_url
+ FROM finding_publications
+ WHERE scan_id = ? AND occurrence_id = ? AND destination_type = ?
+ AND team_id = ? AND project_id IS ? AND external_id = ?
+ """,
+ (
+ scan["id"],
+ publication["occurrenceId"],
+ destination["type"],
+ destination["teamId"],
+ destination.get("projectId"),
+ publication["issueIdentifier"],
+ ),
+ ).fetchone()
+ if row is None:
+ raise SystemExit("A created Linear issue could not be read from scan history.")
+ created.append(
+ {
+ "findingId": row["finding_id"],
+ "occurrenceId": row["occurrence_id"],
+ "issueIdentifier": row["external_id"],
+ **({"url": row["external_url"]} if row["external_url"] is not None else {}),
+ }
+ )
+ result = {"scanId": scan["id"], "destination": destination, "created": created}
+ connection.commit()
+ except BaseException:
+ connection.rollback()
+ raise
+ return result
+
+
+def export_findings(
+ db: WorkbenchPublicationContext,
+ connection: sqlite3.Connection,
+ args: argparse.Namespace,
+) -> dict[str, Any]:
+ scan = db.require_scan(connection, args.scan_id)
+ if scan["status"] != "complete" and not (
+ scan["status"] == "failed" and scan["seal_manifest_digest"]
+ ):
+ raise SystemExit(
+ "Findings can be exported after the scan completes or preserves stopped results."
+ )
+ scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
+ db.require_recorded_manifest_digest(scan, scan_dir)
+ db.verify_manifest_binding(scan, db.read_json_object(scan_dir / db.ARTIFACTS["manifest"]))
+ try:
+ manifest, _, _ = finalize_scan(
+ scan_dir,
+ expected_coverage_mode=db.expected_coverage_mode(scan),
+ )
+ except ContractError as exc:
+ raise SystemExit(str(exc)) from exc
+ db.verify_manifest_binding(scan, manifest)
+ manifest_digest = db.published_manifest_digest(scan_dir, manifest)
+ db.pin_legacy_manifest_digest(connection, scan["id"], manifest_digest)
+ if args.format == "json":
+ path = db.artifact_path(scan_dir, db.ARTIFACTS["findings"], required=True)
+ elif args.format == "sarif":
+ try:
+ write_sarif_projection(scan_dir)
+ except ContractError as exc:
+ raise SystemExit(str(exc)) from exc
+ path = db.artifact_path(scan_dir, "exports/results.sarif", required=True)
+ else:
+ path = write_csv_export(db, connection, scan)
+ if path is None:
+ raise SystemExit(f"Could not export Codex Security findings as {args.format.upper()}.")
+ return {
+ "export": {"format": args.format, "path": str(path)},
+ "scan": db.scan_result(connection, scan),
+ "workspace": db.workspace_state(connection, scan["workspace_id"]),
+ }
+
+
+def write_csv_export(
+ db: WorkbenchPublicationContext,
+ connection: sqlite3.Connection,
+ scan: sqlite3.Row,
+) -> Path:
+ scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
+ output = io.StringIO(newline="")
+ writer = csv.writer(output)
+ deep_scan = scan["mode"] == "deep"
+ candidate_ids_by_occurrence: dict[str, str] = {}
+ if deep_scan:
+ findings_document = db.read_json_object(scan_dir / db.ARTIFACTS["findings"])
+ findings = findings_document.get("findings")
+ if not isinstance(findings, list):
+ raise SystemExit("findings.json must contain a findings array.")
+ for finding in findings:
+ if not isinstance(finding, dict):
+ raise SystemExit("findings.json entries must be objects.")
+ occurrence_id = finding.get("occurrenceId")
+ candidate_id = finding_candidate_id(finding)
+ if isinstance(occurrence_id, str) and isinstance(candidate_id, str):
+ candidate_ids_by_occurrence[occurrence_id] = candidate_id
+ columns = (
+ "occurrence_id",
+ "finding_id",
+ *(("candidate_id",) if deep_scan else ()),
+ "title",
+ "summary",
+ "severity",
+ "confidence",
+ "status",
+ "close_reason",
+ "note",
+ "remediation",
+ "path",
+ "start_line",
+ "end_line",
+ )
+ writer.writerow(columns)
+ for row in finding_export_rows(connection, scan["id"]):
+ writer.writerow(
+ (
+ csv_cell(row["occurrence_id"]),
+ csv_cell(row["finding_id"]),
+ *(
+ (csv_cell(candidate_ids_by_occurrence.get(row["occurrence_id"])),)
+ if deep_scan
+ else ()
+ ),
+ csv_cell(row["title"]),
+ csv_cell(row["summary"]),
+ csv_cell(row["severity"]),
+ csv_cell(row["confidence"]),
+ csv_cell(row["status"]),
+ csv_cell(row["close_reason"]),
+ csv_cell(row["note"]),
+ csv_cell(row["remediation"]),
+ csv_cell(row["relative_path"]),
+ row["start_line"],
+ row["end_line"],
+ )
+ )
+ try:
+ write_scan_local_bytes(
+ scan_dir,
+ "exports/findings.csv",
+ output.getvalue().encode("utf-8"),
+ )
+ except ContractError as exc:
+ raise SystemExit(
+ "exports: expected a regular directory inside the scan directory."
+ ) from exc
+ destination = scan_dir / "exports" / "findings.csv"
+ path = db.available_artifact_path(scan_dir, destination)
+ if path is None:
+ raise SystemExit("findings.csv: expected a regular file inside the scan directory.")
+ return path
+
+
+def finding_export_rows(connection: sqlite3.Connection, scan_id: str) -> sqlite3.Cursor:
+ return connection.execute(
+ """
+ SELECT
+ occurrences.id AS occurrence_id,
+ occurrences.finding_id,
+ occurrences.title,
+ occurrences.summary,
+ occurrences.severity,
+ occurrences.confidence,
+ occurrences.remediation,
+ COALESCE(triage.status, 'open') AS status,
+ triage.close_reason,
+ triage.note,
+ locations.relative_path,
+ locations.start_line,
+ locations.end_line
+ FROM finding_occurrences AS occurrences
+ LEFT JOIN finding_triage AS triage ON triage.occurrence_id = occurrences.id
+ LEFT JOIN finding_locations AS locations
+ ON locations.occurrence_id = occurrences.id
+ AND locations.sort_order = (
+ SELECT primary_location.sort_order
+ FROM finding_locations AS primary_location
+ WHERE primary_location.occurrence_id = occurrences.id
+ ORDER BY
+ CASE WHEN primary_location.role = 'root_control' THEN 0 ELSE 1 END,
+ primary_location.sort_order
+ LIMIT 1
+ )
+ WHERE occurrences.scan_id = ?
+ ORDER BY occurrences.created_at, occurrences.id
+ """,
+ (scan_id,),
+ )
+
+
+if __name__ == "__main__":
+ argparse.ArgumentParser(description=__doc__).parse_args()
diff --git a/plugins/codex-security/scripts/workbench_remediation.py b/plugins/codex-security/scripts/workbench_remediation.py
index c4faf66c6..275eb683a 100644
--- a/plugins/codex-security/scripts/workbench_remediation.py
+++ b/plugins/codex-security/scripts/workbench_remediation.py
@@ -53,6 +53,43 @@ def remediation_claim_is_active(remediation: sqlite3.Row) -> bool:
return parsed > datetime.now(timezone.utc) - timedelta(seconds=lease_seconds)
+def require_transition(current: str, requested: str) -> None:
+ allowed = {
+ "requested": {"requested", "generated", "failed"},
+ "generated": {"generated", "applied", "failed"},
+ "applied": {"applied", "verifying", "failed"},
+ "verifying": {"verifying", "verified", "failed"},
+ "verified": {"verifying", "verified"},
+ "failed": {"generated", "applied", "verifying", "verified", "failed"},
+ }
+ if requested not in allowed.get(current, set()):
+ raise SystemExit(f"Finding remediation cannot move from {current} to {requested}.")
+
+
+def require_pending_action(current: sqlite3.Row, requested: str) -> None:
+ pending_action = current["pending_action"]
+ if pending_action is not None:
+ allowed = {
+ "generate": {"generated", "failed"},
+ "apply": {"applied", "failed"},
+ "verify": {"verifying", "verified", "failed"},
+ }
+ if requested not in allowed[pending_action]:
+ raise SystemExit(
+ f"Pending remediation action {pending_action} cannot record state {requested}."
+ )
+ return
+ required_action = {
+ ("requested", "generated"): "generate",
+ ("generated", "applied"): "apply",
+ ("applied", "verifying"): "verify",
+ }.get((current["state"], requested))
+ if required_action is not None:
+ raise SystemExit(
+ f"Request {required_action} before recording remediation state {requested}."
+ )
+
+
def register_cancel_finding_remediation_request(subparsers: Any) -> None:
parser = subparsers.add_parser("cancel-finding-remediation-request")
parser.add_argument("--occurrence-id", required=True)
diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py
index d6c2f431a..56288ee4c 100644
--- a/plugins/codex-security/scripts/workbench_saved_results.py
+++ b/plugins/codex-security/scripts/workbench_saved_results.py
@@ -252,17 +252,13 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
return False
-def _recovery_source_digests(
- db: Any, connection: Any, scan: Any
-) -> tuple[dict[str, str], bool]:
+def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[str, str], bool]:
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
frozen_sources: dict[str, str] | None = None
include_parent = True
raw_frozen_sources = scan["retained_source_digests_json"]
if raw_frozen_sources is not None:
- frozen_sources = _source_digests(
- json.loads(raw_frozen_sources), "Saved stopped-scan"
- )
+ frozen_sources = _source_digests(json.loads(raw_frozen_sources), "Saved stopped-scan")
include_parent = False
manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
@@ -276,8 +272,7 @@ def _recovery_source_digests(
if not isinstance(manifest_scan, dict):
raise ContractError("Saved scan manifest has no scan object")
if scan["seal_manifest_digest"] is not None or (
- manifest_scan.get("sealedAt") is not None
- or manifest_scan.get("artifacts") is not None
+ manifest_scan.get("sealedAt") is not None or manifest_scan.get("artifacts") is not None
):
if "preservedSources" in manifest_scan:
published_sources = _source_digests(
@@ -300,9 +295,7 @@ def _recovery_source_digests(
try:
_, digest = _read_saved_result(scan_dir, relative, scan["id"])
except (ContractError, OSError, ValueError) as exc:
- raise ContractError(
- "Frozen stopped-scan checkpoint set is incomplete."
- ) from exc
+ raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
if digest != expected_digest:
raise ContractError("checkpoint changed after the scan stopped")
@@ -311,13 +304,9 @@ def _recovery_source_digests(
"FROM deep_scan_workers WHERE scan_id = ?",
(scan["id"],),
).fetchall()
- for relative in (
- set(_saved_result_paths(scan_dir, workers)) - recovery_sources.keys()
- ):
+ for relative in set(_saved_result_paths(scan_dir, workers)) - recovery_sources.keys():
try:
- _, recovery_sources[relative] = _read_saved_result(
- scan_dir, relative, scan["id"]
- )
+ _, recovery_sources[relative] = _read_saved_result(scan_dir, relative, scan["id"])
except (ContractError, OSError, ValueError):
continue
return recovery_sources, include_parent
@@ -795,8 +784,10 @@ def merge_saved_results(
and coverage.get("completeness") in {"complete", "unknown"}
):
coverage["completeness"] = "partial"
- if superseded and not stopped and all(
- valid_finding(finding) for finding in (parent["findings"] if parent else [])
+ if (
+ superseded
+ and not stopped
+ and all(valid_finding(finding) for finding in (parent["findings"] if parent else []))
):
continue
if "threatModel" not in manifest["scan"] and isinstance(draft.get("threatModel"), dict):
@@ -1277,9 +1268,7 @@ def recover_scan_results(db: Any, connection: Any, args: Any) -> dict[str, Any]:
raise SystemExit("Only a stopped scan can recover terminal results.")
if scan["canceled_at"] is not None:
raise SystemExit("Canceled scans cannot recover terminal results.")
- recovery_source_digests, include_parent = _recovery_source_digests(
- db, connection, scan
- )
+ recovery_source_digests, include_parent = _recovery_source_digests(db, connection, scan)
if not preserve_scan_results_locked(
db,
connection,
@@ -1512,9 +1501,7 @@ def cancel_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
return db.workspace_state(connection, scan["workspace_id"])
-def preserve_stopped_results_after_transition(
- db: Any, connection: Any, scan_id: str
-) -> None:
+def preserve_stopped_results_after_transition(db: Any, connection: Any, scan_id: str) -> None:
try:
published = preserve_scan_results_locked(db, connection, scan_id)
except (ContractError, OSError, SystemExit, ValueError) as exc:
diff --git a/plugins/codex-security/scripts/workbench_scan_history.py b/plugins/codex-security/scripts/workbench_scan_history.py
index f231f0d74..522e68ec9 100644
--- a/plugins/codex-security/scripts/workbench_scan_history.py
+++ b/plugins/codex-security/scripts/workbench_scan_history.py
@@ -284,8 +284,7 @@ def list_unmatched_scan_pairs(
if scan["id"] not in matching_findings:
backfill_finding_details(connection, scan)
matching_findings[scan["id"]] = [
- _matching_input(row)
- for row in _scan_findings(connection, scan["id"]).values()
+ _matching_input(row) for row in _scan_findings(connection, scan["id"]).values()
]
batches.append(
{
diff --git a/plugins/codex-security/scripts/workbench_scan_usage.py b/plugins/codex-security/scripts/workbench_scan_usage.py
index db987d20c..a9f2afeda 100644
--- a/plugins/codex-security/scripts/workbench_scan_usage.py
+++ b/plugins/codex-security/scripts/workbench_scan_usage.py
@@ -215,18 +215,10 @@ def _codex_state_database() -> Path | None:
return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None
configured_home = os.environ.get("CODEX_HOME", "").strip()
- codex_home = (
- Path(configured_home).expanduser()
- if configured_home
- else Path.home() / ".codex"
- )
+ codex_home = Path(configured_home).expanduser() if configured_home else Path.home() / ".codex"
configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "").strip()
search_roots = [
- *(
- [Path(configured_sqlite_home).expanduser()]
- if configured_sqlite_home
- else []
- ),
+ *([Path(configured_sqlite_home).expanduser()] if configured_sqlite_home else []),
codex_home,
codex_home / "sqlite",
]
@@ -363,9 +355,7 @@ def _require_state_columns(
}
columns = {str(row["name"]) for row in connection.execute(statements[table])}
if not required.issubset(columns):
- raise ValueError(
- "Codex state graph does not expose the required thread columns."
- )
+ raise ValueError("Codex state graph does not expose the required thread columns.")
def _rollout_path(value: object) -> Path | None:
@@ -411,9 +401,7 @@ def _read_rollout_usage(
event = json.loads(raw_line)
except (UnicodeError, ValueError):
if line_number == 1:
- raise ValueError(
- "The rollout session metadata is unreadable."
- ) from None
+ raise ValueError("The rollout session metadata is unreadable.") from None
if boundary_reached:
warnings.add("rollout_record_invalid")
continue
@@ -456,10 +444,7 @@ def _read_rollout_usage(
warnings.add("thread_outside_scan_window")
return total, warnings
boundary_reached = True
- elif (
- event.get("type") == "event_msg"
- and payload.get("type") == "token_count"
- ):
+ elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
inherited_usage = _token_snapshot(payload)
if inherited_usage is not None:
previous = inherited_usage
@@ -538,9 +523,7 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
usage = info.get("total_token_usage")
if not isinstance(usage, dict):
return None
- cache_write = usage.get(
- "cache_write_input_tokens", usage.get("cache_write_tokens", 0)
- )
+ cache_write = usage.get("cache_write_input_tokens", usage.get("cache_write_tokens", 0))
legacy_cache_write = usage.get("cache_write_tokens")
input_tokens = usage.get("input_tokens")
cached_input_tokens = usage.get("cached_input_tokens", 0)
@@ -556,9 +539,7 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
result: dict[str, int] = {}
for source_key, result_key in TOKEN_FIELDS.items():
value = (
- cache_write
- if source_key == "cache_write_input_tokens"
- else usage.get(source_key, 0)
+ cache_write if source_key == "cache_write_input_tokens" else usage.get(source_key, 0)
)
if type(value) is not int or value < 0:
return None
@@ -567,10 +548,7 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
):
return None
result[result_key] = value
- if (
- result["cachedInputTokens"] + result["cacheWriteInputTokens"]
- > result["inputTokens"]
- ):
+ if result["cachedInputTokens"] + result["cacheWriteInputTokens"] > result["inputTokens"]:
return None
result["totalTokens"] = result["inputTokens"] + result["outputTokens"]
return result
@@ -599,9 +577,7 @@ def _timestamp(value: object) -> datetime | None:
return parsed.astimezone(timezone.utc) if parsed.tzinfo is not None else None
-def _unavailable_usage(
- reason: str, *, warnings: set[str] | None = None
-) -> dict[str, Any]:
+def _unavailable_usage(reason: str, *, warnings: set[str] | None = None) -> dict[str, Any]:
return {
"coverage": "unavailable",
"source": "codex_rollout",
diff --git a/plugins/codex-security/scripts/workbench_schema.py b/plugins/codex-security/scripts/workbench_schema.py
index ff83eb326..653b38f68 100644
--- a/plugins/codex-security/scripts/workbench_schema.py
+++ b/plugins/codex-security/scripts/workbench_schema.py
@@ -842,11 +842,21 @@ def migrate_finding_workflow_review_columns(connection: sqlite3.Connection) -> N
model = ?, effort = ?, settings_digest = ?, prompt_digest = ?, contract_digest = ?
WHERE workflow_id = ? AND review_key = ?""",
(
- binding["version"], binding["codexVersion"], source["repository"],
- source["revision"], source["refsDigest"], source["content"],
- scope.get("repositoryId"), scope.get("allRepositories"), binding["model"],
- binding["effort"], binding.get("settingsDigest"), binding["promptDigest"],
- binding["contractDigest"], row["workflow_id"], row["review_key"],
+ binding["version"],
+ binding["codexVersion"],
+ source["repository"],
+ source["revision"],
+ source["refsDigest"],
+ source["content"],
+ scope.get("repositoryId"),
+ scope.get("allRepositories"),
+ binding["model"],
+ binding["effort"],
+ binding.get("settingsDigest"),
+ binding["promptDigest"],
+ binding["contractDigest"],
+ row["workflow_id"],
+ row["review_key"],
),
)
@@ -867,13 +877,22 @@ def migrate_finding_workflow_columns(connection: sqlite3.Connection) -> None:
scan_status = ?, scan_error = ?, publish_status = ?, publish_error = ?,
dedupe_status = ?, dedupe_error = ?, results_json = ? WHERE id = ?""",
(
- state.get("repositoryPath"), state.get("scanRequestDigest"),
- state.get("scanId"), state.get("scanDir"), state.get("artifactDigest"),
- state.get("destination"), scope.get("repositoryId"), scope.get("allRepositories"),
- stages["scan"]["status"], stages["scan"].get("error"),
- stages["publish"]["status"], stages["publish"].get("error"),
- stages["dedupe"]["status"], stages["dedupe"].get("error"),
- json.dumps(results, allow_nan=False), row["id"],
+ state.get("repositoryPath"),
+ state.get("scanRequestDigest"),
+ state.get("scanId"),
+ state.get("scanDir"),
+ state.get("artifactDigest"),
+ state.get("destination"),
+ scope.get("repositoryId"),
+ scope.get("allRepositories"),
+ stages["scan"]["status"],
+ stages["scan"].get("error"),
+ stages["publish"]["status"],
+ stages["publish"].get("error"),
+ stages["dedupe"]["status"],
+ stages["dedupe"].get("error"),
+ json.dumps(results, allow_nan=False),
+ row["id"],
),
)
@@ -979,12 +998,8 @@ def apply_migrations(
def normalize_pre_release_execution_profile_migrations(
connection: sqlite3.Connection, timestamp: str
) -> None:
- scan_columns = {
- row["name"] for row in connection.execute("PRAGMA table_info(scans)")
- }
- workspace_columns = {
- row["name"] for row in connection.execute("PRAGMA table_info(workspaces)")
- }
+ scan_columns = {row["name"] for row in connection.execute("PRAGMA table_info(scans)")}
+ workspace_columns = {row["name"] for row in connection.execute("PRAGMA table_info(workspaces)")}
legacy_columns = {"execution_model", "reasoning_effort"}
renamed_columns = {
"legacy_execution_model",
@@ -1332,9 +1347,7 @@ def normalize_mirror_lineage_migrations(connection: sqlite3.Connection) -> None:
if not any(migrations.get(version) == name for version, name in mirror_names.items()):
return
if migrations != mirror_names:
- raise SystemExit(
- "The Codex Security database has an unsupported mirror migration history."
- )
+ raise SystemExit("The Codex Security database has an unsupported mirror migration history.")
for old_version, new_version in ((30, 32), (29, 31)):
connection.execute(
"UPDATE schema_migrations SET version = ? WHERE version = ? AND name = ?",
diff --git a/plugins/codex-security/scripts/workbench_target.py b/plugins/codex-security/scripts/workbench_target.py
index 1b1cb0500..a869ebb22 100644
--- a/plugins/codex-security/scripts/workbench_target.py
+++ b/plugins/codex-security/scripts/workbench_target.py
@@ -424,7 +424,10 @@ def source_directory_snapshot_paths(target: Path) -> list[Path]:
paths.append(path)
metadata = path.lstat()
# Name-surrogate reparse points include Windows directory junctions.
- if stat.S_ISDIR(metadata.st_mode) and not getattr(metadata, "st_reparse_tag", 0) & 0x20000000:
+ if (
+ stat.S_ISDIR(metadata.st_mode)
+ and not getattr(metadata, "st_reparse_tag", 0) & 0x20000000
+ ):
pending.append(path)
return sorted(paths)
@@ -438,7 +441,11 @@ def directory_content_digest(
excluded_relative.append(path.relative_to(target))
except ValueError:
continue
- paths = source_directory_snapshot_paths(target) if include_ignored else git_directory_snapshot_paths(target)
+ paths = (
+ source_directory_snapshot_paths(target)
+ if include_ignored
+ else git_directory_snapshot_paths(target)
+ )
if paths is None:
paths = sorted(target.rglob("*"))
digest = hashlib.sha256()
@@ -560,9 +567,7 @@ def copy_git_worktree_files(source: Path, destination: Path, excluded: tuple[Pat
if nested_git_dir is None:
raise SystemExit(f"Could not inspect nested Git working tree: {relative}")
copy_git_worktree_files(source_path, destination_path, excluded)
- (destination_path / ".git").write_text(
- f"gitdir: {nested_git_dir}\n", encoding="utf-8"
- )
+ (destination_path / ".git").write_text(f"gitdir: {nested_git_dir}\n", encoding="utf-8")
else:
raise SystemExit(f"Unsupported Git working-tree file type: {relative}")
copied_target = destination if pathspec == "." else destination / pathspec
diff --git a/plugins/codex-security/scripts/workbench_validation.py b/plugins/codex-security/scripts/workbench_validation.py
index 608ede87a..1d8681c97 100644
--- a/plugins/codex-security/scripts/workbench_validation.py
+++ b/plugins/codex-security/scripts/workbench_validation.py
@@ -58,11 +58,7 @@ def user_text(value: str | None) -> str | None:
def user_context_argument(args: argparse.Namespace) -> str | None:
- value = (
- sys.stdin.read()
- if getattr(args, "user_context_stdin", False)
- else args.user_context
- )
+ value = sys.stdin.read() if getattr(args, "user_context_stdin", False) else args.user_context
return user_text(value)
diff --git a/plugins/codex-security/skills/assess-patch-risk/SKILL.md b/plugins/codex-security/skills/assess-patch-risk/SKILL.md
index c533a629b..fdc6b23c5 100644
--- a/plugins/codex-security/skills/assess-patch-risk/SKILL.md
+++ b/plugins/codex-security/skills/assess-patch-risk/SKILL.md
@@ -60,11 +60,7 @@ Return both a concise Markdown report and a JSON object conforming to [`../../sc
7. top risk drivers, protective factors, and status-quo risk; and
8. unknowns plus the bounded evidence plan when held.
-This skill lives at `<plugin-root>/skills/assess-patch-risk/SKILL.md`, so
-`<plugin-root>` is two directories up. Resolve `<python_command>` to the
-configured Python interpreter (`"$PYTHON"` in POSIX shells or
-`& "$env:PYTHON"` in PowerShell), otherwise use `python` on Windows and
-`python3` on Unix-like hosts.
+This skill lives at `<plugin-root>/skills/assess-patch-risk/SKILL.md`, so `<plugin-root>` is two directories up. Resolve `<python_command>` to the configured Python interpreter (`"$PYTHON"` in POSIX shells or `& "$env:PYTHON"` in PowerShell), otherwise use `python` on Windows and `python3` on Unix-like hosts.
Before returning the result, validate the JSON from any working directory with:
diff --git a/plugins/codex-security/tests/test_workbench_completion_binding.py b/plugins/codex-security/tests/test_workbench_completion_binding.py
index 401496ccf..d933cb66e 100644
--- a/plugins/codex-security/tests/test_workbench_completion_binding.py
+++ b/plugins/codex-security/tests/test_workbench_completion_binding.py
@@ -517,9 +517,7 @@ def test_deep_completion_rejects_invalid_target_even_with_recoverable_inventory(
assert recorded["resultsRecoveryNeeded"] is True
assert json.loads(coverage_path.read_text())["inventoryStrategy"] == ""
- recovered = run_workbench(
- state_dir, "recover-scan-results", "--scan-id", scan_id
- )["scan"]
+ recovered = run_workbench(state_dir, "recover-scan-results", "--scan-id", scan_id)["scan"]
assert recovered["resultsRecoveryNeeded"] is False
preserved_coverage = json.loads(coverage_path.read_text())
diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
index d6d184796..452f9ad74 100644
--- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
+++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py
@@ -728,9 +728,12 @@ def test_workbench_reconciles_monorepo_migration_lineage() -> None:
(31, "freeze stopped scan source digests", retained_digest_applied_at),
(32, "separate deep scan publication failures", publication_error_applied_at),
]
- assert connection.execute(
- "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'finding_publications'"
- ).fetchone() is not None
+ assert (
+ connection.execute(
+ "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'finding_publications'"
+ ).fetchone()
+ is not None
+ )
assert {
row[0]
for row in connection.execute(
diff --git a/plugins/codex-security/tests/test_workbench_standard_deep_results.py b/plugins/codex-security/tests/test_workbench_standard_deep_results.py
index 8ca01b9fd..de2b5ec22 100644
--- a/plugins/codex-security/tests/test_workbench_standard_deep_results.py
+++ b/plugins/codex-security/tests/test_workbench_standard_deep_results.py
@@ -246,8 +246,7 @@ def test_explicit_recovery_rejects_changed_frozen_source(tmp_path: Path) -> None
preserved_sources = json.loads(original_manifest)["scan"]["preservedSources"]
with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:
original_record = connection.execute(
- "SELECT seal_manifest_digest, retained_source_digests_json "
- "FROM scans WHERE id = ?",
+ "SELECT seal_manifest_digest, retained_source_digests_json FROM scans WHERE id = ?",
(scan_id,),
).fetchone()
frozen_path = scan_dir / next(iter(preserved_sources))
@@ -256,13 +255,9 @@ def test_explicit_recovery_rejects_changed_frozen_source(tmp_path: Path) -> None
late = copy.deepcopy(checkpoint)
late["findings"][0]["locations"][0]["startLine"] = 91
late["findings"][0]["locations"][0]["endLine"] = 92
- write_checkpoint(
- result_path.parent / "attempts" / "attempt-01" / "checkpoints", late
- )
+ write_checkpoint(result_path.parent / "attempts" / "attempt-01" / "checkpoints", late)
assert (
- run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"][
- "resultsRecoveryNeeded"
- ]
+ run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"]["resultsRecoveryNeeded"]
is True
)
@@ -282,8 +277,7 @@ def test_explicit_recovery_rejects_changed_frozen_source(tmp_path: Path) -> None
with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:
assert (
connection.execute(
- "SELECT seal_manifest_digest, retained_source_digests_json "
- "FROM scans WHERE id = ?",
+ "SELECT seal_manifest_digest, retained_source_digests_json FROM scans WHERE id = ?",
(scan_id,),
).fetchone()
== original_record
@@ -396,9 +390,7 @@ def test_explicit_recovery_preserves_sealed_parent_with_empty_source_map(
for filename in ("findings.json", "coverage.json", "scan-manifest.json"):
(scan_dir / filename).write_bytes((contract_dir / filename).read_bytes())
sealed_manifest = (scan_dir / "scan-manifest.json").read_bytes()
- parent_finding = json.loads((contract_dir / "findings.json").read_text())[
- "findings"
- ][0]
+ parent_finding = json.loads((contract_dir / "findings.json").read_text())["findings"][0]
with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:
connection.execute(
"UPDATE scans SET seal_manifest_digest = ? WHERE id = ?",
@@ -415,10 +407,7 @@ def test_explicit_recovery_preserves_sealed_parent_with_empty_source_map(
environment={"CODEX_HOME": str(codex_home)},
)
assert (
- json.loads((scan_dir / "scan-manifest.json").read_text())["scan"][
- "preservedSources"
- ]
- == {}
+ json.loads((scan_dir / "scan-manifest.json").read_text())["scan"]["preservedSources"] == {}
)
late_finding = copy.deepcopy(parent_finding)
@@ -498,9 +487,7 @@ def test_explicit_recovery_preserves_sealed_parent_with_empty_source_map(
later_finding["title"] = "Later checkpoint finding"
later = copy.deepcopy(late)
later["findings"] = [later_finding]
- write_checkpoint(
- result_path.parent / "attempts" / "attempt-02" / "checkpoints", later
- )
+ write_checkpoint(result_path.parent / "attempts" / "attempt-02" / "checkpoints", later)
recovered_again = run_workbench(
state_dir,
@@ -558,9 +545,7 @@ def test_explicit_recovery_retries_frozen_parent_after_write_failure(
for filename in ("findings.json", "coverage.json", "scan-manifest.json"):
(scan_dir / filename).write_bytes((contract_dir / filename).read_bytes())
parent_manifest = (scan_dir / "scan-manifest.json").read_bytes()
- parent_finding = json.loads((contract_dir / "findings.json").read_text())[
- "findings"
- ][0]
+ parent_finding = json.loads((contract_dir / "findings.json").read_text())["findings"][0]
parent_scan = json.loads(parent_manifest)["scan"]
if published_sources is None:
assert "preservedSources" not in parent_scan
@@ -617,16 +602,13 @@ def test_explicit_recovery_retries_frozen_parent_after_write_failure(
assert (scan_dir / "scan-manifest.json").read_bytes() == parent_manifest
with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:
seal_digest, frozen_before = connection.execute(
- "SELECT seal_manifest_digest, retained_source_digests_json "
- "FROM scans WHERE id = ?",
+ "SELECT seal_manifest_digest, retained_source_digests_json FROM scans WHERE id = ?",
(scan_id,),
).fetchone()
assert seal_digest is not None
assert json.loads(frozen_before)
assert (
- run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"][
- "resultsRecoveryNeeded"
- ]
+ run_workbench(state_dir, "get-scan", "--scan-id", scan_id)["scan"]["resultsRecoveryNeeded"]
is True
)
@@ -644,9 +626,9 @@ def test_explicit_recovery_retries_frozen_parent_after_write_failure(
parent_finding["title"],
late_finding["title"],
}
- published_sources = json.loads((scan_dir / "scan-manifest.json").read_text())[
- "scan"
- ]["preservedSources"]
+ published_sources = json.loads((scan_dir / "scan-manifest.json").read_text())["scan"][
+ "preservedSources"
+ ]
frozen_sources = json.loads(frozen_before)
assert published_sources.items() >= frozen_sources.items()
parent_sources = published_sources.keys() - frozen_sources.keys()
@@ -655,12 +637,15 @@ def test_explicit_recovery_retries_frozen_parent_after_write_failure(
assert parent_source.startswith("checkpoints/")
assert Path(parent_source).stem == published_sources[parent_source]
with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:
- assert json.loads(
- connection.execute(
- "SELECT retained_source_digests_json FROM scans WHERE id = ?",
- (scan_id,),
- ).fetchone()[0]
- ) == published_sources
+ assert (
+ json.loads(
+ connection.execute(
+ "SELECT retained_source_digests_json FROM scans WHERE id = ?",
+ (scan_id,),
+ ).fetchone()[0]
+ )
+ == published_sources
+ )
def test_unsealed_manifest_without_saved_results_does_not_offer_recovery(
diff --git a/sdk/typescript/tests-ts/publication-store.test.ts b/sdk/typescript/tests-ts/publication-store.test.ts
index d565f19c2..edf19552a 100644
--- a/sdk/typescript/tests-ts/publication-store.test.ts
+++ b/sdk/typescript/tests-ts/publication-store.test.ts
@@ -209,10 +209,12 @@ describe("read-only publication history", () => {
"-c",
[
"import sys",
+ "from dataclasses import replace",
"from pathlib import PureWindowsPath",
"from urllib.parse import unquote, urlsplit",
"sys.path.insert(0, sys.argv[1])",
"import workbench_db as workbench",
+ "import workbench_publication as publication",
"class Captured(Exception): pass",
"def capture(filename, **options):",
" parsed = urlsplit(filename)",
@@ -220,12 +222,12 @@ describe("read-only publication history", () => {
" assert unquote(parsed.path) == str(path)",
" assert parsed.query == 'mode=ro' and options == {'uri': True, 'timeout': 5}",
" raise Captured",
- "workbench.sqlite3.connect = capture",
- "workbench.linear_publication_input = lambda *_args, **_options: ({}, {}, [])",
+ "publication.sqlite3.connect = capture",
+ "publication.linear_publication_input = lambda *_args, **_options: ({}, {}, [])",
"for value in ['C:/state/history.sqlite3', '//server/share/state/history.sqlite3', '//?/C:/state/history.sqlite3', '//?/UNC/server/share/history.sqlite3']:",
" path = PureWindowsPath(value)",
- " workbench.database_path = lambda: path",
- " try: workbench.inspect_linear_publication(None)",
+ " context = replace(workbench._WORKBENCH_PUBLICATION_CONTEXT, database_path=lambda path=path: path)",
+ " try: publication.inspect_linear_publication(context, None)",
" except Captured: pass",
" else: raise AssertionError('SQLite connection was not attempted')",
].join("\n"),