Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test(workbench): share setup for patch cancellation and retries (#1401)

mldangelo-oaiaaed5a9e427c
diff --git a/plugins/codex-security/tests/test_workbench_remediation_cancellation.py b/plugins/codex-security/tests/test_workbench_remediation_cancellation.py
index a7f6a6772..ace5d85f4 100644
--- a/plugins/codex-security/tests/test_workbench_remediation_cancellation.py
+++ b/plugins/codex-security/tests/test_workbench_remediation_cancellation.py
@@ -6,7 +6,11 @@ import uuid
from pathlib import Path
from workbench_test_support import (
+ cancel_remediation_request,
+ request_remediation,
+ request_remediation_action,
run_workbench,
+ set_remediation,
start_saved_scan,
write_completed_contract,
)
@@ -25,52 +29,22 @@ def test_cancel_finding_remediation_request_restores_previous_state(tmp_path: Pa
canceled_request_id = str(uuid.uuid4())
canceled_token = str(uuid.uuid4())
- requested_initial = run_workbench(
- state_dir,
- "request-finding-remediation",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- canceled_request_id,
- "--action-token",
- canceled_token,
+ requested_initial = request_remediation(
+ state_dir, occurrence_id, canceled_request_id, canceled_token
)["scan"]
- canceled_initial = run_workbench(
- state_dir,
- "cancel-finding-remediation-request",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- canceled_request_id,
- "--action-token",
- canceled_token,
+ canceled_initial = cancel_remediation_request(
+ state_dir, occurrence_id, canceled_request_id, canceled_token
)["scan"]
assert canceled_initial["findings"][0]["remediationState"] == {"state": "idle"}
assert canceled_initial["updatedAt"] > requested_initial["updatedAt"]
- replayed_cancel = run_workbench(
- state_dir,
- "cancel-finding-remediation-request",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- canceled_request_id,
- "--action-token",
- canceled_token,
+ replayed_cancel = cancel_remediation_request(
+ state_dir, occurrence_id, canceled_request_id, canceled_token
)["scan"]
assert replayed_cancel["findings"][0]["remediationState"] == {"state": "idle"}
request_id = str(uuid.uuid4())
generation_token = str(uuid.uuid4())
- run_workbench(
- state_dir,
- "request-finding-remediation",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- generation_token,
- )
+ request_remediation(state_dir, occurrence_id, request_id, generation_token)
patch_path = scan_dir / "remediation.patch"
patch_path.write_text(
"diff --git a/source.txt b/source.txt\n"
@@ -80,18 +54,12 @@ def test_cancel_finding_remediation_request_restores_previous_state(tmp_path: Pa
"-vulnerable\n"
"+fixed\n"
)
- generated = run_workbench(
+ generated = set_remediation(
state_dir,
- "set-finding-remediation",
- "--occurrence-id",
occurrence_id,
- "--request-id",
request_id,
- "--action-token",
generation_token,
- "--expected-version",
"1",
- "--state",
"generated",
"--patch-path",
patch_path.name,
@@ -102,25 +70,9 @@ def test_cancel_finding_remediation_request_restores_previous_state(tmp_path: Pa
replacement_request_id = str(uuid.uuid4())
replacement_token = str(uuid.uuid4())
- run_workbench(
- state_dir,
- "request-finding-remediation",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- replacement_request_id,
- "--action-token",
- replacement_token,
- )
- canceled_replacement = run_workbench(
- state_dir,
- "cancel-finding-remediation-request",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- replacement_request_id,
- "--action-token",
- replacement_token,
+ request_remediation(state_dir, occurrence_id, replacement_request_id, replacement_token)
+ canceled_replacement = cancel_remediation_request(
+ state_dir, occurrence_id, replacement_request_id, replacement_token
)["scan"]
restored = canceled_replacement["findings"][0]["remediationState"]
assert restored["requestId"] == request_id
@@ -129,92 +81,49 @@ def test_cancel_finding_remediation_request_restores_previous_state(tmp_path: Pa
assert restored["patchPath"] == patch_path.name
apply_token = str(uuid.uuid4())
- run_workbench(
- state_dir,
- "request-finding-remediation-action",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--expected-version",
- str(restored["version"]),
- "--action",
- "apply",
- "--action-token",
- apply_token,
+ request_remediation_action(
+ state_dir, occurrence_id, request_id, str(restored["version"]), "apply", apply_token
)
- canceled_apply = run_workbench(
- state_dir,
- "cancel-finding-remediation-request",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- apply_token,
- )["scan"]
+ canceled_apply = cancel_remediation_request(state_dir, occurrence_id, request_id, apply_token)[
+ "scan"
+ ]
restored_after_apply = canceled_apply["findings"][0]["remediationState"]
assert restored_after_apply["state"] == "generated"
assert restored_after_apply["pendingAction"] is None
assert restored_after_apply["actionClaimToken"] is None
apply_token = str(uuid.uuid4())
- apply_requested = run_workbench(
+ apply_requested = request_remediation_action(
state_dir,
- "request-finding-remediation-action",
- "--occurrence-id",
occurrence_id,
- "--request-id",
request_id,
- "--expected-version",
str(restored_after_apply["version"]),
- "--action",
"apply",
- "--action-token",
apply_token,
)["scan"]
subprocess.run(["git", "apply", "--no-index", str(patch_path)], cwd=target, check=True)
- applied = run_workbench(
+ applied = set_remediation(
state_dir,
- "set-finding-remediation",
- "--occurrence-id",
occurrence_id,
- "--request-id",
request_id,
- "--action-token",
apply_token,
- "--expected-version",
str(apply_requested["findings"][0]["remediationState"]["version"]),
- "--state",
"applied",
"--base-revision",
"unversioned",
)["scan"]
verify_token = str(uuid.uuid4())
- run_workbench(
+ request_remediation_action(
state_dir,
- "request-finding-remediation-action",
- "--occurrence-id",
occurrence_id,
- "--request-id",
request_id,
- "--expected-version",
str(applied["findings"][0]["remediationState"]["version"]),
- "--action",
"verify",
- "--action-token",
verify_token,
)
- canceled_verify = run_workbench(
- state_dir,
- "cancel-finding-remediation-request",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- verify_token,
+ canceled_verify = cancel_remediation_request(
+ state_dir, occurrence_id, request_id, verify_token
)["scan"]
restored_after_verify = canceled_verify["findings"][0]["remediationState"]
assert restored_after_verify["state"] == "applied"
diff --git a/plugins/codex-security/tests/test_workbench_remediation_retry.py b/plugins/codex-security/tests/test_workbench_remediation_retry.py
index 399c07329..ffd552c3b 100644
--- a/plugins/codex-security/tests/test_workbench_remediation_retry.py
+++ b/plugins/codex-security/tests/test_workbench_remediation_retry.py
@@ -8,7 +8,14 @@ from pathlib import Path
from typing import Any
from workbench_test_support import (
- run_workbench,
+ cancel_remediation_request,
+ claim_remediation_resend,
+ mark_remediation_delivered,
+ request_remediation,
+ request_remediation_action,
+ scan_command,
+ set_remediation,
+ set_triage,
start_saved_scan,
write_completed_contract,
)
@@ -23,20 +30,8 @@ def update_remediation(
state: str,
*extra: str,
) -> dict[str, Any]:
- result = run_workbench(
- state_dir,
- "set-finding-remediation",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- action_token,
- "--expected-version",
- str(expected_version),
- "--state",
- state,
- *extra,
+ result = set_remediation(
+ state_dir, occurrence_id, request_id, action_token, str(expected_version), state, *extra
)
return result["scan"]["findings"][0]["remediationState"]
@@ -49,31 +44,13 @@ def test_failed_remediation_steps_can_retry_or_regenerate(tmp_path: Path) -> Non
source.write_text("vulnerable\n")
scan_id, scan_dir = start_saved_scan(state_dir, target, tmp_path / "scans")
write_completed_contract(scan_dir, scan_id, target, relative_path=source.name)
- completed = run_workbench(state_dir, "complete-scan", "--scan-id", scan_id)["scan"]
+ completed = scan_command(state_dir, "complete-scan", scan_id)["scan"]
occurrence_id = str(completed["findings"][0]["occurrenceId"])
request_id = str(uuid.uuid4())
action_token = str(uuid.uuid4())
- run_workbench(
- state_dir,
- "request-finding-remediation",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- action_token,
- )
- run_workbench(
- state_dir,
- "mark-finding-remediation-delivered",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- action_token,
- )
+ request_remediation(state_dir, occurrence_id, request_id, action_token)
+ mark_remediation_delivered(state_dir, occurrence_id, request_id, action_token)
failed = update_remediation(
state_dir,
occurrence_id,
@@ -88,12 +65,9 @@ def test_failed_remediation_steps_can_retry_or_regenerate(tmp_path: Path) -> Non
assert failed["actionClaimToken"] is None
assert failed["actionDeliveredAt"] is None
- closed = run_workbench(
+ closed = set_triage(
state_dir,
- "set-finding-triage",
- "--occurrence-id",
occurrence_id,
- "--status",
"closed",
"--close-reason",
"wont_fix",
@@ -101,89 +75,33 @@ def test_failed_remediation_steps_can_retry_or_regenerate(tmp_path: Path) -> Non
"Closing is allowed after the remediation worker has failed.",
)["scan"]
assert closed["findings"][0]["triage"]["status"] == "closed"
- run_workbench(
- state_dir,
- "set-finding-triage",
- "--occurrence-id",
- occurrence_id,
- "--status",
- "open",
- )
+ set_triage(state_dir, occurrence_id, "open")
resend_token = str(uuid.uuid4())
- resent = run_workbench(
- state_dir,
- "claim-finding-remediation-resend",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- resend_token,
- )["scan"]
+ resent = claim_remediation_resend(state_dir, occurrence_id, request_id, resend_token)["scan"]
assert resent["findings"][0]["remediationState"]["actionClaimToken"] == resend_token
- delivered = run_workbench(
- state_dir,
- "mark-finding-remediation-delivered",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- resend_token,
- )["scan"]
+ delivered = mark_remediation_delivered(state_dir, occurrence_id, request_id, resend_token)[
+ "scan"
+ ]
assert delivered["findings"][0]["remediationState"]["actionDeliveredAt"]
- canceled = run_workbench(
- state_dir,
- "cancel-finding-remediation-request",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- resend_token,
- )["scan"]["findings"][0]["remediationState"]
+ canceled = cancel_remediation_request(state_dir, occurrence_id, request_id, resend_token)[
+ "scan"
+ ]["findings"][0]["remediationState"]
assert canceled["state"] == "failed"
assert canceled["pendingAction"] == "generate"
assert canceled["actionClaimToken"] is None
- replayed = run_workbench(
- state_dir,
- "cancel-finding-remediation-request",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- resend_token,
- )["scan"]["findings"][0]["remediationState"]
+ replayed = cancel_remediation_request(state_dir, occurrence_id, request_id, resend_token)[
+ "scan"
+ ]["findings"][0]["remediationState"]
assert replayed == canceled
resend_token = str(uuid.uuid4())
- run_workbench(
- state_dir,
- "claim-finding-remediation-resend",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- resend_token,
- )
+ claim_remediation_resend(state_dir, occurrence_id, request_id, resend_token)
failed_request_id = request_id
request_id = str(uuid.uuid4())
action_token = str(uuid.uuid4())
- blocked = run_workbench(
- state_dir,
- "request-finding-remediation",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- action_token,
- check=False,
- )
+ blocked = request_remediation(state_dir, occurrence_id, request_id, action_token, check=False)
assert blocked["returncode"] != 0
with sqlite3.connect(state_dir / "workbench.sqlite3") as connection:
connection.execute(
@@ -194,16 +112,9 @@ def test_failed_remediation_steps_can_retry_or_regenerate(tmp_path: Path) -> Non
""",
("2000-01-01T00:00:00Z", failed_request_id),
)
- regenerated = run_workbench(
- state_dir,
- "request-finding-remediation",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- action_token,
- )["scan"]["findings"][0]["remediationState"]
+ regenerated = request_remediation(state_dir, occurrence_id, request_id, action_token)["scan"][
+ "findings"
+ ][0]["remediationState"]
assert regenerated["state"] == "requested"
patch_path = scan_dir / "remediation.patch"
@@ -230,20 +141,7 @@ def test_failed_remediation_steps_can_retry_or_regenerate(tmp_path: Path) -> Non
)
apply_token = str(uuid.uuid4())
- run_workbench(
- state_dir,
- "request-finding-remediation-action",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--expected-version",
- "2",
- "--action",
- "apply",
- "--action-token",
- apply_token,
- )
+ request_remediation_action(state_dir, occurrence_id, request_id, "2", "apply", apply_token)
subprocess.run(["git", "apply", "--no-index", str(patch_path)], cwd=target, check=True)
failed = update_remediation(
state_dir,
@@ -257,16 +155,7 @@ def test_failed_remediation_steps_can_retry_or_regenerate(tmp_path: Path) -> Non
)
assert failed["pendingAction"] == "apply"
apply_retry_token = str(uuid.uuid4())
- run_workbench(
- state_dir,
- "claim-finding-remediation-resend",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- apply_retry_token,
- )
+ claim_remediation_resend(state_dir, occurrence_id, request_id, apply_retry_token)
applied = update_remediation(
state_dir,
occurrence_id,
@@ -281,30 +170,8 @@ def test_failed_remediation_steps_can_retry_or_regenerate(tmp_path: Path) -> Non
assert applied["summary"] is None
verify_token = str(uuid.uuid4())
- run_workbench(
- state_dir,
- "request-finding-remediation-action",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--expected-version",
- "5",
- "--action",
- "verify",
- "--action-token",
- verify_token,
- )
- run_workbench(
- state_dir,
- "mark-finding-remediation-delivered",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- verify_token,
- )
+ request_remediation_action(state_dir, occurrence_id, request_id, "5", "verify", verify_token)
+ mark_remediation_delivered(state_dir, occurrence_id, request_id, verify_token)
update_remediation(
state_dir,
occurrence_id,
@@ -329,25 +196,9 @@ def test_failed_remediation_steps_can_retry_or_regenerate(tmp_path: Path) -> Non
assert failed["actionClaimToken"] is None
assert failed["actionDeliveredAt"] is None
verify_retry_token = str(uuid.uuid4())
- run_workbench(
- state_dir,
- "claim-finding-remediation-resend",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- verify_retry_token,
- )
- delivered = run_workbench(
- state_dir,
- "mark-finding-remediation-delivered",
- "--occurrence-id",
- occurrence_id,
- "--request-id",
- request_id,
- "--action-token",
- verify_retry_token,
+ claim_remediation_resend(state_dir, occurrence_id, request_id, verify_retry_token)
+ delivered = mark_remediation_delivered(
+ state_dir, occurrence_id, request_id, verify_retry_token
)["scan"]["findings"][0]["remediationState"]
assert delivered["actionDeliveredAt"]
verified = update_remediation(
diff --git a/plugins/codex-security/tests/workbench_test_support.py b/plugins/codex-security/tests/workbench_test_support.py
index bbbb46bd7..8584839b4 100644
--- a/plugins/codex-security/tests/workbench_test_support.py
+++ b/plugins/codex-security/tests/workbench_test_support.py
@@ -263,6 +263,82 @@ def workspace_command(
return run_workbench(state_dir, command, "--workspace-id", workspace_id, *extra, **options)
+def request_remediation(
+ state_dir: Path,
+ occurrence_id: str,
+ request_id: str,
+ action_token: str,
+ *,
+ check: bool = True,
+) -> dict[str, object]:
+ return run_workbench(
+ state_dir,
+ "request-finding-remediation",
+ "--occurrence-id",
+ occurrence_id,
+ "--request-id",
+ request_id,
+ "--action-token",
+ action_token,
+ check=check,
+ )
+
+
+def set_remediation(
+ state_dir: Path,
+ occurrence_id: str,
+ request_id: str,
+ action_token: str,
+ expected_version: str,
+ state: str,
+ *extra: str,
+ check: bool = True,
+) -> dict[str, object]:
+ return run_workbench(
+ state_dir,
+ "set-finding-remediation",
+ "--occurrence-id",
+ occurrence_id,
+ "--request-id",
+ request_id,
+ "--action-token",
+ action_token,
+ "--expected-version",
+ expected_version,
+ "--state",
+ state,
+ *extra,
+ check=check,
+ )
+
+
+def request_remediation_action(
+ state_dir: Path,
+ occurrence_id: str,
+ request_id: str,
+ expected_version: str,
+ action: str,
+ action_token: str,
+ *,
+ check: bool = True,
+) -> dict[str, object]:
+ return run_workbench(
+ state_dir,
+ "request-finding-remediation-action",
+ "--occurrence-id",
+ occurrence_id,
+ "--request-id",
+ request_id,
+ "--expected-version",
+ expected_version,
+ "--action",
+ action,
+ "--action-token",
+ action_token,
+ check=check,
+ )
+
+
def save_workspace(
state_dir: Path,
workspace_id: str,
@@ -312,6 +388,63 @@ def update_progress(
)
+def claim_remediation_resend(
+ state_dir: Path,
+ occurrence_id: str,
+ request_id: str,
+ action_token: str,
+ *,
+ check: bool = True,
+) -> dict[str, object]:
+ return run_workbench(
+ state_dir,
+ "claim-finding-remediation-resend",
+ "--occurrence-id",
+ occurrence_id,
+ "--request-id",
+ request_id,
+ "--action-token",
+ action_token,
+ check=check,
+ )
+
+
+def cancel_remediation_request(
+ state_dir: Path,
+ occurrence_id: str,
+ request_id: str,
+ action_token: str,
+) -> dict[str, object]:
+ return run_workbench(
+ state_dir,
+ "cancel-finding-remediation-request",
+ "--occurrence-id",
+ occurrence_id,
+ "--request-id",
+ request_id,
+ "--action-token",
+ action_token,
+ )
+
+
+def mark_remediation_delivered(
+ state_dir: Path,
+ occurrence_id: str,
+ request_id: str,
+ action_token: str,
+) -> dict[str, object]:
+ return run_workbench(
+ state_dir,
+ "mark-finding-remediation-delivered",
+ "--occurrence-id",
+ occurrence_id,
+ "--request-id",
+ request_id,
+ "--action-token",
+ action_token,
+ )
+
+
def fail_deep_scan(state_dir, codex_home, scan_id, *, message="Worker stopped.", deep_status=None):
return scan_command(
state_dir,