Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

test: consolidate progress runners and publication fixtures (#2327)

Michael D'Angelo6658112e45c1
diff --git a/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.ts b/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.ts
index 4e13e9022..42db23250 100644
--- a/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.ts
+++ b/plugins/codex-security/mcp-app/tests/deep_scan_coverage_fixture.ts
@@ -20,8 +20,7 @@ type FixtureRuntime = typeof import("../src/deep-scan/artifacts.js") &
interface CoverageFixtureOptions {
resume?: boolean;
continueAfterResume?: boolean;
- stopAfterDraft?: boolean;
- stopBeforeDraft?: boolean;
+ outcome?: "completion" | "recovery" | "no parent" | "frozen recovery";
receiptRetry?: boolean;
firstCheckpointReceipt?: "shared" | "worker";
sameAttemptCloseout?: boolean;
@@ -40,7 +39,6 @@ interface CoverageFixtureOptions {
otherWorkerCandidateIds?: string[];
retryFindings?: JsonObject[][];
questionRows?: (string | JsonObject)[];
- interruptPublication?: boolean;
}
interface ProjectedRecord {
id?: string;
@@ -129,8 +127,7 @@ export async function publishCoverageFixture(
{
resume = false,
continueAfterResume = false,
- stopAfterDraft = false,
- stopBeforeDraft = false,
+ outcome = "completion",
receiptRetry = false,
firstCheckpointReceipt,
sameAttemptCloseout = false,
@@ -149,7 +146,6 @@ export async function publishCoverageFixture(
otherWorkerCandidateIds = ["candidate-1"],
retryFindings,
questionRows,
- interruptPublication = false,
}: CoverageFixtureOptions = {},
) {
const targetPath = path.join(root, "target");
@@ -174,7 +170,7 @@ export async function publishCoverageFixture(
const runWorkbench = async (args: string[]) => {
const workbenchPath = path.join(pluginRoot, "scripts", "workbench_db.py");
const command = [workbenchPath, ...args];
- if (interruptPublication && args[0] === "fail-scan") {
+ if (outcome === "frozen recovery" && args[0] === "fail-scan") {
command.unshift(
"-c",
`
@@ -823,7 +819,7 @@ runpy.run_path(sys.argv[0], run_name="__main__")
threadId,
retryDelaysMs: extraReceiptRetry ? [1, 1] : [1],
onComplete: async (draft, signal, publication) => {
- if (stopBeforeDraft)
+ if (outcome === "no parent")
throw new Error("Synthetic stop before parent draft.");
await recordCodexSecurityScanDraftViaWorkbench(
context,
@@ -845,7 +841,7 @@ runpy.run_path(sys.argv[0], run_name="__main__")
}
assert.equal(
terminal.status,
- stopBeforeDraft ? "failed" : "succeeded",
+ outcome === "no parent" ? "failed" : "succeeded",
terminal.error ?? "Unexpected Deep Scan terminal status.",
);
if (!retryFindings) {
@@ -877,7 +873,7 @@ runpy.run_path(sys.argv[0], run_name="__main__")
"v1 reducers remain readable by earlier binaries",
);
}
- if (stopBeforeDraft) {
+ if (outcome === "no parent") {
const recovered = await runWorkbench([
"recover-scan-results",
"--scan-id",
@@ -888,7 +884,7 @@ runpy.run_path(sys.argv[0], run_name="__main__")
const published = await readFile(coveragePath, "utf8");
await runWorkbench(["recover-scan-results", "--scan-id", run.scanId]);
assert.equal(await readFile(coveragePath, "utf8"), published);
- } else if (stopAfterDraft) {
+ } else if (outcome !== "completion") {
const stopped = await runWorkbench([
"fail-scan",
"--scan-id",
@@ -896,7 +892,10 @@ runpy.run_path(sys.argv[0], run_name="__main__")
"--message",
"Synthetic stop after parent draft.",
]);
- assert.equal(stopped.scan.resultsRecoveryNeeded, interruptPublication);
+ assert.equal(
+ stopped.scan.resultsRecoveryNeeded,
+ outcome === "frozen recovery",
+ );
const recovered = await runWorkbench([
"recover-scan-results",
"--scan-id",
@@ -922,7 +921,7 @@ runpy.run_path(sys.argv[0], run_name="__main__")
const coverage = await readFixtureCoverage(run.scanDir);
assert.deepEqual(
coverage.deferred.map((row) => row.id),
- stopAfterDraft || stopBeforeDraft ? ["scan-stopped"] : [],
+ outcome === "completion" ? [] : ["scan-stopped"],
);
if (sameAttemptCloseout) {
assert.deepEqual(
diff --git a/plugins/codex-security/mcp-app/tests/support/source-references.ts b/plugins/codex-security/mcp-app/tests/support/source-references.ts
deleted file mode 100644
index 792164ad8..000000000
--- a/plugins/codex-security/mcp-app/tests/support/source-references.ts
+++ /dev/null
@@ -1,9 +0,0 @@
-export const sourceReferences =
- (worker: { id: string }) =>
- <T extends { provenance: object }>(finding: T, index: number) => ({
- ...finding,
- provenance: {
- ...finding.provenance,
- sourceFindingIds: [`${worker.id}:${index}`],
- },
- });
diff --git a/plugins/codex-security/mcp-app/tests/test_coverage_provenance.ts b/plugins/codex-security/mcp-app/tests/test_coverage_provenance.ts
index 634867a6e..6b6a79120 100644
--- a/plugins/codex-security/mcp-app/tests/test_coverage_provenance.ts
+++ b/plugins/codex-security/mcp-app/tests/test_coverage_provenance.ts
@@ -10,27 +10,25 @@ import {
} from "./deep_scan_coverage_fixture.ts";
for (const resume of [false, true]) {
- for (const outcome of ["recovery", "no parent"]) {
+ for (const outcome of ["recovery", "no parent"] as const) {
test(`worker-local same-attempt closeout survives ${resume ? "resumed" : "live"} ${outcome}`, () =>
publishReceiptFixture({
resume,
firstCheckpointReceipt: "worker",
sameAttemptCloseout: true,
- stopAfterDraft: outcome === "recovery",
- stopBeforeDraft: outcome === "no parent",
+ outcome,
}));
}
}
for (const resume of [false, true]) {
- for (const outcome of ["completion", "recovery", "no parent"]) {
+ for (const outcome of ["completion", "recovery", "no parent"] as const) {
test(`coverage provenance survives ${resume ? "reconstructed" : "live"} reduction and ${outcome}`, async () => {
const root = await mkdtemp(path.join(tmpdir(), "coverage-provenance-"));
try {
const { scanDir } = await publishCoverageFixture(root, "partial", {
resume,
- stopAfterDraft: outcome === "recovery",
- stopBeforeDraft: outcome === "no parent",
+ outcome,
});
const coverage = await readFixtureCoverage(scanDir);
assert.equal("completeness" in coverage, false);
@@ -119,13 +117,12 @@ for (const resume of [false, true]) {
});
}
-for (const outcome of ["completion", "recovery", "frozen recovery"]) {
+for (const outcome of ["completion", "recovery", "frozen recovery"] as const) {
test(`open questions are retained once through ${outcome}`, async () => {
const root = await mkdtemp(path.join(tmpdir(), "question-coverage-"));
try {
const { scanDir } = await publishCoverageFixture(root, "complete", {
- stopAfterDraft: outcome !== "completion",
- interruptPublication: outcome === "frozen recovery",
+ outcome,
questionRows: [
" Which deployment controls apply? ",
{
diff --git a/plugins/codex-security/mcp-app/tests/test_retry_coverage_dispositions.ts b/plugins/codex-security/mcp-app/tests/test_retry_coverage_dispositions.ts
index 832565d3d..4e1095134 100644
--- a/plugins/codex-security/mcp-app/tests/test_retry_coverage_dispositions.ts
+++ b/plugins/codex-security/mcp-app/tests/test_retry_coverage_dispositions.ts
@@ -74,7 +74,7 @@ for (const resume of [false, true]) {
provenance: { source: "local_plugin", candidateId },
}));
const { scanDir } = await publishCoverageFixture(root, "partial", {
- stopBeforeDraft: true,
+ outcome: "no parent",
resume,
otherWorkerCandidateIds: cases.map((row) => row.candidateId),
retryCoverage: resolveOnRetry
diff --git a/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts_namespaces.ts b/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts_namespaces.ts
index 3fb6d2956..f927585e4 100644
--- a/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts_namespaces.ts
+++ b/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts_namespaces.ts
@@ -2,14 +2,13 @@ import test from "node:test";
import { publishReceiptFixture } from "./deep_scan_coverage_fixture.ts";
for (const resume of [false, true]) {
- for (const outcome of ["completion", "recovery", "no parent"]) {
+ for (const outcome of ["completion", "recovery", "no parent"] as const) {
for (const retryPending of [false, true]) {
test(`archived receipt and pending records survive ${resume ? "reconstruction" : "live retry"} and ${outcome} with ${retryPending ? "an updated gap" : "a clean retry"}`, () =>
publishReceiptFixture({
receiptRetry: true,
retryPending,
- stopAfterDraft: outcome === "recovery",
- stopBeforeDraft: outcome === "no parent",
+ outcome,
resume,
}));
}
@@ -23,8 +22,7 @@ for (const resume of [false, true]) {
receiptRetry: true,
receiptSpelling,
activeReceiptSpelling,
- stopAfterDraft: outcome === "recovery",
- stopBeforeDraft: outcome === "no parent",
+ outcome,
resume,
}));
}
diff --git a/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts_ownership.ts b/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts_ownership.ts
index 10d6d961f..855bb9f96 100644
--- a/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts_ownership.ts
+++ b/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts_ownership.ts
@@ -100,7 +100,7 @@ for (const resume of [false, true]) {
}
for (const resume of [false, true]) {
- for (const outcome of ["completion", "recovery", "no parent"]) {
+ for (const outcome of ["completion", "recovery", "no parent"] as const) {
for (const [name, firstCheckpointReceipt, receiptCollision] of [
["shared same-name collision", "shared", true],
["shared distinct-name control", "shared", false],
@@ -111,8 +111,7 @@ for (const resume of [false, true]) {
firstCheckpointReceipt,
receiptCollision,
resume,
- stopAfterDraft: outcome === "recovery",
- stopBeforeDraft: outcome === "no parent",
+ outcome,
}));
}
}
diff --git a/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts_shared.ts b/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts_shared.ts
index 405e20890..d4a225caa 100644
--- a/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts_shared.ts
+++ b/plugins/codex-security/mcp-app/tests/test_retry_coverage_receipts_shared.ts
@@ -2,13 +2,12 @@ import test from "node:test";
import { publishReceiptFixture } from "./deep_scan_coverage_fixture.ts";
for (const resume of [false, true]) {
- for (const outcome of ["completion", "recovery", "no parent"]) {
+ for (const outcome of ["completion", "recovery", "no parent"] as const) {
test(`shared scan receipts survive ${resume ? "reconstruction" : "live retry"} and ${outcome}`, () =>
publishReceiptFixture({
receiptRetry: true,
sharedReceipt: true,
- stopAfterDraft: outcome === "recovery",
- stopBeforeDraft: outcome === "no parent",
+ outcome,
resume,
}));
@@ -16,8 +15,7 @@ for (const resume of [false, true]) {
publishReceiptFixture({
receiptRetry: true,
receiptSpelling: "shared scan",
- stopAfterDraft: outcome === "recovery",
- stopBeforeDraft: outcome === "no parent",
+ outcome,
resume,
}));
@@ -27,8 +25,7 @@ for (const resume of [false, true]) {
receiptRetry: true,
sharedReceipt,
emptyReceipt: true,
- stopAfterDraft: outcome === "recovery",
- stopBeforeDraft: outcome === "no parent",
+ outcome,
resume,
}));
}
@@ -37,8 +34,7 @@ for (const resume of [false, true]) {
receiptRetry: true,
receiptSpelling: "shared scan",
emptyReceipt: true,
- stopAfterDraft: outcome === "recovery",
- stopBeforeDraft: outcome === "no parent",
+ outcome,
resume,
}));
@@ -46,8 +42,7 @@ for (const resume of [false, true]) {
publishReceiptFixture({
receiptRetry: true,
receiptCollision: true,
- stopAfterDraft: outcome === "recovery",
- stopBeforeDraft: outcome === "no parent",
+ outcome,
resume,
}));
@@ -56,8 +51,7 @@ for (const resume of [false, true]) {
receiptRetry: true,
receiptSpelling: "shared scan",
receiptCollision: true,
- stopAfterDraft: outcome === "recovery",
- stopBeforeDraft: outcome === "no parent",
+ outcome,
resume,
}));
}
diff --git a/plugins/codex-security/tests/conftest.py b/plugins/codex-security/tests/conftest.py
index f1d9d1975..c7b038c7d 100644
--- a/plugins/codex-security/tests/conftest.py
+++ b/plugins/codex-security/tests/conftest.py
@@ -7,6 +7,14 @@ from pathlib import Path
import pytest
+pytest.register_assert_rewrite("publication_fixtures")
+from publication_fixtures import ( # noqa: F401 -- pytest discovers these shared fixtures here.
+ deadline_scan,
+ finish_after_crashes,
+ publication_scan,
+ saturated_scan,
+)
+
@pytest.fixture(scope="session")
def workbench_api():
diff --git a/plugins/codex-security/tests/publication_fixtures.py b/plugins/codex-security/tests/publication_fixtures.py
new file mode 100644
index 000000000..9a768e71b
--- /dev/null
+++ b/plugins/codex-security/tests/publication_fixtures.py
@@ -0,0 +1,327 @@
+from __future__ import annotations
+
+import copy
+import json
+import sqlite3
+import uuid
+from argparse import Namespace
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from deep_deadline_crash import interrupt_finish_before_commit
+from workbench_test_support import read_review_state, write_completed_contract
+
+
+class PublicationScan(SimpleNamespace):
+ def add_worker(self, connection=None, *, status="succeeded") -> Path:
+ worker_id = str(uuid.uuid4())
+ output = self.scan_dir / "workers" / worker_id
+ output.mkdir(parents=True)
+ result = output / "result.json"
+ connection = self.connection if connection is None else connection
+ with connection:
+ connection.execute(
+ "INSERT INTO deep_scan_workers (id, scan_id, kind, status, merge_state, "
+ "prompt_path, artifact_dir, result_manifest_path, attempt, created_at, "
+ "updated_at, completed_at) VALUES (?, ?, 'discovery', ?, ?, ?, ?, ?, 1, ?, ?, ?)",
+ (
+ worker_id,
+ self.scan_id,
+ status,
+ "merged" if status == "succeeded" else "none",
+ str(output / "prompt.md"),
+ str(output),
+ str(result),
+ self.timestamp,
+ self.timestamp,
+ self.timestamp,
+ ),
+ )
+ return result
+
+ def canceled_discovery_output(self, connection=None, *, attempt=1):
+ connection = self.connection if connection is None else connection
+ result = self.add_worker(connection, status="canceled")
+ worker_id = result.parent.name
+ output = self.scan_dir / "artifacts" / "deep_discovery" / "workers" / worker_id / "output"
+ output.parent.mkdir(parents=True)
+ result.parent.rename(output)
+ with connection:
+ connection.execute(
+ "UPDATE deep_scan_workers SET artifact_dir = ?, prompt_path = ?, "
+ "result_manifest_path = NULL, attempt = ? WHERE id = ?",
+ (str(output), str(output.parent / "prompt.md"), attempt, worker_id),
+ )
+ return worker_id, output
+
+ def finish(self, connection=None, *, terminal_reason="saturated"):
+ return self.api["deep_scan"].finish_deep_scan(
+ self.connection if connection is None else connection,
+ Namespace(
+ scan_id=self.scan_id,
+ terminal_reason=terminal_reason,
+ manifest_path=str(self.scan_dir / "scan-manifest.json"),
+ staged_manifest_path=None,
+ omitted_worker_id=[],
+ ),
+ )["deepScan"]
+
+ def complete(self, connection=None, *, prepare_only=False):
+ return self.api["complete_scan"](
+ self.connection if connection is None else connection,
+ Namespace(scan_id=self.scan_id, claim_token=None, cost_json=None),
+ prepare_only=prepare_only,
+ )["scan"]
+
+ def fail(self, message="Stopped."):
+ return self.api["saved_results"].fail_scan(
+ self.api["_WORKBENCH_DB_CONTEXT"],
+ self.connection,
+ Namespace(scan_id=self.scan_id, claim_token=None, cost_json=None, message=message),
+ )["scan"]
+
+ def preserve(self, connection=None):
+ return self.api["saved_results"].preserve_scan_results(
+ self.api["_WORKBENCH_DB_CONTEXT"],
+ self.connection if connection is None else connection,
+ Namespace(
+ scan_id=self.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
+ ),
+ )["scan"]
+
+ def recover(self):
+ return self.api["saved_results"].recover_scan_results(
+ self.api["_WORKBENCH_DB_CONTEXT"], self.connection, Namespace(scan_id=self.scan_id)
+ )["scan"]
+
+
+def publication_scan(workbench_api, workbench_db, tmp_path, monkeypatch):
+ monkeypatch.setenv("CODEX_SECURITY_STATE_DIR", str(tmp_path / "state"))
+ monkeypatch.setenv("CODEX_HOME", str(tmp_path / "codex-home"))
+ deep = workbench_api["deep_scan"]
+ monkeypatch.setattr(
+ deep,
+ "_dependencies",
+ SimpleNamespace(
+ **workbench_api,
+ preserve_stopped_results=workbench_api["preserve_stopped_results_after_transition"],
+ ),
+ )
+
+ def create(*, mode="deep", scope=".", paths=None):
+ target = tmp_path / "target"
+ target.mkdir()
+ (target / "subdir").mkdir()
+ (target / "subdir" / "extract.py").write_text("# Synthetic scan target\n")
+ if paths is not None:
+ for directory in [*paths, "unrelated"]:
+ if not (target / directory).is_file():
+ (target / directory).mkdir(exist_ok=True)
+ (target / directory / "extract.py").write_text("# Synthetic scoped target\n")
+ include_paths = paths if paths is not None else [scope]
+ scan_dir = tmp_path / "scan"
+ scan_dir.mkdir(mode=0o700)
+ registered = workbench_api["register_cli_scan"](
+ workbench_db,
+ Namespace(
+ repository=str(target),
+ scan_dir=str(scan_dir),
+ recipe_json=json.dumps(
+ {
+ "config": {},
+ "mode": mode,
+ "repository": str(target),
+ "target": {
+ "kind": "paths" if paths is not None or scope != "." else "repository",
+ "paths": include_paths if paths is not None or scope != "." else [],
+ },
+ }
+ ),
+ registration_json_stdin=False,
+ recipe_json_stdin=False,
+ parent_scan_id=None,
+ archive_existing=False,
+ archived_scan_dir=None,
+ ),
+ )
+ scan_id = registered["scanId"]
+ timestamp = workbench_db.execute(
+ "SELECT started_at FROM scans WHERE id = ?", (scan_id,)
+ ).fetchone()[0]
+ if mode == "deep":
+ # Start with a finished Deep result so these tests only need to save it.
+ with workbench_db:
+ workbench_db.execute(
+ "INSERT INTO deep_scan_runs (scan_id, schema_version, workflow_version, "
+ "status, phase, workers, subagents, stop_after_no_new, max_discovery_runs, "
+ "manifest_path, terminal_reason, created_at, updated_at, completed_at) "
+ "VALUES (?, 1, 'deep-security-scan/v1', 'succeeded', 'terminal', 1, 0, 1, 1, "
+ "?, 'saturated', ?, ?, ?)",
+ (
+ scan_id,
+ str(scan_dir / "scan-manifest.json"),
+ timestamp,
+ timestamp,
+ timestamp,
+ ),
+ )
+ coverage_mode = (
+ "scoped_path"
+ if paths is not None or scope != "."
+ else "deep_repository"
+ if mode == "deep"
+ else "repository"
+ )
+ write_completed_contract(
+ scan_dir,
+ scan_id,
+ target,
+ include_paths=include_paths,
+ relative_path="subdir/extract.py",
+ coverage_mode=coverage_mode,
+ inventory_strategy="scoped_path" if scope != "." else "repository",
+ )
+ manifest = json.loads((scan_dir / "scan-manifest.json").read_text())
+ findings = json.loads((scan_dir / "findings.json").read_text())["findings"]
+ coverage = json.loads((scan_dir / "coverage.json").read_text())
+ if mode == "deep":
+ coverage.update(surfaces=[], explicitExclusions=[], deferred=[])
+ else:
+ coverage["openQuestions"] = [{"question": "Which deployment controls apply?"}]
+ for field in ("documentType", "schemaVersion", "scanId"):
+ coverage.pop(field)
+ # Use the same draft format as the writer, before finalization adds metadata.
+ manifest = {"scan": {key: manifest["scan"][key] for key in ("target", "scope")}}
+ findings[0]["severity"]["changeConditions"] = "Reassess if the upload route is removed."
+ findings[0]["provenance"]["sourceFindings"] = [
+ {"id": "review-1:candidate-1", "finding": {"summary": "Retained original wording."}}
+ ]
+ for name, value in (
+ ("scan-manifest.json", manifest),
+ ("findings.json", {"findings": findings}),
+ ("coverage.json", coverage),
+ ):
+ (scan_dir / name).write_text(json.dumps(value))
+ return PublicationScan(
+ api=workbench_api,
+ connection=workbench_db,
+ scan_id=scan_id,
+ scan_dir=scan_dir,
+ timestamp=timestamp,
+ findings=findings,
+ coverage=coverage,
+ )
+
+ return create
+
+
+def saturated_scan(publication_scan):
+ scan = publication_scan()
+ completed_result = scan.add_worker()
+ completed_result.write_text(
+ json.dumps(
+ {
+ "scanId": scan.scan_id,
+ "complete": True,
+ "findings": scan.findings,
+ "coverage": scan.coverage,
+ }
+ )
+ )
+ reducer_result = scan.add_worker()
+ reducer_result.write_bytes(completed_result.read_bytes())
+ with scan.connection:
+ scan.connection.execute(
+ "UPDATE deep_scan_workers SET completion_sequence = 1 WHERE id = ?",
+ (completed_result.parent.name,),
+ )
+ scan.connection.execute(
+ "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' WHERE id = ?",
+ (reducer_result.parent.name,),
+ )
+ scan.connection.execute(
+ "INSERT INTO deep_scan_dedup_inputs "
+ "(scan_id, dedup_worker_id, discovery_worker_id, input_order) VALUES (?, ?, ?, 0)",
+ (scan.scan_id, reducer_result.parent.name, completed_result.parent.name),
+ )
+ scan.connection.execute(
+ "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', "
+ "consecutive_no_new = stop_after_no_new, completion_sequence = 1, "
+ "max_discovery_runs = 3, discovery_runs_dispatched = 1, "
+ "manifest_path = NULL, terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?",
+ (scan.scan_id,),
+ )
+ scan.completed_worker_id = completed_result.parent.name
+ scan.reducer_id = reducer_result.parent.name
+ return scan
+
+
+def deadline_scan(saturated_scan):
+ with saturated_scan.connection:
+ saturated_scan.connection.execute(
+ "UPDATE deep_scan_runs SET created_at = '2000-01-01T00:00:00+00:00' WHERE scan_id = ?",
+ (saturated_scan.scan_id,),
+ )
+ return saturated_scan
+
+
+def finish_after_crashes(deadline_scan, tmp_path):
+ def run():
+ scan = deadline_scan
+ database = tmp_path / "crash-recovery.db"
+ with sqlite3.connect(database) as connection:
+ scan.connection.backup(connection)
+
+ publications = []
+ for hash_seed in (1, 17, 83):
+ interrupt_finish_before_commit(database, scan, hash_seed)
+ publications.append(read_review_state(scan.connection, scan.scan_dir))
+ with sqlite3.connect(database) as connection:
+ status = connection.execute(
+ "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,)
+ ).fetchone()[0]
+ assert status == "running"
+ with sqlite3.connect(database) as connection:
+ connection.row_factory = sqlite3.Row
+ scan.finish(connection, terminal_reason="capped")
+ completed = scan.complete(connection)
+ coverage = read_review_state(scan.connection, scan.scan_dir)
+ assert completed["progress"]["status"] == "complete"
+ assert all(publication == coverage for publication in publications)
+ return completed, coverage
+
+ return run
+
+
+def assert_published_aggregate(scan):
+ findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
+ for finding in findings:
+ for field in ("findingId", "occurrenceId", "fingerprints"):
+ value = finding.pop(field)
+ assert value
+ assert findings == scan.findings
+ manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())
+ if "coverageRef" in manifest["scan"]:
+ coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
+ for field in ("documentType", "schemaVersion", "scanId"):
+ coverage.pop(field)
+ assert coverage == scan.coverage
+ else:
+ assert manifest["scan"]["review"] == {
+ "excludedPaths": [item["pattern"] for item in scan.coverage["explicitExclusions"]],
+ }
+ assert not (scan.scan_dir / "coverage.json").exists()
+ assert (scan.scan_dir / "report.md").is_file()
+
+
+def worker_finding(scan, anchor):
+ finding = copy.deepcopy(scan.findings[0])
+ finding["identity"] = {"anchor": anchor}
+ finding["title"] = anchor
+ finding["provenance"] = {"source": "local_plugin"}
+ return finding
diff --git a/plugins/codex-security/tests/test_checkpoint_publication_authority.py b/plugins/codex-security/tests/test_checkpoint_publication_authority.py
index e7ba05168..63ea81688 100644
--- a/plugins/codex-security/tests/test_checkpoint_publication_authority.py
+++ b/plugins/codex-security/tests/test_checkpoint_publication_authority.py
@@ -12,7 +12,6 @@ from argparse import Namespace
from pathlib import Path
import pytest
-from test_deep_scan_successful_publication import publication_scan as publication_scan
from test_workbench_standard_deep_results import deep_scan_fixture, worker_paths
from workbench_test_support import (
interrupt_publication,
diff --git a/plugins/codex-security/tests/test_checkpoint_recovery_fallbacks.py b/plugins/codex-security/tests/test_checkpoint_recovery_fallbacks.py
index 8019f39fc..b6dcb5aba 100644
--- a/plugins/codex-security/tests/test_checkpoint_recovery_fallbacks.py
+++ b/plugins/codex-security/tests/test_checkpoint_recovery_fallbacks.py
@@ -7,7 +7,6 @@ from argparse import Namespace
from pathlib import Path
import pytest
-from test_deep_scan_successful_publication import publication_scan as publication_scan
from workbench_test_support import (
interrupt_publication,
read_review_state,
diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py b/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py
index 831ddd1a9..3e8da2ed8 100644
--- a/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py
+++ b/plugins/codex-security/tests/test_deep_scan_deadline_contracts.py
@@ -5,11 +5,7 @@ import json
import os
import pytest
-from test_deep_scan_deadline_recovery import worker_finding
-from test_deep_scan_stop_conditions import canceled_discovery_output, finish
-from test_deep_scan_stop_conditions import deadline_scan as deadline_scan
-from test_deep_scan_stop_conditions import publication_scan as publication_scan
-from test_deep_scan_stop_conditions import saturated_scan as saturated_scan
+from publication_fixtures import worker_finding
from workbench_test_support import (
read_review_state,
saved_draft,
@@ -35,7 +31,7 @@ def test_deadline_requires_boolean_completion_before_superseding_worker_history(
workbench_api, workbench_db, deadline_scan, source, marker
):
scan = deadline_scan
- _, output = canceled_discovery_output(workbench_db, scan, attempt=2)
+ _, output = scan.canceled_discovery_output(attempt=2)
if source == "archived-checkpoint":
output = output.parent / "attempts" / "attempt-1"
earlier = worker_finding(scan, "earlier-worker-finding")
@@ -77,7 +73,7 @@ def test_deadline_requires_boolean_completion_before_superseding_worker_history(
os.utime(result, ns=(200, 200))
original = {path: path.read_bytes() for path in (checkpoint, result)}
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
terminal = marker is True or marker == "omitted"
@@ -118,7 +114,7 @@ def test_deadline_validates_each_recovered_finding_against_the_completed_contrac
workbench_api, workbench_db, deadline_scan, source, invalid
):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2)
+ worker_id, output = scan.canceled_discovery_output(attempt=2)
if source == "archived-result":
output = output.parent / "attempts" / "attempt-1"
earlier = worker_finding(scan, "earlier-worker-finding")
@@ -180,7 +176,7 @@ def test_deadline_validates_each_recovered_finding_against_the_completed_contrac
select_checkpoint(output, selected, mtime_ns=300)
original[selected] = selected.read_bytes()
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
@@ -215,17 +211,15 @@ def test_deadline_validates_each_recovered_finding_against_the_completed_contrac
{"format": "markdown", "content": "# Reviewed boundaries"},
],
)
-def test_deadline_recovers_optional_worker_model_without_losing_findings(
- workbench_api, workbench_db, deadline_scan, model
-):
+def test_deadline_recovers_optional_worker_model_without_losing_findings(deadline_scan, model):
scan = deadline_scan
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
finding = worker_finding(scan, "worker-model-finding")
draft = {**saved_draft(scan.scan_id, findings=[finding]), "threatModel": model}
checkpoint = write_checkpoint(output / "checkpoints", draft)
original = checkpoint.read_bytes()
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 2
@@ -242,9 +236,7 @@ def test_deadline_recovers_optional_worker_model_without_losing_findings(
assert checkpoint.read_bytes() == original
-def test_deadline_still_rejects_malformed_canonical_model(
- workbench_api, workbench_db, deadline_scan
-):
+def test_deadline_still_rejects_malformed_canonical_model(deadline_scan):
from finalize_scan_contract import ContractError
scan = deadline_scan
@@ -252,15 +244,13 @@ def test_deadline_still_rejects_malformed_canonical_model(
manifest = json.loads(manifest_file.read_text())
manifest["scan"]["threatModel"] = {"assets": []}
manifest_file.write_text(json.dumps(manifest))
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
write_checkpoint(output / "checkpoints", saved_draft(scan.scan_id))
with pytest.raises(ContractError, match="threatModel"):
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
-def test_deadline_still_rejects_canonical_findings_outside_the_completed_contract(
- workbench_api, workbench_db, deadline_scan
-):
+def test_deadline_still_rejects_canonical_findings_outside_the_completed_contract(deadline_scan):
from finalize_scan_contract import ContractError
scan = deadline_scan
@@ -269,24 +259,24 @@ def test_deadline_still_rejects_canonical_findings_outside_the_completed_contrac
findings_path = scan.scan_dir / "findings.json"
findings_path.write_text(json.dumps({"findings": scan.findings}))
original = findings_path.read_bytes()
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
write_checkpoint(
output / "checkpoints",
saved_draft(scan.scan_id, findings=[finding]),
)
with pytest.raises(ContractError, match="rationale"):
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
assert findings_path.read_bytes() == original
@pytest.mark.parametrize("label", [" \t ", " Reviewed Résumé 😀 "])
def test_deadline_applies_existing_nonblank_surface_label_contract(
- workbench_api, workbench_db, deadline_scan, label
+ workbench_db, deadline_scan, label
):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(workbench_db, scan)
+ worker_id, output = scan.canceled_discovery_output()
checkpoint = write_checkpoint(
output / "checkpoints",
saved_draft(
@@ -299,7 +289,7 @@ def test_deadline_applies_existing_nonblank_surface_label_contract(
)
original = checkpoint.read_bytes()
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
assert len(json.loads((scan.scan_dir / "findings.json").read_text())["findings"]) == 2
@@ -322,10 +312,10 @@ def test_deadline_applies_existing_nonblank_surface_label_contract(
)
@pytest.mark.parametrize("malformed", [False, True], ids=["valid-unicode", "lone-surrogate"])
def test_deadline_applies_existing_unicode_contract_per_worker_record(
- workbench_api, workbench_db, deadline_scan, field, malformed
+ workbench_db, deadline_scan, field, malformed
):
scan = deadline_scan
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
text = "Review " + (chr(0xD800) if malformed else "Résumé 😀")
finding = worker_finding(scan, "unicode-worker-finding")
draft = saved_draft(scan.scan_id, findings=[finding])
@@ -353,7 +343,7 @@ def test_deadline_applies_existing_unicode_contract_per_worker_record(
select_checkpoint(output, checkpoint)
original = checkpoint.read_bytes()
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
@@ -378,7 +368,7 @@ def test_deadline_recovers_historical_closure_text_without_losing_a_valid_siblin
workbench_api, workbench_db, deadline_scan, reason
):
scan = deadline_scan
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
previous = write_checkpoint(
output / "checkpoints",
saved_draft(
@@ -406,7 +396,7 @@ def test_deadline_recovers_historical_closure_text_without_losing_a_valid_siblin
os.utime(path, ns=(200, 200))
original = {path: path.read_bytes() for path in (previous, checkpoint, result)}
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
invalid = not reason.strip()
@@ -464,10 +454,10 @@ def test_deadline_recovers_historical_closure_text_without_losing_a_valid_siblin
],
)
def test_malformed_terminal_coverage_preserves_only_earlier_coverage(
- workbench_api, workbench_db, deadline_scan, terminal_findings, malformed, selected_head
+ workbench_db, deadline_scan, terminal_findings, malformed, selected_head
):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(workbench_db, scan)
+ worker_id, output = scan.canceled_discovery_output()
finding = worker_finding(scan, "earlier-worker-finding")
rejected = worker_finding(scan, "rejected-sibling")
rejected["provenance"]["candidateId"] = "rejected-sibling"
@@ -529,7 +519,7 @@ def test_malformed_terminal_coverage_preserves_only_earlier_coverage(
os.utime(result, ns=(200, 200))
original = {path: path.read_bytes() for path in (checkpoint, result)}
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
scan.complete()
findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
@@ -559,7 +549,7 @@ def test_malformed_terminal_coverage_preserves_only_earlier_coverage(
@pytest.mark.parametrize("canceled_worker", [False, True])
def test_optional_export_failure_does_not_block_deadline(
- workbench_api, workbench_db, deadline_scan, tmp_path, capsys, canceled_worker
+ deadline_scan, tmp_path, capsys, canceled_worker
):
scan = deadline_scan
sentinel = tmp_path / "untouched.sarif"
@@ -568,13 +558,13 @@ def test_optional_export_failure_does_not_block_deadline(
export.parent.mkdir()
export.symlink_to(sentinel)
if canceled_worker:
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
write_checkpoint(
output / "checkpoints",
saved_draft(scan.scan_id, findings=[worker_finding(scan, "saved-worker-finding")]),
)
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
assert completed["progress"]["status"] == "complete"
diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py
index 4b2306532..8d268684c 100644
--- a/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py
+++ b/plugins/codex-security/tests/test_deep_scan_deadline_recovery.py
@@ -8,19 +8,7 @@ import sqlite3
import pytest
from deep_deadline_crash import interrupt_finish_before_commit
-from test_deep_scan_stop_conditions import (
- canceled_discovery_output,
- finish,
-)
-from test_deep_scan_stop_conditions import (
- deadline_scan as deadline_scan,
-)
-from test_deep_scan_stop_conditions import (
- publication_scan as publication_scan,
-)
-from test_deep_scan_stop_conditions import (
- saturated_scan as saturated_scan,
-)
+from publication_fixtures import worker_finding
from workbench_test_support import (
read_review_state,
saved_draft,
@@ -30,14 +18,6 @@ from workbench_test_support import (
)
-def worker_finding(scan, anchor):
- finding = copy.deepcopy(scan.findings[0])
- finding["identity"] = {"anchor": anchor}
- finding["title"] = anchor
- finding["provenance"] = {"source": "local_plugin"}
- return finding
-
-
@pytest.mark.parametrize(
("task_state", "task_id", "marker_count"),
[
@@ -47,13 +27,13 @@ def worker_finding(scan, anchor):
],
)
def test_deadline_completion_preserves_existing_tasks_and_recovered_findings(
- workbench_api, workbench_db, deadline_scan, task_state, task_id, marker_count
+ workbench_db, deadline_scan, task_state, task_id, marker_count
):
scan = deadline_scan
task = {"id": task_id, "reason": "Existing review outcome."}
scan.coverage[task_state] = [task]
(scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))
- worker_id, output = canceled_discovery_output(workbench_db, scan)
+ worker_id, output = scan.canceled_discovery_output()
recovered = worker_finding(scan, "saved-before-deadline")
checkpoint = write_checkpoint(
output / "checkpoints",
@@ -65,7 +45,7 @@ def test_deadline_completion_preserves_existing_tasks_and_recovered_findings(
)
original_checkpoint = checkpoint.read_bytes()
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
assert completed["progress"]["status"] == "complete"
@@ -104,38 +84,7 @@ def test_deadline_completion_preserves_existing_tasks_and_recovered_findings(
assert checkpoint.read_bytes() == original_checkpoint
-def finish_after_crashes(workbench_api, workbench_db, deadline_scan, tmp_path):
- def run():
- scan = deadline_scan
- database = tmp_path / "crash-recovery.db"
- with sqlite3.connect(database) as connection:
- workbench_db.backup(connection)
-
- publications = []
- for hash_seed in (1, 17, 83):
- interrupt_finish_before_commit(database, scan, hash_seed)
- publications.append(read_review_state(workbench_db, scan.scan_dir))
- with sqlite3.connect(database) as connection:
- status = connection.execute(
- "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,)
- ).fetchone()[0]
- assert status == "running"
- with sqlite3.connect(database) as connection:
- connection.row_factory = sqlite3.Row
- finish(workbench_api, connection, scan, terminal_reason="capped")
- completed = scan.complete(connection)
- coverage = read_review_state(workbench_db, scan.scan_dir)
- assert completed["progress"]["status"] == "complete"
- assert all(publication == coverage for publication in publications)
- return completed, coverage
-
- return run
-
-
-def test_discarded_worker_finding_leaves_its_review_pending(
- workbench_api, workbench_db, deadline_scan
-):
+def test_discarded_worker_finding_leaves_its_review_pending(workbench_db, deadline_scan):
scan = deadline_scan
canonical = {
"id": "accepted",
@@ -145,7 +94,7 @@ def test_discarded_worker_finding_leaves_its_review_pending(
}
scan.coverage["surfaces"] = [canonical]
(scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))
- worker_id, output = canceled_discovery_output(workbench_db, scan)
+ worker_id, output = scan.canceled_discovery_output()
invalid = worker_finding(scan, "incomplete-worker-finding")
invalid.pop("remediation")
checkpoint = write_checkpoint(
@@ -165,7 +114,7 @@ def test_discarded_worker_finding_leaves_its_review_pending(
)
original = checkpoint.read_bytes()
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
coverage = read_review_state(workbench_db, scan.scan_dir)
@@ -185,10 +134,10 @@ def test_discarded_worker_finding_leaves_its_review_pending(
"invalid_terminal", [False, True], ids=["valid-terminal", "invalid-terminal"]
)
def test_terminal_worker_authority_requires_valid_findings(
- workbench_api, workbench_db, deadline_scan, invalid_terminal
+ workbench_db, deadline_scan, invalid_terminal
):
scan = deadline_scan
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
previous = worker_finding(scan, "earlier-valid-finding")
checkpoint = write_checkpoint(
output / "checkpoints",
@@ -218,7 +167,7 @@ def test_terminal_worker_authority_requires_valid_findings(
)
original = {path: path.read_bytes() for path in (checkpoint, result)}
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
scan.complete()
findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
@@ -229,11 +178,9 @@ def test_terminal_worker_authority_requires_valid_findings(
assert {path: path.read_bytes() for path in original} == original
-def test_malformed_worker_finding_preserves_a_valid_sibling_outcome(
- workbench_api, workbench_db, deadline_scan
-):
+def test_malformed_worker_finding_preserves_a_valid_sibling_outcome(workbench_db, deadline_scan):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(workbench_db, scan)
+ worker_id, output = scan.canceled_discovery_output()
earlier = worker_finding(scan, "rejected-candidate")
earlier["provenance"]["candidateId"] = "candidate"
write_checkpoint(
@@ -256,7 +203,7 @@ def test_malformed_worker_finding_preserves_a_valid_sibling_outcome(
mtime_ns=200,
)
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
scan.complete()
findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
@@ -270,9 +217,7 @@ def test_malformed_worker_finding_preserves_a_valid_sibling_outcome(
@pytest.mark.parametrize("invalid", ["surfaces", "missing-receipt"])
-def test_deadline_keeps_canonical_coverage_validation_strict(
- workbench_api, workbench_db, deadline_scan, invalid
-):
+def test_deadline_keeps_canonical_coverage_validation_strict(workbench_db, deadline_scan, invalid):
from finalize_scan_contract import ContractError
scan = deadline_scan
@@ -291,11 +236,11 @@ def test_deadline_keeps_canonical_coverage_validation_strict(
coverage_file = scan.scan_dir / "coverage.json"
coverage_file.write_text(json.dumps(scan.coverage))
original = coverage_file.read_bytes()
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
write_checkpoint(output / "checkpoints", saved_draft(scan.scan_id))
with pytest.raises(ContractError, match="surfaces|receiptRefs"):
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
assert coverage_file.read_bytes() == original
assert (
@@ -308,7 +253,7 @@ def test_deadline_keeps_canonical_coverage_validation_strict(
@pytest.mark.parametrize("worker_writeup_exists", [False, True], ids=["missing", "present"])
def test_worker_writeups_remain_bound_to_their_source_directory(
- workbench_api, workbench_db, deadline_scan, worker_writeup_exists
+ deadline_scan, worker_writeup_exists
):
scan = deadline_scan
report_path = "findings/review/review.md"
@@ -320,7 +265,7 @@ def test_worker_writeups_remain_bound_to_their_source_directory(
originals = {parent_writeup: parent_writeup.read_bytes()}
sources = []
for archived in (False, True):
- _, output = canceled_discovery_output(workbench_db, scan, attempt=2 if archived else 1)
+ _, output = scan.canceled_discovery_output(attempt=2 if archived else 1)
source = output.parent / "attempts" / "attempt-1" if archived else output
sources.append(source.relative_to(scan.scan_dir).as_posix())
finding = worker_finding(scan, "archived-worker" if archived else "current-worker")
@@ -334,7 +279,7 @@ def test_worker_writeups_remain_bound_to_their_source_directory(
)
originals[checkpoint] = checkpoint.read_bytes()
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
@@ -377,7 +322,6 @@ def test_worker_writeups_remain_bound_to_their_source_directory(
],
)
def test_deadline_recovery_is_idempotent_after_crash_before_commit(
- workbench_api,
workbench_db,
deadline_scan,
tmp_path,
@@ -396,7 +340,7 @@ def test_deadline_recovery_is_idempotent_after_crash_before_commit(
connection = sqlite3.connect(database)
connection.row_factory = sqlite3.Row
workbench_db.backup(connection)
- worker_id, output = canceled_discovery_output(connection, scan)
+ worker_id, output = scan.canceled_discovery_output(connection)
surface_id = f"{worker_id}/entry" if worker_like_id else "entry"
surfaces = [
{
@@ -439,7 +383,7 @@ def test_deadline_recovery_is_idempotent_after_crash_before_commit(
).fetchone()[0]
== "running"
)
- finish(workbench_api, connection, scan, terminal_reason="capped")
+ scan.finish(connection, terminal_reason="capped")
scan.complete(connection)
after = read_review_state(workbench_db, scan.scan_dir)
@@ -456,10 +400,10 @@ def test_deadline_recovery_is_idempotent_after_crash_before_commit(
@pytest.mark.parametrize("earlier_checkpoint_first", [False, True])
@pytest.mark.parametrize("colliding_ids", [False, True])
def test_deadline_rejection_history_is_stable_across_repeated_crashes(
- workbench_db, deadline_scan, finish_after_crashes, earlier_checkpoint_first, colliding_ids
+ deadline_scan, finish_after_crashes, earlier_checkpoint_first, colliding_ids
):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(workbench_db, scan)
+ worker_id, output = scan.canceled_discovery_output()
finding = worker_finding(scan, "reviewed-candidate")
finding["provenance"]["candidateId"] = "candidate"
# Check both hash filename orders without adding invalid or unrelated checkpoints.
diff --git a/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py b/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py
index 6af911ecc..ae439b21d 100644
--- a/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py
+++ b/plugins/codex-security/tests/test_deep_scan_deadline_surface_links.py
@@ -4,11 +4,6 @@ import json
import os
import pytest
-from test_deep_scan_deadline_recovery import finish_after_crashes as finish_after_crashes
-from test_deep_scan_stop_conditions import canceled_discovery_output, finish
-from test_deep_scan_stop_conditions import deadline_scan as deadline_scan
-from test_deep_scan_stop_conditions import publication_scan as publication_scan
-from test_deep_scan_stop_conditions import saturated_scan as saturated_scan
from workbench_test_support import (
read_review_state,
replay_saved_results,
@@ -32,10 +27,10 @@ def pending(identity, reason, surface_id="entry"):
"parent_collision", [pytest.param(False, marks=pytest.mark.cross_platform), True]
)
def test_deadline_surface_links_follow_sources_across_repeated_crashes(
- workbench_db, deadline_scan, finish_after_crashes, parent_collision
+ deadline_scan, finish_after_crashes, parent_collision
):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2)
+ worker_id, output = scan.canceled_discovery_output(attempt=2)
archived = output.parent / "attempts" / "attempt-01"
shared = pending("shared-review", "Review all retained observations.")
checkpoints = []
@@ -77,7 +72,7 @@ def test_deadline_surface_links_follow_sources_across_repeated_crashes(
@pytest.mark.parametrize("with_reduction", [False, True], ids=["zero-accepted", "reduced"])
def test_deadline_retains_all_targets_of_one_legacy_surface_id(
- workbench_api, workbench_db, publication_scan, request, with_reduction
+ workbench_db, publication_scan, request, with_reduction
):
scan = request.getfixturevalue("deadline_scan") if with_reduction else publication_scan()
if not with_reduction:
@@ -88,7 +83,7 @@ def test_deadline_retains_all_targets_of_one_legacy_surface_id(
"terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?",
(scan.scan_id,),
)
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
checkpoint = write_checkpoint(
output / "checkpoints",
saved_draft(
@@ -99,7 +94,7 @@ def test_deadline_retains_all_targets_of_one_legacy_surface_id(
)
original = checkpoint.read_bytes()
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
scan.complete()
coverage = read_review_state(workbench_db, scan.scan_dir)
@@ -112,11 +107,9 @@ def test_deadline_retains_all_targets_of_one_legacy_surface_id(
assert checkpoint.read_bytes() == original
-def test_deadline_deferred_only_checkpoint_keeps_prior_surface_link(
- workbench_api, workbench_db, deadline_scan
-):
+def test_deadline_deferred_only_checkpoint_keeps_prior_surface_link(workbench_db, deadline_scan):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(workbench_db, scan)
+ worker_id, output = scan.canceled_discovery_output()
earlier = write_checkpoint(
output / "checkpoints",
saved_draft(scan.scan_id, surfaces=[surface("Earlier review", "prior")]),
@@ -129,7 +122,7 @@ def test_deadline_deferred_only_checkpoint_keeps_prior_surface_link(
os.utime(current, ns=(200, 200))
originals = {path: path.read_bytes() for path in (earlier, current)}
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
scan.complete()
coverage = read_review_state(workbench_db, scan.scan_dir)
@@ -139,11 +132,9 @@ def test_deadline_deferred_only_checkpoint_keeps_prior_surface_link(
assert {path: path.read_bytes() for path in originals} == originals
-def test_deadline_reopened_generic_surface_copy_retains_its_link(
- workbench_api, workbench_db, deadline_scan
-):
+def test_deadline_reopened_generic_surface_copy_retains_its_link(workbench_db, deadline_scan):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(workbench_db, scan)
+ worker_id, output = scan.canceled_discovery_output()
canonical = surface("Canonical parent", f"{worker_id}/entry")
scan.coverage["surfaces"] = [canonical]
(scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))
@@ -172,7 +163,7 @@ def test_deadline_reopened_generic_surface_copy_retains_its_link(
head = select_checkpoint(output, checkpoint)
originals[head] = head.read_bytes()
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
scan.complete()
coverage = read_review_state(workbench_db, scan.scan_dir)
@@ -217,10 +208,10 @@ def test_stopped_recovery_preserves_source_link_union(workbench_api, tmp_path):
@pytest.mark.parametrize("changed_task", [False, True], ids=["copied-task", "updated-task"])
def test_deadline_retains_equal_task_origins_from_selected_worker_result(
- workbench_db, deadline_scan, finish_after_crashes, changed_task
+ deadline_scan, finish_after_crashes, changed_task
):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2)
+ worker_id, output = scan.canceled_discovery_output(attempt=2)
archived = output.parent / "attempts" / "attempt-01"
previous_task = pending("download-review", "Download review pending.")
archived_draft = saved_draft(
@@ -270,10 +261,10 @@ def test_deadline_retains_equal_task_origins_from_selected_worker_result(
"selected_result", [False, True], ids=["raw-checkpoint", "selected-result"]
)
def test_deadline_dropped_legacy_surface_does_not_rebind_its_pending_task(
- workbench_db, deadline_scan, finish_after_crashes, selected_result
+ deadline_scan, finish_after_crashes, selected_result
):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2)
+ worker_id, output = scan.canceled_discovery_output(attempt=2)
archived = output.parent / "attempts" / "attempt-01"
download_task = pending("download-review", "Download review pending.")
archived_checkpoint = write_checkpoint(
@@ -328,11 +319,11 @@ def test_deadline_dropped_legacy_surface_does_not_rebind_its_pending_task(
["parent-collision", "current-checkpoints", "retry-checkpoints", "coalesced-history"],
)
def test_deadline_inherited_surface_links_follow_historical_observations(
- workbench_db, deadline_scan, finish_after_crashes, history
+ deadline_scan, finish_after_crashes, history
):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(
- workbench_db, scan, attempt=2 if history == "retry-checkpoints" else 1
+ worker_id, output = scan.canceled_discovery_output(
+ attempt=2 if history == "retry-checkpoints" else 1
)
archived = output.parent / "attempts" / "attempt-01"
first_output = archived if history == "retry-checkpoints" else output
@@ -404,10 +395,10 @@ def test_deadline_inherited_surface_links_follow_historical_observations(
ids=["same-id", "changed-id", "legacy-duplicate-id"],
)
def test_deadline_candidate_supersession_preserves_independent_surface_links(
- workbench_db, deadline_scan, finish_after_crashes, changed_surface_id, legacy_duplicate_id
+ deadline_scan, finish_after_crashes, changed_surface_id, legacy_duplicate_id
):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(workbench_db, scan)
+ worker_id, output = scan.canceled_discovery_output()
canonical_id = f"{worker_id}/entry"
canonical_surface = surface("Canonical parent", canonical_id)
canonical_task = pending("parent-review", "Parent review remains.", canonical_id)
diff --git a/plugins/codex-security/tests/test_deep_scan_publication_authority.py b/plugins/codex-security/tests/test_deep_scan_publication_authority.py
index 91ef1b079..d5aab2a1b 100644
--- a/plugins/codex-security/tests/test_deep_scan_publication_authority.py
+++ b/plugins/codex-security/tests/test_deep_scan_publication_authority.py
@@ -12,7 +12,6 @@ from pathlib import Path
from types import SimpleNamespace
import pytest
-from test_deep_scan_successful_publication import publication_scan as publication_scan
from workbench_test_support import read_review_state
diff --git a/plugins/codex-security/tests/test_deep_scan_publication_replay.py b/plugins/codex-security/tests/test_deep_scan_publication_replay.py
index ccae59463..34871113b 100644
--- a/plugins/codex-security/tests/test_deep_scan_publication_replay.py
+++ b/plugins/codex-security/tests/test_deep_scan_publication_replay.py
@@ -9,7 +9,6 @@ from pathlib import Path
import pytest
from test_deep_scan_publication_authority import stage_publication
-from test_deep_scan_successful_publication import publication_scan as publication_scan
_CRASH_PUBLICATION = """
import json, os, runpy, sqlite3, sys
diff --git a/plugins/codex-security/tests/test_deep_scan_stop_conditions.py b/plugins/codex-security/tests/test_deep_scan_stop_conditions.py
index d6a859265..0d9a54024 100644
--- a/plugins/codex-security/tests/test_deep_scan_stop_conditions.py
+++ b/plugins/codex-security/tests/test_deep_scan_stop_conditions.py
@@ -4,13 +4,9 @@ import copy
import json
import os
import sqlite3
-from argparse import Namespace
import pytest
-from test_deep_scan_successful_publication import assert_published_aggregate
-from test_deep_scan_successful_publication import (
- publication_scan as publication_scan,
-)
+from publication_fixtures import assert_published_aggregate
from workbench_test_support import (
read_review_state,
saved_draft,
@@ -19,90 +15,8 @@ from workbench_test_support import (
)
-def saturated_scan(publication_scan, workbench_db):
- scan = publication_scan()
- completed_result = scan.add_worker()
- completed_result.write_text(
- json.dumps(
- {
- "scanId": scan.scan_id,
- "complete": True,
- "findings": scan.findings,
- "coverage": scan.coverage,
- }
- )
- )
- reducer_result = scan.add_worker()
- reducer_result.write_bytes(completed_result.read_bytes())
- with workbench_db:
- workbench_db.execute(
- "UPDATE deep_scan_workers SET completion_sequence = 1 WHERE id = ?",
- (completed_result.parent.name,),
- )
- workbench_db.execute(
- "UPDATE deep_scan_workers SET kind = 'dedup', merge_state = 'none' WHERE id = ?",
- (reducer_result.parent.name,),
- )
- workbench_db.execute(
- "INSERT INTO deep_scan_dedup_inputs "
- "(scan_id, dedup_worker_id, discovery_worker_id, input_order) VALUES (?, ?, ?, 0)",
- (scan.scan_id, reducer_result.parent.name, completed_result.parent.name),
- )
- workbench_db.execute(
- "UPDATE deep_scan_runs SET status = 'running', phase = 'discovery', "
- "consecutive_no_new = stop_after_no_new, completion_sequence = 1, "
- "max_discovery_runs = 3, discovery_runs_dispatched = 1, "
- "manifest_path = NULL, terminal_reason = NULL, completed_at = NULL WHERE scan_id = ?",
- (scan.scan_id,),
- )
- scan.completed_worker_id = completed_result.parent.name
- scan.reducer_id = reducer_result.parent.name
- return scan
-
-
-def finish(workbench_api, connection, scan, *, terminal_reason="saturated"):
- return workbench_api["deep_scan"].finish_deep_scan(
- connection,
- Namespace(
- scan_id=scan.scan_id,
- terminal_reason=terminal_reason,
- manifest_path=str(scan.scan_dir / "scan-manifest.json"),
- staged_manifest_path=None,
- omitted_worker_id=[],
- ),
- )["deepScan"]
-
-
-def deadline_scan(saturated_scan, workbench_db):
- with workbench_db:
- workbench_db.execute(
- "UPDATE deep_scan_runs SET created_at = '2000-01-01T00:00:00+00:00' WHERE scan_id = ?",
- (saturated_scan.scan_id,),
- )
- return saturated_scan
-
-
-def canceled_discovery_output(connection, scan, *, attempt=1):
- result = scan.add_worker(connection, status="canceled")
- worker_id = result.parent.name
- output = scan.scan_dir / "artifacts" / "deep_discovery" / "workers" / worker_id / "output"
- output.parent.mkdir(parents=True)
- result.parent.rename(output)
- with connection:
- connection.execute(
- "UPDATE deep_scan_workers SET artifact_dir = ?, prompt_path = ?, "
- "result_manifest_path = NULL, attempt = ? WHERE id = ?",
- (str(output), str(output.parent / "prompt.md"), attempt, worker_id),
- )
- return worker_id, output
-
-
@pytest.mark.parametrize("missing_writeup", [False, True], ids=["present", "missing"])
-def test_deadline_preserves_optional_reduced_writeup_behavior(
- workbench_api, workbench_db, deadline_scan, missing_writeup
-):
+def test_deadline_preserves_optional_reduced_writeup_behavior(deadline_scan, missing_writeup):
scan = deadline_scan
report_path = "findings/reduced/reduced.md"
scan.findings[0]["writeup"] = {"reportPath": report_path}
@@ -111,13 +25,13 @@ def test_deadline_preserves_optional_reduced_writeup_behavior(
report = scan.scan_dir / report_path
report.parent.mkdir(parents=True)
report.write_text("# Reduced finding\n")
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
write_checkpoint(
output / "checkpoints",
saved_draft(scan.scan_id, deferred=[{"reason": "Review the remaining upload paths."}]),
)
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
assert completed["progress"]["status"] == "complete"
@@ -130,10 +44,10 @@ def test_deadline_preserves_optional_reduced_writeup_behavior(
def test_deadline_recovers_valid_findings_from_an_invalid_archived_retry(
- workbench_api, workbench_db, deadline_scan
+ workbench_api, deadline_scan
):
scan = deadline_scan
- worker_id, output = canceled_discovery_output(workbench_db, scan, attempt=2)
+ worker_id, output = scan.canceled_discovery_output(attempt=2)
archived = output.parent / "attempts" / "attempt-1"
archived.mkdir(parents=True)
valid = copy.deepcopy(scan.findings[0])
@@ -153,7 +67,7 @@ def test_deadline_recovers_valid_findings_from_an_invalid_archived_retry(
saved_draft(scan.scan_id, deferred=[{"reason": "Retry review remains unfinished."}]),
)
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
@@ -172,9 +86,7 @@ def test_deadline_recovers_valid_findings_from_an_invalid_archived_retry(
@pytest.mark.parametrize("invalid", ["finding", "unsafe-writeup"])
-def test_deadline_does_not_recover_invalid_reduced_findings(
- workbench_api, workbench_db, deadline_scan, invalid
-):
+def test_deadline_does_not_recover_invalid_reduced_findings(workbench_db, deadline_scan, invalid):
from finalize_scan_contract import ContractError
scan = deadline_scan
@@ -185,11 +97,11 @@ def test_deadline_does_not_recover_invalid_reduced_findings(
findings_path = scan.scan_dir / "findings.json"
findings_path.write_text(json.dumps({"findings": scan.findings}))
original = findings_path.read_bytes()
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
write_checkpoint(output / "checkpoints", saved_draft(scan.scan_id))
with pytest.raises(ContractError, match="remediation|reportPath"):
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
assert findings_path.read_bytes() == original
assert (
@@ -200,11 +112,9 @@ def test_deadline_does_not_recover_invalid_reduced_findings(
)
-def test_deadline_preserves_worker_findings_with_optional_and_shared_identity(
- workbench_api, workbench_db, deadline_scan
-):
+def test_deadline_preserves_worker_findings_with_optional_and_shared_identity(deadline_scan):
scan = deadline_scan
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
no_identity = copy.deepcopy(scan.findings[0])
no_identity.pop("identity")
no_identity["title"] = "Worker finding without authored identity"
@@ -222,7 +132,7 @@ def test_deadline_preserves_worker_findings_with_optional_and_shared_identity(
saved_draft(scan.scan_id, findings=[no_identity, first, second, malformed]),
)
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
@@ -233,11 +143,9 @@ def test_deadline_preserves_worker_findings_with_optional_and_shared_identity(
assert any("remediation" in warning for warning in completed["warnings"])
-def test_deadline_resolves_worker_local_deferred_ids_before_scoping(
- workbench_api, workbench_db, deadline_scan
-):
+def test_deadline_resolves_worker_local_deferred_ids_before_scoping(workbench_db, deadline_scan):
scan = deadline_scan
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
write_checkpoint(
output / "checkpoints",
saved_draft(scan.scan_id, deferred=[{"id": "review", "reason": "Review pending."}]),
@@ -248,7 +156,7 @@ def test_deadline_resolves_worker_local_deferred_ids_before_scoping(
latest = write_checkpoint(output / "checkpoints", current, mtime_ns=200)
select_checkpoint(output, latest)
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
scan.complete()
coverage = read_review_state(workbench_db, scan.scan_dir)
@@ -256,10 +164,10 @@ def test_deadline_resolves_worker_local_deferred_ids_before_scoping(
def test_deadline_does_not_turn_invalid_worker_receipts_into_parent_artifacts(
- workbench_api, workbench_db, deadline_scan
+ workbench_db, deadline_scan
):
scan = deadline_scan
- _, output = canceled_discovery_output(workbench_db, scan)
+ _, output = scan.canceled_discovery_output()
(output / "notes.txt").write_text("Not a worker receipt under artifacts/.\n")
write_checkpoint(
output / "checkpoints",
@@ -276,7 +184,7 @@ def test_deadline_does_not_turn_invalid_worker_receipts_into_parent_artifacts(
),
)
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
completed = scan.complete()
surface = read_review_state(workbench_db, scan.scan_dir)["surfaces"][0]
@@ -313,9 +221,7 @@ def test_deadline_preserves_worker_receipts_and_surface_links(
(scan.scan_dir / "coverage.json").write_text(json.dumps(scan.coverage))
expected = {}
for label in ("upload", "download"):
- worker_id, output = canceled_discovery_output(
- workbench_db, scan, attempt=2 if archived else 1
- )
+ worker_id, output = scan.canceled_discovery_output(attempt=2 if archived else 1)
source = output.parent / "attempts" / "attempt-1" if archived else output
receipt = source / "artifacts" / "review.txt"
receipt.parent.mkdir(parents=True)
@@ -341,7 +247,7 @@ def test_deadline_preserves_worker_receipts_and_surface_links(
)
expected[label] = (worker_id, receipt, checkpoint, checkpoint.read_bytes())
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
scan.complete()
coverage = read_review_state(workbench_db, scan.scan_dir)
@@ -366,9 +272,7 @@ def test_deadline_preserves_worker_receipts_and_surface_links(
assert checkpoint.read_bytes() == original
-def test_saturated_finish_cancels_unfinished_discovery_without_using_its_drafts(
- workbench_api, workbench_db, saturated_scan
-):
+def test_saturated_finish_cancels_unfinished_discovery_without_using_its_drafts(saturated_scan):
scan = saturated_scan
unfinished = []
for status in ("queued", "running"):
@@ -376,7 +280,7 @@ def test_saturated_finish_cancels_unfinished_discovery_without_using_its_drafts(
result.write_text("{unfinished worker draft")
unfinished.append(result)
- finished = finish(workbench_api, workbench_db, scan)
+ finished = scan.finish()
assert finished["status"] == "succeeded"
assert finished["terminalReason"] == "saturated"
@@ -395,9 +299,7 @@ def test_saturated_finish_cancels_unfinished_discovery_without_using_its_drafts(
assert_published_aggregate(scan)
-def test_saturated_finish_retains_failed_discovery_diagnostic(
- workbench_api, workbench_db, saturated_scan
-):
+def test_saturated_finish_retains_failed_discovery_diagnostic(workbench_db, saturated_scan):
scan = saturated_scan
result = scan.add_worker(status="failed")
error = "Worker stopped while persisting its result."
@@ -407,7 +309,7 @@ def test_saturated_finish_retains_failed_discovery_diagnostic(
(error, result.parent.name),
)
- finished = finish(workbench_api, workbench_db, scan)
+ finished = scan.finish()
assert finished["status"] == "succeeded"
failed_worker = next(
@@ -420,9 +322,7 @@ def test_saturated_finish_retains_failed_discovery_diagnostic(
@pytest.mark.parametrize("stop", ["deadline", "run-limit", "saturated"])
-def test_unfinished_checkpoints_after_a_completed_reduction(
- workbench_api, workbench_db, saturated_scan, stop
-):
+def test_unfinished_checkpoints_after_a_completed_reduction(workbench_db, saturated_scan, stop):
scan = saturated_scan
result = scan.add_worker(status="canceled")
recovered = copy.deepcopy(scan.findings[0])
@@ -493,7 +393,7 @@ def test_unfinished_checkpoints_after_a_completed_reduction(
)
reason = "saturated" if stop == "saturated" else "capped"
- finished = finish(workbench_api, workbench_db, scan, terminal_reason=reason)
+ finished = scan.finish(terminal_reason=reason)
assert finished["status"] == "succeeded"
worker = next(item for item in finished["workers"] if item["id"] == result.parent.name)
@@ -539,14 +439,12 @@ def test_unfinished_checkpoints_after_a_completed_reduction(
assert checkpoint.read_bytes() == source_bytes
names = ("scan-manifest.json", "findings.json", "report.md")
sealed = {name: (scan.scan_dir / name).read_bytes() for name in names}
- finish(workbench_api, workbench_db, scan, terminal_reason=reason)
+ scan.finish(terminal_reason=reason)
scan.complete()
assert {name: (scan.scan_dir / name).read_bytes() for name in names} == sealed
-def test_deadline_after_all_reviews_complete_keeps_reduced_coverage(
- workbench_api, workbench_db, saturated_scan
-):
+def test_deadline_after_all_reviews_complete_keeps_reduced_coverage(workbench_db, saturated_scan):
scan = saturated_scan
with workbench_db:
workbench_db.execute(
@@ -554,7 +452,7 @@ def test_deadline_after_all_reviews_complete_keeps_reduced_coverage(
(scan.scan_id,),
)
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
scan.complete()
assert_published_aggregate(scan)
@@ -566,7 +464,7 @@ def test_deadline_after_all_reviews_complete_keeps_reduced_coverage(
ids=["saturated-setup", "saturated-reducer", "capped-discovery"],
)
def test_finish_preserves_other_worker_failure_checks(
- workbench_api, workbench_db, saturated_scan, terminal_reason, kind
+ workbench_db, saturated_scan, terminal_reason, kind
):
scan = saturated_scan
result = scan.add_worker(status="failed")
@@ -582,7 +480,7 @@ def test_finish_preserves_other_worker_failure_checks(
)
with pytest.raises(SystemExit, match="after a worker has failed"):
- finish(workbench_api, workbench_db, scan, terminal_reason=terminal_reason)
+ scan.finish(terminal_reason=terminal_reason)
run = workbench_db.execute(
"SELECT status, terminal_reason FROM deep_scan_runs WHERE scan_id = ?", (scan.scan_id,)
@@ -643,7 +541,7 @@ def test_rejected_deadline_publication_does_not_leave_parent_observations(
"WHEN NEW.status = 'succeeded' BEGIN SELECT RAISE(ABORT, 'deadline commit failed'); END"
)
with pytest.raises(sqlite3.IntegrityError, match="deadline commit failed"):
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ scan.finish(terminal_reason="capped")
assert observations() == before
assert {
@@ -658,9 +556,7 @@ def test_rejected_deadline_publication_does_not_leave_parent_observations(
)
with workbench_db:
workbench_db.execute("DROP TRIGGER reject_deadline_commit")
- assert (
- finish(workbench_api, workbench_db, scan, terminal_reason="capped")["status"] == "succeeded"
- )
+ assert scan.finish(terminal_reason="capped")["status"] == "succeeded"
if existing_head:
coverage = json.loads((scan.scan_dir / "scan-manifest.json").read_text())["scan"][
"reviewState"
@@ -669,7 +565,7 @@ def test_rejected_deadline_publication_does_not_leave_parent_observations(
def test_zero_accepted_deadline_preserves_worker_checkpoint_links_and_receipts(
- workbench_api, workbench_db, publication_scan
+ workbench_db, publication_scan
):
scan = publication_scan()
workers = {}
@@ -718,7 +614,7 @@ def test_zero_accepted_deadline_preserves_worker_checkpoint_links_and_receipts(
(scan.scan_id,),
)
- finished = finish(workbench_api, workbench_db, scan, terminal_reason="capped")
+ finished = scan.finish(terminal_reason="capped")
assert finished["status"] == "succeeded"
assert finished["completionSequence"] == 0
diff --git a/plugins/codex-security/tests/test_deep_scan_successful_publication.py b/plugins/codex-security/tests/test_deep_scan_successful_publication.py
index 9128a1a2f..19d2ad175 100644
--- a/plugins/codex-security/tests/test_deep_scan_successful_publication.py
+++ b/plugins/codex-security/tests/test_deep_scan_successful_publication.py
@@ -4,218 +4,18 @@ import copy
import json
import os
import shutil
-import uuid
from argparse import Namespace
from pathlib import Path
-from types import SimpleNamespace
import pytest
+from publication_fixtures import assert_published_aggregate
from workbench_test_support import (
saved_coverage,
saved_draft,
write_checkpoint,
- write_completed_contract,
)
-class PublicationScan(SimpleNamespace):
- def add_worker(self, connection=None, *, status="succeeded") -> Path:
- worker_id = str(uuid.uuid4())
- output = self.scan_dir / "workers" / worker_id
- output.mkdir(parents=True)
- result = output / "result.json"
- connection = self.connection if connection is None else connection
- with connection:
- connection.execute(
- "INSERT INTO deep_scan_workers (id, scan_id, kind, status, merge_state, "
- "prompt_path, artifact_dir, result_manifest_path, attempt, created_at, "
- "updated_at, completed_at) VALUES (?, ?, 'discovery', ?, ?, ?, ?, ?, 1, ?, ?, ?)",
- (
- worker_id,
- self.scan_id,
- status,
- "merged" if status == "succeeded" else "none",
- str(output / "prompt.md"),
- str(output),
- str(result),
- self.timestamp,
- self.timestamp,
- self.timestamp,
- ),
- )
- return result
-
- def complete(self, connection=None, *, prepare_only=False):
- return self.api["complete_scan"](
- self.connection if connection is None else connection,
- Namespace(scan_id=self.scan_id, claim_token=None, cost_json=None),
- prepare_only=prepare_only,
- )["scan"]
-
- def fail(self, message="Stopped."):
- return self.api["saved_results"].fail_scan(
- self.api["_WORKBENCH_DB_CONTEXT"],
- self.connection,
- Namespace(scan_id=self.scan_id, claim_token=None, cost_json=None, message=message),
- )["scan"]
-
- def preserve(self, connection=None):
- return self.api["saved_results"].preserve_scan_results(
- self.api["_WORKBENCH_DB_CONTEXT"],
- self.connection if connection is None else connection,
- Namespace(
- scan_id=self.scan_id, claim_token=None, thread_id=None, coordinator_generation=None
- ),
- )["scan"]
-
- def recover(self):
- return self.api["saved_results"].recover_scan_results(
- self.api["_WORKBENCH_DB_CONTEXT"], self.connection, Namespace(scan_id=self.scan_id)
- )["scan"]
-
-
-def publication_scan(workbench_api, workbench_db, tmp_path, monkeypatch):
- monkeypatch.setenv("CODEX_SECURITY_STATE_DIR", str(tmp_path / "state"))
- monkeypatch.setenv("CODEX_HOME", str(tmp_path / "codex-home"))
- deep = workbench_api["deep_scan"]
- monkeypatch.setattr(
- deep,
- "_dependencies",
- SimpleNamespace(
- **workbench_api,
- preserve_stopped_results=workbench_api["preserve_stopped_results_after_transition"],
- ),
- )
-
- def create(*, mode="deep", scope=".", paths=None):
- target = tmp_path / "target"
- target.mkdir()
- (target / "subdir").mkdir()
- (target / "subdir" / "extract.py").write_text("# Synthetic scan target\n")
- if paths is not None:
- for directory in [*paths, "unrelated"]:
- if not (target / directory).is_file():
- (target / directory).mkdir(exist_ok=True)
- (target / directory / "extract.py").write_text("# Synthetic scoped target\n")
- include_paths = paths if paths is not None else [scope]
- scan_dir = tmp_path / "scan"
- scan_dir.mkdir(mode=0o700)
- registered = workbench_api["register_cli_scan"](
- workbench_db,
- Namespace(
- repository=str(target),
- scan_dir=str(scan_dir),
- recipe_json=json.dumps(
- {
- "config": {},
- "mode": mode,
- "repository": str(target),
- "target": {
- "kind": "paths" if paths is not None or scope != "." else "repository",
- "paths": include_paths if paths is not None or scope != "." else [],
- },
- }
- ),
- registration_json_stdin=False,
- recipe_json_stdin=False,
- parent_scan_id=None,
- archive_existing=False,
- archived_scan_dir=None,
- ),
- )
- scan_id = registered["scanId"]
- timestamp = workbench_db.execute(
- "SELECT started_at FROM scans WHERE id = ?", (scan_id,)
- ).fetchone()[0]
- if mode == "deep":
- # Start with a finished Deep result so these tests only need to save it.
- with workbench_db:
- workbench_db.execute(
- "INSERT INTO deep_scan_runs (scan_id, schema_version, workflow_version, "
- "status, phase, workers, subagents, stop_after_no_new, max_discovery_runs, "
- "manifest_path, terminal_reason, created_at, updated_at, completed_at) "
- "VALUES (?, 1, 'deep-security-scan/v1', 'succeeded', 'terminal', 1, 0, 1, 1, "
- "?, 'saturated', ?, ?, ?)",
- (
- scan_id,
- str(scan_dir / "scan-manifest.json"),
- timestamp,
- timestamp,
- timestamp,
- ),
- )
- coverage_mode = (
- "scoped_path"
- if paths is not None or scope != "."
- else "deep_repository"
- if mode == "deep"
- else "repository"
- )
- write_completed_contract(
- scan_dir,
- scan_id,
- target,
- include_paths=include_paths,
- relative_path="subdir/extract.py",
- coverage_mode=coverage_mode,
- inventory_strategy="scoped_path" if scope != "." else "repository",
- )
- manifest = json.loads((scan_dir / "scan-manifest.json").read_text())
- findings = json.loads((scan_dir / "findings.json").read_text())["findings"]
- coverage = json.loads((scan_dir / "coverage.json").read_text())
- if mode == "deep":
- coverage.update(surfaces=[], explicitExclusions=[], deferred=[])
- else:
- coverage["openQuestions"] = [{"question": "Which deployment controls apply?"}]
- for field in ("documentType", "schemaVersion", "scanId"):
- coverage.pop(field)
- # Use the same draft format as the writer, before finalization adds metadata.
- manifest = {"scan": {key: manifest["scan"][key] for key in ("target", "scope")}}
- findings[0]["severity"]["changeConditions"] = "Reassess if the upload route is removed."
- findings[0]["provenance"]["sourceFindings"] = [
- {"id": "review-1:candidate-1", "finding": {"summary": "Retained original wording."}}
- ]
- for name, value in (
- ("scan-manifest.json", manifest),
- ("findings.json", {"findings": findings}),
- ("coverage.json", coverage),
- ):
- (scan_dir / name).write_text(json.dumps(value))
- return PublicationScan(
- api=workbench_api,
- connection=workbench_db,
- scan_id=scan_id,
- scan_dir=scan_dir,
- timestamp=timestamp,
- findings=findings,
- coverage=coverage,
- )
-
- return create
-
-
-def assert_published_aggregate(scan):
- findings = json.loads((scan.scan_dir / "findings.json").read_text())["findings"]
- for finding in findings:
- for field in ("findingId", "occurrenceId", "fingerprints"):
- value = finding.pop(field)
- assert value
- assert findings == scan.findings
- manifest = json.loads((scan.scan_dir / "scan-manifest.json").read_text())
- if "coverageRef" in manifest["scan"]:
- coverage = json.loads((scan.scan_dir / "coverage.json").read_text())
- for field in ("documentType", "schemaVersion", "scanId"):
- coverage.pop(field)
- assert coverage == scan.coverage
- else:
- assert manifest["scan"]["review"] == {
- "excludedPaths": [item["pattern"] for item in scan.coverage["explicitExclusions"]],
- }
- assert not (scan.scan_dir / "coverage.json").exists()
- assert (scan.scan_dir / "report.md").is_file()
-
-
@pytest.mark.parametrize("mode", ["standard", "deep", "deep-nested"])
@pytest.mark.parametrize("linked_writeup", [False, True], ids=["inline", "linked"])
def test_publication_renders_each_source_remediation(publication_scan, mode, linked_writeup):
diff --git a/plugins/codex-security/tests/test_partial_coverage_projection.py b/plugins/codex-security/tests/test_partial_coverage_projection.py
index decb01507..a0d2020cc 100644
--- a/plugins/codex-security/tests/test_partial_coverage_projection.py
+++ b/plugins/codex-security/tests/test_partial_coverage_projection.py
@@ -8,7 +8,6 @@ import uuid
from argparse import Namespace
import pytest
-from test_deep_scan_successful_publication import publication_scan as publication_scan
from workbench_test_support import (
read_review_state,
select_checkpoint,
diff --git a/plugins/codex-security/tests/test_root_candidate_authority.py b/plugins/codex-security/tests/test_root_candidate_authority.py
index c33be9b5a..b9ed0e7b0 100644
--- a/plugins/codex-security/tests/test_root_candidate_authority.py
+++ b/plugins/codex-security/tests/test_root_candidate_authority.py
@@ -7,7 +7,6 @@ import os
from argparse import Namespace
import pytest
-from test_deep_scan_successful_publication import publication_scan as publication_scan
from workbench_test_support import (
read_review_state,
saved_draft,
diff --git a/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py b/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py
index de3d34da4..833492b52 100644
--- a/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py
+++ b/plugins/codex-security/tests/test_workbench_recovery_outcome_validation.py
@@ -5,7 +5,6 @@ import json
from types import SimpleNamespace
import pytest
-from test_deep_scan_successful_publication import publication_scan as publication_scan
from workbench_test_support import read_review_state, saved_draft, write_checkpoint
diff --git a/plugins/codex-security/tests/test_worker_receipt_admission.py b/plugins/codex-security/tests/test_worker_receipt_admission.py
index dbfa24645..b90bdaa24 100644
--- a/plugins/codex-security/tests/test_worker_receipt_admission.py
+++ b/plugins/codex-security/tests/test_worker_receipt_admission.py
@@ -8,7 +8,6 @@ from argparse import Namespace
from pathlib import Path
import pytest
-from test_deep_scan_successful_publication import publication_scan as publication_scan
from test_partial_coverage_projection import stop_and_recover_projection
from test_partial_coverage_projection import stop_scan as stop_scan
from test_saved_deferred_identity import recover, save_worker
diff --git a/sdk/typescript/tests-ts/api-preflight-config.test.ts b/sdk/typescript/tests-ts/api-preflight-config.test.ts
index 686a1f06c..cd0859655 100644
--- a/sdk/typescript/tests-ts/api-preflight-config.test.ts
+++ b/sdk/typescript/tests-ts/api-preflight-config.test.ts
@@ -8,42 +8,16 @@ import {
} from "../src/api.js";
import {
mergedCodexConfig,
- CHEAPERINFERENCE_CODEX_PROVIDER,
- FIREWORKS_CODEX_PROVIDER,
- OPENROUTER_CODEX_PROVIDER,
scanModelProvider,
writeCodexConfig,
type JsonObject,
} from "../src/config.js";
-import { TestClient } from "./support/api-client.js";
+import { EXTERNAL_PROVIDER_CASES, TestClient } from "./support/api-client.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";
const { temporaryDirectory, cleanup } = createApiTestFixtures(
"codex-security-preflight-",
);
-const EXTERNAL_PROVIDER_CASES = [
- [
- "OpenRouter",
- "openrouter",
- "OPENROUTER_API_KEY",
- "anthropic/claude-sonnet-4.5",
- OPENROUTER_CODEX_PROVIDER,
- ],
- [
- "Fireworks AI",
- "fireworks",
- "FIREWORKS_API_KEY",
- "accounts/fireworks/models/qwen3-235b-a22b",
- FIREWORKS_CODEX_PROVIDER,
- ],
- [
- "Cheaper Inference",
- "cheaperinference",
- "CHEAPER_INFERENCE_API_KEY",
- "gpt-5.4",
- CHEAPERINFERENCE_CODEX_PROVIDER,
- ],
-] as const;
afterEach(cleanup);
diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts
index b9c57e705..f375c48bd 100644
--- a/sdk/typescript/tests-ts/api.test.ts
+++ b/sdk/typescript/tests-ts/api.test.ts
@@ -62,9 +62,7 @@ import {
initialCredentialsAvailable,
} from "../src/api.js";
import {
- CHEAPERINFERENCE_CODEX_PROVIDER,
DEFAULT_CODEX_CONFIG,
- FIREWORKS_CODEX_PROVIDER,
OPENROUTER_CODEX_PROVIDER,
resolveCodexProfile,
type JsonObject,
@@ -89,6 +87,7 @@ import {
PLUGIN_ROOT,
} from "./plugin-root.js";
import {
+ EXTERNAL_PROVIDER_CASES,
mockScanRegistration,
mockWorkbench,
shellEnvironmentReference,
@@ -175,29 +174,6 @@ async function scanDirectories() {
return { ...directories, scanDir };
}
-const EXTERNAL_PROVIDER_CASES = [
- [
- "OpenRouter",
- "openrouter",
- "OPENROUTER_API_KEY",
- "anthropic/claude-sonnet-4.5",
- OPENROUTER_CODEX_PROVIDER,
- ],
- [
- "Fireworks AI",
- "fireworks",
- "FIREWORKS_API_KEY",
- "accounts/fireworks/models/qwen3-235b-a22b",
- FIREWORKS_CODEX_PROVIDER,
- ],
- [
- "Cheaper Inference",
- "cheaperinference",
- "CHEAPER_INFERENCE_API_KEY",
- "gpt-5.4",
- CHEAPERINFERENCE_CODEX_PROVIDER,
- ],
-] as const;
const BEDROCK_AUTHENTICATION_CASES = [
[
"Bedrock bearer token",
@@ -9106,23 +9082,23 @@ if __name__ == "__main__":
},
);
+ const progress = (
+ phase: ScanProgress["phase"],
+ filesCompleted: number,
+ filesTotal: number,
+ ): ScanProgress => ({ phase, filesCompleted, filesTotal });
+ const inventory = (total: number) => progress("discovery", 0, total);
+ const progressText = (value: ProgressScenarioValue) =>
+ typeof value === "string"
+ ? value
+ : [value]
+ .flat()
+ .map((p) => progressMarker(p.phase, p.filesCompleted, p.filesTotal))
+ .join("\n");
+
describe("partial parent source ordering", () => {
- const progress = (
- phase: ScanProgress["phase"],
- filesCompleted: number,
- filesTotal: number,
- ): ScanProgress => ({ phase, filesCompleted, filesTotal });
- const inventory = (total: number) => progress("discovery", 0, total);
const phase = progress("threat_model", 0, 5);
- type Step =
- | readonly ["save" | "live", string, ScanProgress | string]
- | readonly ["worker", ScanProgress]
- | readonly ["poll"];
- const scenarios: readonly {
- name: string;
- expected: ScanProgress;
- steps: readonly Step[];
- }[] = [
+ const scenarios: readonly ProgressScenario[] = [
{
name: "an earlier exact match before pending repeated inventory",
expected: progress("discovery", 5, 9),
@@ -9259,113 +9235,8 @@ if __name__ == "__main__":
scenarios.flatMap((scenario) =>
[false, true].map((workerFirst) => ({ ...scenario, workerFirst })),
),
- )(
- "preserves $name (workers first=$workerFirst)",
- async ({ steps, expected, workerFirst }) => {
- const { root, repository, codexHome, scanDir } =
- await scanDirectories();
- const updates: ScanProgress[] = [];
- let tracker: ScanCostTracker | undefined;
- const originalStart = ScanCostTracker.prototype.start;
- const starting = spyOn(
- ScanCostTracker.prototype,
- "start",
- ).mockImplementation(function (
- this: ScanCostTracker,
- threadId: string,
- ) {
- originalStart.call(this, threadId);
- if (threadId === "thread-1") tracker = this;
- });
- let parentPath = "",
- workerPath = "";
- const originalReaddir = fsPromises.readdir;
- const listing = spyOn(fsPromises, "readdir").mockImplementation((async (
- ...args: Parameters<typeof originalReaddir>
- ) => {
- const entries = await originalReaddir(...args);
- if (String(args[0]) !== dirname(parentPath)) return entries;
- const first = basename(workerFirst ? workerPath : parentPath);
- return entries.toSorted(
- (left, right) =>
- Number(String(right.name) === first) -
- Number(String(left.name) === first),
- );
- }) as typeof originalReaddir);
- const text = (value: string | ScanProgress) =>
- typeof value === "string"
- ? value
- : progressMarker(
- value.phase,
- value.filesCompleted,
- value.filesTotal,
- );
- const client = TestClient.withDependencies({
- ...scanRuntimeDependencies(codexHome, scanDir),
- runWorkbench: registerEightFileScan,
- createCodex: codexFactory(async () => {
- await copyCompletedScan(root);
- parentPath = await writeUsageSession(codexHome, "thread-1", {});
- workerPath = await writeUsageSession(
- codexHome,
- "worker-thread",
- {},
- { parent: "thread-1" },
- );
- async function* events(): AsyncGenerator<ThreadEvent> {
- expect(tracker).toBeDefined();
- await tracker!.stop();
- for (const [index, step] of steps.entries()) {
- if (step[0] === "poll") await tracker!.refresh();
- else if (step[0] === "live")
- yield completedCommand(
- step[1],
- text(step[2]),
- `item-${index}`,
- );
- else if (step[0] === "save")
- await appendFile(
- parentPath,
- jsonLines([
- nativeCommand(step[1], text(step[2]), `native-${index}`),
- ]) + "\n",
- );
- else
- await appendFile(
- workerPath,
- jsonLines([
- {
- type: "response_item",
- payload: {
- type: "custom_tool_call_output",
- call_id: `worker-${index}`,
- output: text(step[1]),
- },
- },
- ]) + "\n",
- );
- }
- await tracker!.refresh();
- await tracker!.refresh();
- await tracker!.stop();
- }
- return { events: completedEvents("thread-1", events()) };
- }),
- });
- try {
- await client.run(repository, {
- onProgress: (update) => updates.push(update),
- });
- expect(updates.at(-1)).toEqual(expected);
- await tracker!.refresh();
- await tracker!.stop();
- expect(updates.at(-1)).toEqual(expected);
- } finally {
- await client.close();
- starting.mockRestore();
- listing.mockRestore();
- }
- },
+ )("preserves $name (workers first=$workerFirst)", (scenario) =>
+ runProgressScenario(scenario),
);
});
@@ -9401,30 +9272,7 @@ if __name__ == "__main__":
async ({ scenario, workerFirst, pollFirst, deliverLast }) => {
const { root, repository, codexHome, scanDir } = await scanDirectories();
const updates: ScanProgress[] = [];
- let tracker: ScanCostTracker | undefined;
- const originalStart = ScanCostTracker.prototype.start;
- const starting = spyOn(
- ScanCostTracker.prototype,
- "start",
- ).mockImplementation(function (this: ScanCostTracker, threadId: string) {
- originalStart.call(this, threadId);
- if (threadId === "thread-1") tracker = this;
- });
- let parentPath = "";
- let workerPath = "";
- const originalReaddir = fsPromises.readdir;
- const listing = spyOn(fsPromises, "readdir").mockImplementation((async (
- ...args: Parameters<typeof originalReaddir>
- ) => {
- const entries = await originalReaddir(...args);
- if (String(args[0]) !== dirname(parentPath)) return entries;
- const first = basename(workerFirst ? workerPath : parentPath);
- return entries.toSorted(
- (left, right) =>
- Number(String(right.name) === first) -
- Number(String(left.name) === first),
- );
- }) as typeof originalReaddir);
+ using rollouts = observeProgressRollouts(workerFirst);
const inventory = (total: number) =>
progressMarker("discovery", 0, total);
const expected: ScanProgress =
@@ -9439,20 +9287,13 @@ if __name__ == "__main__":
filesCompleted: 0,
filesTotal: scenario.endsWith("shrink") ? 8 : 5,
};
- const client = TestClient.withDependencies({
+ await using client = TestClient.withDependencies({
...scanRuntimeDependencies(codexHome, scanDir),
runWorkbench: registerEightFileScan,
createCodex: codexFactory(async () => {
await copyCompletedScan(root);
- parentPath = await writeUsageSession(codexHome, "thread-1", {});
- workerPath = await writeUsageSession(
- codexHome,
- "worker-thread",
- {},
- {
- parent: "thread-1",
- },
- );
+ const { parentPath, workerPath } =
+ await rollouts.writeSessions(codexHome);
const save = (command: string, output: string) =>
appendFile(
parentPath,
@@ -9473,8 +9314,9 @@ if __name__ == "__main__":
]) + "\n",
);
async function* events(): AsyncGenerator<ThreadEvent> {
+ const tracker = rollouts.tracker!;
expect(tracker).toBeDefined();
- await tracker!.stop();
+ await tracker.stop();
if (scenario.startsWith("close")) {
const total = scenario === "close growth" ? 9 : 8;
await save("old-first", inventory(total));
@@ -9511,7 +9353,7 @@ if __name__ == "__main__":
return handle;
},
);
- const pending = tracker!.refresh();
+ const pending = tracker.refresh();
try {
await closing.promise;
yield completedCommand(
@@ -9536,7 +9378,7 @@ if __name__ == "__main__":
? progressMarker("reporting", 3, 8)
: inventory(scenario === "unwritten growth" ? 8 : 9),
);
- if (pollFirst) await tracker!.refresh();
+ if (pollFirst) await tracker.refresh();
// Earlier live callbacks were dropped. This different command
// has not been persisted, even though its output repeats a row.
yield completedCommand("current", current, "live-current");
@@ -9545,9 +9387,9 @@ if __name__ == "__main__":
scenario.includes("changed") ? `${command} displayed` : command;
await save("first", inventory(9));
yield completedCommand(display("first"), inventory(9));
- await tracker!.refresh();
+ await tracker.refresh();
await receipt(workerPath, progressMarker("attack_path", 5, 9));
- await tracker!.refresh();
+ await tracker.refresh();
expect(updates.at(-1)).toEqual({
phase: "discovery",
filesCompleted: 5,
@@ -9563,16 +9405,16 @@ if __name__ == "__main__":
if (batch && pollFirst) {
for (const [command, output] of commands)
await save(command, output);
- await tracker!.refresh();
+ await tracker.refresh();
}
for (const [command, output] of commands) {
if (pollFirst && !batch) {
await save(command, output);
- await tracker!.refresh();
+ await tracker.refresh();
}
yield completedCommand(display(command), output);
if (!pollFirst) await save(command, output);
- if (!batch) await tracker!.refresh();
+ if (!batch) await tracker.refresh();
}
} else {
const middle = scenario === "delayed raw inventory" ? 9 : 8;
@@ -9582,13 +9424,13 @@ if __name__ == "__main__":
if (scenario === "delayed raw inventory")
await receipt(parentPath, inventory(5));
else await save("last", inventory(5));
- if (pollFirst) await tracker!.refresh();
+ if (pollFirst) await tracker.refresh();
yield completedCommand(
"first displayed",
inventory(8),
"live-first",
);
- await tracker!.refresh();
+ await tracker.refresh();
yield completedCommand(
"middle displayed",
inventory(middle),
@@ -9601,8 +9443,8 @@ if __name__ == "__main__":
yield completedCommand("last", inventory(5), "live-last");
}
}
- await tracker!.refresh();
- await tracker!.refresh();
+ await tracker.refresh();
+ await tracker.refresh();
if (scenario !== "unwritten phase")
await receipt(
workerPath,
@@ -9615,54 +9457,34 @@ if __name__ == "__main__":
scenario === "delayed repeated inventory" ? 8 : 9,
),
);
- await tracker!.refresh();
- await tracker!.stop();
+ await tracker.refresh();
+ await tracker.stop();
}
return { events: completedEvents("thread-1", events()) };
}),
});
- try {
- await client.run(repository, {
- onProgress: (update) => updates.push(update),
- });
- expect(updates.at(-1)).toEqual(expected);
- await tracker!.refresh();
- await tracker!.stop();
- expect(updates.at(-1)).toEqual(expected);
- } finally {
- await client.close();
- starting.mockRestore();
- listing.mockRestore();
- }
+ await client.run(repository, {
+ onProgress: (update) => updates.push(update),
+ });
+ expect(updates.at(-1)).toEqual(expected);
+ await rollouts.tracker!.refresh();
+ await rollouts.tracker!.stop();
+ expect(updates.at(-1)).toEqual(expected);
},
);
describe("retained parent source evidence", () => {
- const progress = (
- phase: ScanProgress["phase"],
- filesCompleted: number,
- filesTotal: number,
- ): ScanProgress => ({ phase, filesCompleted, filesTotal });
- type Step =
- | readonly ["save" | "live", string, ScanProgress | string]
- | readonly ["worker", ScanProgress]
- | readonly ["message", "response" | "native", ScanProgress]
- | readonly ["poll"];
- const scenarios: readonly {
- name: string;
- expected: ScanProgress;
- steps: readonly Step[];
- }[] = [
+ const scenarios: readonly ProgressScenario[] = [
{
name: "duplicate inventory projections as evidence only",
expected: progress("validation", 1, 9),
steps: [
- ["message", "response", progress("discovery", 0, 8)],
+ ["save-response", "inventory", progress("discovery", 0, 8)],
["poll"],
["live", "inventory changed", progress("discovery", 0, 6)],
["save", "inventory", progress("discovery", 0, 6)],
["save", "later inventory", progress("discovery", 0, 9)],
- ["message", "native", progress("discovery", 0, 8)],
+ ["save-message", undefined, progress("discovery", 0, 8)],
["save", "review", progress("validation", 1, 6)],
["live", "review", progress("validation", 1, 6)],
["poll"],
@@ -9889,182 +9711,20 @@ if __name__ == "__main__":
),
)(
"retains $name (workers first=$workerFirst, stop=$stopBeforeResolution)",
- async ({ steps, expected, workerFirst, stopBeforeResolution }) => {
- const { root, repository, codexHome, scanDir } =
- await scanDirectories();
- const updates: ScanProgress[] = [];
- let tracker: ScanCostTracker | undefined;
- const originalStart = ScanCostTracker.prototype.start;
- const starting = spyOn(
- ScanCostTracker.prototype,
- "start",
- ).mockImplementation(function (
- this: ScanCostTracker,
- threadId: string,
- ) {
- originalStart.call(this, threadId);
- if (threadId === "thread-1") tracker = this;
- });
- let parentPath = "",
- workerPath = "";
- const originalReaddir = fsPromises.readdir;
- const listing = spyOn(fsPromises, "readdir").mockImplementation((async (
- ...args: Parameters<typeof originalReaddir>
- ) => {
- const entries = await originalReaddir(...args);
- if (String(args[0]) !== dirname(parentPath)) return entries;
- const first = basename(workerFirst ? workerPath : parentPath);
- return entries.toSorted(
- (left, right) =>
- Number(String(right.name) === first) -
- Number(String(left.name) === first),
- );
- }) as typeof originalReaddir);
- const text = (value: string | ScanProgress) =>
- typeof value === "string"
- ? value
- : progressMarker(
- value.phase,
- value.filesCompleted,
- value.filesTotal,
- );
- const client = TestClient.withDependencies({
- ...scanRuntimeDependencies(codexHome, scanDir),
- runWorkbench: registerEightFileScan,
- createCodex: codexFactory(async () => {
- await copyCompletedScan(root);
- parentPath = await writeUsageSession(codexHome, "thread-1", {});
- workerPath = await writeUsageSession(
- codexHome,
- "worker-thread",
- {},
- { parent: "thread-1" },
- );
- async function* events(): AsyncGenerator<ThreadEvent> {
- expect(tracker).toBeDefined();
- await tracker!.stop();
- const anchor = steps.findLastIndex(
- (step) =>
- (step[0] === "save" || step[0] === "live") &&
- step[1] === "anchor",
- );
- const resolution =
- anchor === -1
- ? steps.findLastIndex((step) => step[0] === "save")
- : anchor;
- for (const [index, step] of steps.entries()) {
- if (stopBeforeResolution && index === resolution) {
- await tracker!.stop();
- await tracker!.refresh();
- }
- if (step[0] === "poll") await tracker!.refresh();
- else if (step[0] === "live")
- yield completedCommand(
- step[1],
- text(step[2]),
- `item-${index}`,
- );
- else if (step[0] === "save")
- await appendFile(
- parentPath,
- jsonLines([
- nativeCommand(step[1], text(step[2]), `native-${index}`),
- ]) + "\n",
- );
- else if (step[0] === "message")
- await appendFile(
- parentPath,
- jsonLines([
- step[1] === "native"
- ? {
- type: "event_msg",
- payload: {
- type: "item_completed",
- item: {
- type: "AgentMessage",
- content: [
- { type: "Text", text: text(step[2]) },
- ],
- },
- },
- }
- : {
- type: "response_item",
- payload: {
- type: "message",
- role: "assistant",
- content: [
- { type: "output_text", text: text(step[2]) },
- ],
- },
- },
- ]) + "\n",
- );
- else
- await appendFile(
- workerPath,
- jsonLines([
- {
- type: "response_item",
- payload: {
- type: "custom_tool_call_output",
- call_id: `worker-${index}`,
- output: text(step[1]),
- },
- },
- ]) + "\n",
- );
- }
- await tracker!.refresh();
- await tracker!.refresh();
- await tracker!.stop();
- }
- return { events: completedEvents("thread-1", events()) };
- }),
- });
- try {
- await client.run(repository, {
- onProgress: (update) => updates.push(update),
- });
- expect(updates.at(-1)).toEqual(expected);
- const settled = [...updates];
- await tracker!.refresh();
- await tracker!.stop();
- expect(updates).toEqual(settled);
- } finally {
- await client.close();
- starting.mockRestore();
- listing.mockRestore();
- }
- },
+ (scenario) =>
+ runProgressScenario(scenario, {
+ resolutionIndex: lastAnchorOrSave(scenario.steps),
+ }),
);
});
describe("command proof and inventory admission", () => {
- const progress = (
- phase: ScanProgress["phase"],
- filesCompleted: number,
- filesTotal: number,
- ): ScanProgress => ({ phase, filesCompleted, filesTotal });
const inventory = progress("discovery", 0, 8);
const phase = progress("threat_model", 0, 8);
- const commandProgress = [inventory, phase]
- .map((p) => progressMarker(p.phase, p.filesCompleted, p.filesTotal))
- .join("\n");
- type Step =
- | readonly [
- "save" | "live",
- string,
- ScanProgress | readonly ScanProgress[] | string,
- ]
- | readonly ["worker", ScanProgress]
- | readonly ["poll"];
- const scenarios: readonly {
- name: string;
- expected: ScanProgress;
+ const commandProgress = progressText([inventory, phase]);
+ const scenarios: readonly (ProgressScenario & {
unresolvedExpected?: ScanProgress;
- steps: readonly Step[];
- }[] = [
+ })[] = [
{
name: "a multi-marker command before a smaller inventory",
expected: progress("validation", 1, 5),
@@ -10162,139 +9822,23 @@ if __name__ == "__main__":
),
)(
"reconciles $name (workers first=$workerFirst, stop=$stopBeforeResolution, changed=$changedDisplay)",
- async ({
- steps,
- expected,
- unresolvedExpected,
- workerFirst,
- stopBeforeResolution,
- changedDisplay,
- }) => {
- const { root, repository, codexHome, scanDir } =
- await scanDirectories();
- const updates: ScanProgress[] = [];
- let tracker: ScanCostTracker | undefined;
- const originalStart = ScanCostTracker.prototype.start;
- const starting = spyOn(
- ScanCostTracker.prototype,
- "start",
- ).mockImplementation(function (
- this: ScanCostTracker,
- threadId: string,
- ) {
- originalStart.call(this, threadId);
- if (threadId === "thread-1") tracker = this;
- });
- let parentPath = "",
- workerPath = "";
- const originalReaddir = fsPromises.readdir;
- const listing = spyOn(fsPromises, "readdir").mockImplementation((async (
- ...args: Parameters<typeof originalReaddir>
- ) => {
- const entries = await originalReaddir(...args);
- if (String(args[0]) !== dirname(parentPath)) return entries;
- const first = basename(workerFirst ? workerPath : parentPath);
- return entries.toSorted(
- (left, right) =>
- Number(String(right.name) === first) -
- Number(String(left.name) === first),
- );
- }) as typeof originalReaddir);
- const text = (
- value: string | ScanProgress | readonly ScanProgress[],
- ) =>
- typeof value === "string"
- ? value
- : [value]
- .flat()
- .map((p) =>
- progressMarker(p.phase, p.filesCompleted, p.filesTotal),
+ ({ steps, expected, unresolvedExpected, changedDisplay, ...schedule }) =>
+ runProgressScenario(
+ {
+ ...schedule,
+ steps: changedDisplay
+ ? steps.map((step) =>
+ step[0] === "live" && step[1] !== "anchor"
+ ? (["live", `${step[1]} visible`, step[2]] as const)
+ : step,
)
- .join("\n");
- const client = TestClient.withDependencies({
- ...scanRuntimeDependencies(codexHome, scanDir),
- runWorkbench: registerEightFileScan,
- createCodex: codexFactory(async () => {
- await copyCompletedScan(root);
- parentPath = await writeUsageSession(codexHome, "thread-1", {});
- workerPath = await writeUsageSession(
- codexHome,
- "worker-thread",
- {},
- { parent: "thread-1" },
- );
- async function* events(): AsyncGenerator<ThreadEvent> {
- expect(tracker).toBeDefined();
- await tracker!.stop();
- const anchor = steps.findLastIndex(
- (step) =>
- (step[0] === "save" || step[0] === "live") &&
- step[1] === "anchor",
- );
- const resolution =
- anchor === -1
- ? steps.findLastIndex((step) => step[0] === "save")
- : anchor;
- for (const [index, step] of steps.entries()) {
- if (stopBeforeResolution && index === resolution) {
- await tracker!.stop();
- await tracker!.refresh();
- }
- if (step[0] === "poll") await tracker!.refresh();
- else if (step[0] === "live")
- yield completedCommand(
- changedDisplay && step[1] !== "anchor"
- ? `${step[1]} visible`
- : step[1],
- text(step[2]),
- `item-${index}`,
- );
- else if (step[0] === "save")
- await appendFile(
- parentPath,
- jsonLines([
- nativeCommand(step[1], text(step[2]), `native-${index}`),
- ]) + "\n",
- );
- else
- await appendFile(
- workerPath,
- jsonLines([
- {
- type: "response_item",
- payload: {
- type: "custom_tool_call_output",
- call_id: `worker-${index}`,
- output: text(step[1]),
- },
- },
- ]) + "\n",
- );
- }
- await tracker!.refresh();
- await tracker!.refresh();
- await tracker!.stop();
- }
- return { events: completedEvents("thread-1", events()) };
- }),
- });
- try {
- await client.run(repository, {
- onProgress: (update) => updates.push(update),
- });
- expect(updates.at(-1)).toEqual(
- changedDisplay ? (unresolvedExpected ?? expected) : expected,
- );
- const settled = [...updates];
- await tracker!.refresh();
- await tracker!.stop();
- expect(updates).toEqual(settled);
- } finally {
- await client.close();
- starting.mockRestore();
- listing.mockRestore();
- }
- },
+ : steps,
+ expected: changedDisplay
+ ? (unresolvedExpected ?? expected)
+ : expected,
+ },
+ { resolutionIndex: lastAnchorOrSave(steps) },
+ ),
);
});
@@ -10305,7 +9849,6 @@ if __name__ == "__main__":
| "save"
| "save-quiet"
| "live"
- | "save-message"
| "live-message"
| "save-response"
| "live-changed"
@@ -10313,6 +9856,11 @@ if __name__ == "__main__":
string,
ProgressScenarioValue,
]
+ | readonly [
+ "save-message",
+ label: string | undefined,
+ value: ProgressScenarioValue,
+ ]
| readonly [
"worker" | "additional-worker" | "check" | "scope" | "receipt",
ScanProgress,
@@ -10334,19 +9882,15 @@ if __name__ == "__main__":
type SplitProgressScenario = ProgressScenario & {
partitions?: readonly boolean[];
};
- const progress = (
- phase: ScanProgress["phase"],
- filesCompleted: number,
- filesTotal: number,
- ): ScanProgress => ({ phase, filesCompleted, filesTotal });
- const inventory = (total: number) => progress("discovery", 0, total);
- const progressText = (value: ProgressScenarioValue) =>
- typeof value === "string"
- ? value
- : [value]
- .flat()
- .map((p) => progressMarker(p.phase, p.filesCompleted, p.filesTotal))
- .join("\n");
+ function lastAnchorOrSave(steps: readonly ProgressScenarioStep[]): number {
+ const anchor = steps.findLastIndex(
+ (step) =>
+ (step[0] === "save" || step[0] === "live") && step[1] === "anchor",
+ );
+ return anchor === -1
+ ? steps.findLastIndex((step) => step[0] === "save")
+ : anchor;
+ }
function lastLiveAnchor(steps: readonly ProgressScenarioStep[]): number {
return steps.findLastIndex(
@@ -10356,39 +9900,7 @@ if __name__ == "__main__":
);
}
- async function runProgressScenario(
- {
- steps,
- expected,
- initialScope = 8,
- workerFirst,
- stopBeforeResolution,
- split,
- partition = split ? "each" : "batch",
- }: {
- steps: readonly ProgressScenarioStep[];
- expected: ScanProgress;
- initialScope?: number | null;
- workerFirst: boolean;
- stopBeforeResolution: boolean;
- split?: boolean;
- partition?: "batch" | "staged" | "each";
- },
- {
- resolutionIndex = -1,
- pollReceipts = true,
- }: {
- resolutionIndex?: number;
- pollReceipts?: boolean;
- } = {},
- ): Promise<void> {
- const { root, repository, codexHome, scanDir } = await scanDirectories();
- const updates: ScanProgress[] = [];
- const checkpoints: {
- actual: ScanProgress | undefined;
- expected: ScanProgress;
- scopeOnly?: boolean;
- }[] = [];
+ function observeProgressRollouts(workerFirst: boolean) {
let tracker: ScanCostTracker | undefined;
const originalStart = ScanCostTracker.prototype.start;
const starting = spyOn(
@@ -10398,8 +9910,8 @@ if __name__ == "__main__":
originalStart.call(this, threadId);
if (threadId === "thread-1") tracker = this;
});
- let parentPath = "",
- workerPath = "";
+ let parentPath = "";
+ let workerPath = "";
const originalReaddir = fsPromises.readdir;
const listing = spyOn(fsPromises, "readdir").mockImplementation((async (
...args: Parameters<typeof originalReaddir>
@@ -10413,7 +9925,64 @@ if __name__ == "__main__":
Number(String(left.name) === first),
);
}) as typeof originalReaddir);
- const client = TestClient.withDependencies({
+ return {
+ get tracker() {
+ return tracker;
+ },
+ async writeSessions(codexHome: string) {
+ parentPath = await writeUsageSession(codexHome, "thread-1", {});
+ workerPath = await writeUsageSession(
+ codexHome,
+ "worker-thread",
+ {},
+ {
+ parent: "thread-1",
+ },
+ );
+ return { parentPath, workerPath };
+ },
+ [Symbol.dispose]() {
+ starting.mockRestore();
+ listing.mockRestore();
+ },
+ };
+ }
+
+ async function runProgressScenario(
+ {
+ steps,
+ expected,
+ initialScope = 8,
+ workerFirst,
+ stopBeforeResolution,
+ split,
+ partition = split ? "each" : "batch",
+ }: {
+ steps: readonly ProgressScenarioStep[];
+ expected: ScanProgress;
+ initialScope?: number | null;
+ workerFirst: boolean;
+ stopBeforeResolution?: boolean;
+ split?: boolean;
+ partition?: "batch" | "staged" | "each";
+ },
+ {
+ resolutionIndex = -1,
+ pollReceipts = true,
+ }: {
+ resolutionIndex?: number;
+ pollReceipts?: boolean;
+ } = {},
+ ): Promise<void> {
+ const { root, repository, codexHome, scanDir } = await scanDirectories();
+ const updates: ScanProgress[] = [];
+ const checkpoints: {
+ actual: ScanProgress | undefined;
+ expected: ScanProgress;
+ scopeOnly?: boolean;
+ }[] = [];
+ using rollouts = observeProgressRollouts(workerFirst);
+ await using client = TestClient.withDependencies({
...scanRuntimeDependencies(codexHome, scanDir),
runWorkbench: async (
_options: unknown,
@@ -10428,35 +9997,31 @@ if __name__ == "__main__":
: mockWorkbench(args, input),
createCodex: codexFactory(async () => {
await copyCompletedScan(root);
- parentPath = await writeUsageSession(codexHome, "thread-1", {});
- workerPath = await writeUsageSession(
- codexHome,
- "worker-thread",
- {},
- { parent: "thread-1" },
- );
+ const { parentPath, workerPath } =
+ await rollouts.writeSessions(codexHome);
async function* events(): AsyncGenerator<ThreadEvent> {
+ const tracker = rollouts.tracker!;
expect(tracker).toBeDefined();
- await tracker!.stop();
+ await tracker.stop();
for (const [index, step] of steps.entries()) {
if (stopBeforeResolution && index === resolutionIndex) {
- await tracker!.stop();
- await tracker!.refresh();
+ await tracker.stop();
+ await tracker.refresh();
}
if (step[0] === "begin") {
- tracker!.beginParentTurn(true);
+ tracker.beginParentTurn(true);
continue;
}
if (step[0] === "resolve") {
if (stopBeforeResolution) {
- await tracker!.stop();
- await tracker!.refresh();
+ await tracker.stop();
+ await tracker.refresh();
}
continue;
}
if (step[0] === "boundary") {
- if (partition !== "batch") await tracker!.refresh();
- } else if (step[0] === "poll") await tracker!.refresh();
+ if (partition !== "batch") await tracker.refresh();
+ } else if (step[0] === "poll") await tracker.refresh();
else if (step[0] === "check" || step[0] === "scope")
checkpoints.push({
actual: updates.at(-1),
@@ -10504,7 +10069,8 @@ if __name__ == "__main__":
type: "item_completed",
item: {
type: "AgentMessage",
- id: `native-${index}`,
+ id:
+ step[1] === undefined ? undefined : `native-${index}`,
content: [
{ type: "Text", text: progressText(step[2]) },
],
@@ -10559,36 +10125,30 @@ if __name__ == "__main__":
(step[0].startsWith("save") ||
(pollReceipts && step[0] === "receipt"))
)
- await tracker!.refresh();
+ await tracker.refresh();
}
- await tracker!.refresh();
- await tracker!.refresh();
- await tracker!.stop();
+ await tracker.refresh();
+ await tracker.refresh();
+ await tracker.stop();
}
return { events: completedEvents("thread-1", events()) };
}),
});
- try {
- await client.run(repository, {
- onProgress: (update) => updates.push(update),
- });
- expect(updates.at(-1)).toEqual(expected);
- for (const checkpoint of checkpoints)
- if (checkpoint.scopeOnly)
- expect(checkpoint.actual).toMatchObject({
- filesCompleted: checkpoint.expected.filesCompleted,
- filesTotal: checkpoint.expected.filesTotal,
- });
- else expect(checkpoint.actual).toEqual(checkpoint.expected);
- const settled = [...updates];
- await tracker!.refresh();
- await tracker!.stop();
- expect(updates).toEqual(settled);
- } finally {
- await client.close();
- starting.mockRestore();
- listing.mockRestore();
- }
+ await client.run(repository, {
+ onProgress: (update) => updates.push(update),
+ });
+ expect(updates.at(-1)).toEqual(expected);
+ for (const checkpoint of checkpoints)
+ if (checkpoint.scopeOnly)
+ expect(checkpoint.actual).toMatchObject({
+ filesCompleted: checkpoint.expected.filesCompleted,
+ filesTotal: checkpoint.expected.filesTotal,
+ });
+ else expect(checkpoint.actual).toEqual(checkpoint.expected);
+ const settled = [...updates];
+ await rollouts.tracker!.refresh();
+ await rollouts.tracker!.stop();
+ expect(updates).toEqual(settled);
}
describe("displayed receipt eligibility after inventory recovery", () => {
@@ -11265,53 +10825,74 @@ if __name__ == "__main__":
);
});
- describe("review eligibility and ordered projections", () => {
- const scenarios: SplitProgressScenario[] = [];
- const add = (
- name: string,
- state: ScanProgress,
- steps: ProgressScenarioStep[],
- partitions?: readonly boolean[],
- ) => {
- const next =
- state.filesCompleted < state.filesTotal
- ? progress("reporting", state.filesCompleted + 1, state.filesTotal)
- : progress("validation", state.filesCompleted, state.filesTotal);
- const expected =
- next.filesCompleted === next.filesTotal
- ? next
- : progress(next.phase, state.filesTotal, state.filesTotal);
- scenarios.push({
- name,
- expected,
- partitions,
- steps: [
- ...steps,
- ["poll"],
- ["check", state],
- [
- "worker",
- progress("attack_path", state.filesTotal + 1, state.filesTotal + 2),
- ],
- ["poll"],
- ["check", state],
- ["save", "followup", next],
- ["poll"],
- ["check", next],
- [
- "additional-worker",
- progress("attack_path", state.filesTotal, state.filesTotal),
- ],
- ["poll"],
- ["check", expected],
+ function progressWithWorkerFollowup(
+ name: string,
+ state: ScanProgress,
+ steps: readonly ProgressScenarioStep[],
+ partitions?: readonly boolean[],
+ ): SplitProgressScenario {
+ const next =
+ state.filesCompleted < state.filesTotal
+ ? progress("reporting", state.filesCompleted + 1, state.filesTotal)
+ : progress("validation", state.filesCompleted, state.filesTotal);
+ const expected =
+ next.filesCompleted === next.filesTotal
+ ? next
+ : progress(next.phase, state.filesTotal, state.filesTotal);
+ return {
+ name,
+ expected,
+ partitions,
+ steps: [
+ ...steps,
+ ["poll"],
+ ["check", state],
+ [
+ "worker",
+ progress("attack_path", state.filesTotal + 1, state.filesTotal + 2),
],
- });
+ ["poll"],
+ ["check", state],
+ ["save", "followup", next],
+ ["poll"],
+ ["check", next],
+ [
+ "additional-worker",
+ progress("attack_path", state.filesTotal, state.filesTotal),
+ ],
+ ["poll"],
+ ["check", expected],
+ ],
};
+ }
+
+ function testProgressSchedules(scenarios: readonly SplitProgressScenario[]) {
+ test.each(
+ scenarios.flatMap((scenario) =>
+ [false, true].flatMap((workerFirst) =>
+ (scenario.partitions ?? [false, true]).flatMap((split) =>
+ [false, true].map((stopBeforeResolution) => ({
+ ...scenario,
+ workerFirst,
+ split,
+ stopBeforeResolution,
+ })),
+ ),
+ ),
+ ),
+ )(
+ "retains $name (workers first=$workerFirst, stop=$stopBeforeResolution, split=$split)",
+ (scenario) => runProgressScenario(scenario),
+ );
+ }
+
+ describe("review eligibility and ordered projections", () => {
+ const scenarios: SplitProgressScenario[] = [];
for (const mode of ["exact", "changed", "after inventory"] as const) {
for (const eligible of [false, true]) {
const initialTotal = eligible ? 7 : 6;
const review = progress("discovery", 1, 7);
- add(
+ const scenario = progressWithWorkerFollowup(
`accepted review after scope correction (mode=${mode}, eligible=${eligible})`,
eligible ? review : progress("threat_model", 9, 10),
[
@@ -11341,6 +10922,7 @@ if __name__ == "__main__":
["save", "valid", progress("threat_model", 9, 10)],
],
);
+ scenarios.push(scenario);
}
}
for (const mode of [
@@ -11350,7 +10932,7 @@ if __name__ == "__main__":
"review exact",
] as const) {
const live = mode === "exact" ? "live" : "live-changed";
- add(
+ const scenario = progressWithWorkerFollowup(
`repeated inventories around review (mode=${mode})`,
progress("reporting", 3, 8),
[
@@ -11378,71 +10960,16 @@ if __name__ == "__main__":
["worker", progress("attack_path", 6, 9)],
],
);
+ scenarios.push(scenario);
}
- test.each(
- scenarios.flatMap((scenario) =>
- [false, true].flatMap((workerFirst) =>
- (scenario.partitions ?? [false, true]).flatMap((split) =>
- [false, true].map((stopBeforeResolution) => ({
- ...scenario,
- workerFirst,
- split,
- stopBeforeResolution,
- })),
- ),
- ),
- ),
- )(
- "retains $name (workers first=$workerFirst, stop=$stopBeforeResolution, split=$split)",
- (scenario) => runProgressScenario(scenario),
- );
+ testProgressSchedules(scenarios);
});
describe("inventory authority before scope projection", () => {
const scenarios: SplitProgressScenario[] = [];
- const add = (
- name: string,
- state: ScanProgress,
- steps: ProgressScenarioStep[],
- partitions?: readonly boolean[],
- ) => {
- const next =
- state.filesCompleted < state.filesTotal
- ? progress("reporting", state.filesCompleted + 1, state.filesTotal)
- : progress("validation", state.filesCompleted, state.filesTotal);
- const expected =
- next.filesCompleted === next.filesTotal
- ? next
- : progress(next.phase, state.filesTotal, state.filesTotal);
- scenarios.push({
- name,
- expected,
- partitions,
- steps: [
- ...steps,
- ["poll"],
- ["check", state],
- [
- "worker",
- progress("attack_path", state.filesTotal + 1, state.filesTotal + 2),
- ],
- ["poll"],
- ["check", state],
- ["save", "followup", next],
- ["poll"],
- ["check", next],
- [
- "additional-worker",
- progress("attack_path", state.filesTotal, state.filesTotal),
- ],
- ["poll"],
- ["check", expected],
- ],
- });
- };
for (const persisted of ["late", "early", "absent"] as const) {
for (const phase of ["reporting", "attack_path"] as const) {
- add(
+ const scenario = progressWithWorkerFollowup(
`saved authority before fallback (persisted=${persisted}, phase=${phase})`,
progress("validation", 3, 5),
[
@@ -11469,6 +10996,7 @@ if __name__ == "__main__":
: []),
],
);
+ scenarios.push(scenario);
}
}
for (const mode of [
@@ -11477,7 +11005,7 @@ if __name__ == "__main__":
"review delivered",
"prior inventory",
] as const) {
- add(
+ const scenario = progressWithWorkerFollowup(
`bounded inventory after review (mode=${mode})`,
mode === "prior inventory"
? progress("validation", 3, 5)
@@ -11503,68 +11031,13 @@ if __name__ == "__main__":
["worker", progress("attack_path", 7, 8)],
],
);
+ scenarios.push(scenario);
}
- test.each(
- scenarios.flatMap((scenario) =>
- [false, true].flatMap((workerFirst) =>
- (scenario.partitions ?? [false, true]).flatMap((split) =>
- [false, true].map((stopBeforeResolution) => ({
- ...scenario,
- workerFirst,
- split,
- stopBeforeResolution,
- })),
- ),
- ),
- ),
- )(
- "retains $name (workers first=$workerFirst, stop=$stopBeforeResolution, split=$split)",
- (scenario) => runProgressScenario(scenario),
- );
+ testProgressSchedules(scenarios);
});
describe("inventory proof across completion intervals", () => {
const scenarios: SplitProgressScenario[] = [];
- const add = (
- name: string,
- state: ScanProgress,
- steps: ProgressScenarioStep[],
- partitions?: readonly boolean[],
- ) => {
- const next =
- state.filesCompleted < state.filesTotal
- ? progress("reporting", state.filesCompleted + 1, state.filesTotal)
- : progress("validation", state.filesCompleted, state.filesTotal);
- const expected =
- next.filesCompleted === next.filesTotal
- ? next
- : progress(next.phase, state.filesTotal, state.filesTotal);
- scenarios.push({
- name,
- expected,
- partitions,
- steps: [
- ...steps,
- ["poll"],
- ["check", state],
- [
- "worker",
- progress("attack_path", state.filesTotal + 1, state.filesTotal + 2),
- ],
- ["poll"],
- ["check", state],
- ["save", "followup", next],
- ["poll"],
- ["check", next],
- [
- "additional-worker",
- progress("attack_path", state.filesTotal, state.filesTotal),
- ],
- ["poll"],
- ["check", expected],
- ],
- });
- };
for (const total of [9, 10]) {
for (const tail of ["same", "shrink", "none"] as const) {
for (const exact of [false, true]) {
@@ -11576,7 +11049,7 @@ if __name__ == "__main__":
? []
: [inventory(tail === "same" ? total : 5)]),
];
- add(
+ const scenario = progressWithWorkerFollowup(
`eligible inventory before trailing zero (total=${total}, tail=${tail}, exact=${exact})`,
progress("reporting", reviewed + 1, total),
[
@@ -11594,13 +11067,14 @@ if __name__ == "__main__":
["save", "report", progress("reporting", reviewed + 1, total)],
],
);
+ scenarios.push(scenario);
}
}
}
for (const exact of [false, true]) {
for (const intermediate of ["complete", "omitted"] as const) {
const review = progress("validation", 1, 10);
- add(
+ const scenario = progressWithWorkerFollowup(
`intermediate completion bound (exact=${exact}, review=${intermediate})`,
progress("reporting", 3, 10),
[
@@ -11638,13 +11112,14 @@ if __name__ == "__main__":
: []),
],
);
+ scenarios.push(scenario);
}
}
for (const exact of [false, true]) {
for (const zero of [false, true]) {
for (const eligible of [false, true]) {
const review = progress("validation", 1, eligible ? 5 : 8);
- add(
+ const scenario = progressWithWorkerFollowup(
`projection witness across zero phase (exact=${exact}, zero=${zero}, eligible=${eligible})`,
eligible
? review
@@ -11663,73 +11138,18 @@ if __name__ == "__main__":
["live", "anchor", "done"],
],
);
+ scenarios.push(scenario);
}
}
}
- test.each(
- scenarios.flatMap((scenario) =>
- [false, true].flatMap((workerFirst) =>
- (scenario.partitions ?? [false, true]).flatMap((split) =>
- [false, true].map((stopBeforeResolution) => ({
- ...scenario,
- workerFirst,
- split,
- stopBeforeResolution,
- })),
- ),
- ),
- ),
- )(
- "retains $name (workers first=$workerFirst, stop=$stopBeforeResolution, split=$split)",
- (scenario) => runProgressScenario(scenario),
- );
+ testProgressSchedules(scenarios);
});
describe("bounded inventory marker evidence", () => {
const scenarios: SplitProgressScenario[] = [];
- const add = (
- name: string,
- state: ScanProgress,
- steps: ProgressScenarioStep[],
- partitions?: readonly boolean[],
- ) => {
- const next =
- state.filesCompleted < state.filesTotal
- ? progress("reporting", state.filesCompleted + 1, state.filesTotal)
- : progress("validation", state.filesCompleted, state.filesTotal);
- const expected =
- next.filesCompleted === next.filesTotal
- ? next
- : progress(next.phase, state.filesTotal, state.filesTotal);
- scenarios.push({
- name,
- expected,
- partitions,
- steps: [
- ...steps,
- ["poll"],
- ["check", state],
- [
- "worker",
- progress("attack_path", state.filesTotal + 1, state.filesTotal + 2),
- ],
- ["poll"],
- ["check", state],
- ["save", "followup", next],
- ["poll"],
- ["check", next],
- [
- "additional-worker",
- progress("attack_path", state.filesTotal, state.filesTotal),
- ],
- ["poll"],
- ["check", expected],
- ],
- });
- };
for (const command of ["changed", "exact"] as const) {
const live = command === "changed" ? "live-changed" : "live";
- add(
+ const scenario = progressWithWorkerFollowup(
`bounded inventory after saved review (${command})`,
progress(command === "exact" ? "preflight" : "validation", 7, 8),
[
@@ -11749,11 +11169,12 @@ if __name__ == "__main__":
["live", "anchor", "done"],
],
);
+ scenarios.push(scenario);
}
for (const total of [9, 5]) {
for (const delayed of [true, false]) {
const markers = [progress("threat_model", 0, 8), inventory(total)];
- add(
+ const scenario = progressWithWorkerFollowup(
`scope-setting phase (total=${total}, delayed=${delayed})`,
inventory(total),
[
@@ -11772,6 +11193,7 @@ if __name__ == "__main__":
["live", "current", markers],
],
);
+ scenarios.push(scenario);
}
const messageTotal = total === 9 ? 10 : total;
for (const kind of [
@@ -11779,7 +11201,7 @@ if __name__ == "__main__":
"persisted message",
"command",
] as const) {
- add(
+ const scenario = progressWithWorkerFollowup(
`bounded inventory evidence (${kind}, total=${messageTotal})`,
progress("validation", messageTotal - 1, messageTotal),
[
@@ -11804,6 +11226,7 @@ if __name__ == "__main__":
],
],
);
+ scenarios.push(scenario);
}
}
for (const command of ["changed", "exact"] as const) {
@@ -11812,7 +11235,7 @@ if __name__ == "__main__":
? [inventory(9), progress("discovery", 1, 9)]
: [progress("discovery", 1, 8), inventory(9)];
const total = before ? 9 : 8;
- add(
+ const scenario = progressWithWorkerFollowup(
`omitted command marker order (${command}, inventory before=${before})`,
progress(before ? "reporting" : "validation", before ? 4 : 3, total),
[
@@ -11830,11 +11253,12 @@ if __name__ == "__main__":
["save", "report", progress("reporting", 4, 9)],
],
);
+ scenarios.push(scenario);
}
}
for (const kind of ["message", "command"] as const) {
const live = kind === "message" ? "live-message" : "live-changed";
- add(
+ const scenario = progressWithWorkerFollowup(
`separate inventory after omitted review (${kind})`,
progress("validation", 3, 8),
[
@@ -11857,71 +11281,16 @@ if __name__ == "__main__":
["save", "report", progress("reporting", 4, 9)],
],
);
+ scenarios.push(scenario);
}
- test.each(
- scenarios.flatMap((scenario) =>
- [false, true].flatMap((workerFirst) =>
- (scenario.partitions ?? [false, true]).flatMap((split) =>
- [false, true].map((stopBeforeResolution) => ({
- ...scenario,
- workerFirst,
- split,
- stopBeforeResolution,
- })),
- ),
- ),
- ),
- )(
- "retains $name (workers first=$workerFirst, stop=$stopBeforeResolution, split=$split)",
- (scenario) => runProgressScenario(scenario),
- );
+ testProgressSchedules(scenarios);
});
describe("inventory occurrence and cutoff revalidation", () => {
const scenarios: SplitProgressScenario[] = [];
- const add = (
- name: string,
- state: ScanProgress,
- steps: ProgressScenarioStep[],
- partitions?: readonly boolean[],
- ) => {
- const next =
- state.filesCompleted < state.filesTotal
- ? progress("reporting", state.filesCompleted + 1, state.filesTotal)
- : progress("validation", state.filesCompleted, state.filesTotal);
- const expected =
- next.filesCompleted === next.filesTotal
- ? next
- : progress(next.phase, state.filesTotal, state.filesTotal);
- scenarios.push({
- name,
- expected,
- partitions,
- steps: [
- ...steps,
- ["poll"],
- ["check", state],
- [
- "worker",
- progress("attack_path", state.filesTotal + 1, state.filesTotal + 2),
- ],
- ["poll"],
- ["check", state],
- ["save", "followup", next],
- ["poll"],
- ["check", next],
- [
- "additional-worker",
- progress("attack_path", state.filesTotal, state.filesTotal),
- ],
- ["poll"],
- ["check", expected],
- ],
- });
- };
for (const control of ["dropped", "delivered", "eligible"] as const) {
const review = progress("validation", 3, control === "eligible" ? 8 : 10);
- add(
+ const scenario = progressWithWorkerFollowup(
`repeated native inventory (${control})`,
control === "eligible" ? review : progress("reporting", 8, 10),
[
@@ -11944,9 +11313,10 @@ if __name__ == "__main__":
["save", "final-report", progress("reporting", 8, 10)],
],
);
+ scenarios.push(scenario);
}
for (const control of ["staged", "exact", "single batch"] as const) {
- add(
+ const scenario = progressWithWorkerFollowup(
`review cutoff after stronger inventory proof (${control})`,
progress("reporting", 4, 4),
[
@@ -11972,6 +11342,7 @@ if __name__ == "__main__":
],
control === "single batch" ? [false] : undefined,
);
+ scenarios.push(scenario);
}
for (const control of [
"delayed anchor",
@@ -11979,7 +11350,7 @@ if __name__ == "__main__":
"eligible",
] as const) {
const review = progress("discovery", 3, control === "eligible" ? 5 : 7);
- add(
+ const scenario = progressWithWorkerFollowup(
`omitted inventory changes review eligibility (${control})`,
progress("reporting", 4, control === "eligible" ? 5 : 4),
[
@@ -12003,56 +11374,19 @@ if __name__ == "__main__":
["live", "anchor", "finished"],
],
);
+ scenarios.push(scenario);
}
- test.each(
- scenarios.flatMap((scenario) =>
- [false, true].flatMap((workerFirst) =>
- (scenario.partitions ?? [false, true]).flatMap((split) =>
- [false, true].map((stopBeforeResolution) => ({
- ...scenario,
- workerFirst,
- split,
- stopBeforeResolution,
- })),
- ),
- ),
- ),
- )(
- "retains $name (workers first=$workerFirst, stop=$stopBeforeResolution, split=$split)",
- (scenario) => runProgressScenario(scenario),
- );
+ testProgressSchedules(scenarios);
});
describe("inventory cutoffs from retained source proof", () => {
const scenarios: ProgressScenario[] = [];
for (const control of [false, true]) {
- const finish = (
- state: ScanProgress,
- next: ScanProgress,
- ): ProgressScenarioStep[] => [
- ["poll"],
- ["check", state],
- [
- "worker",
- progress("attack_path", state.filesTotal + 1, state.filesTotal + 2),
- ],
- ["poll"],
- ["check", state],
- ["save", "followup", next],
- ["poll"],
- ["check", next],
- [
- "additional-worker",
- progress("attack_path", state.filesTotal, state.filesTotal),
- ],
- ["poll"],
- ["check", progress(next.phase, state.filesTotal, state.filesTotal)],
- ];
scenarios.push(
- {
- name: `restore scope after rejecting a provisional inventory (saved baseline=${control})`,
- expected: progress("reporting", 8, 8),
- steps: [
+ progressWithWorkerFollowup(
+ `restore scope after rejecting a provisional inventory (saved baseline=${control})`,
+ progress("validation", 3, 8),
+ [
...(control
? ([
["save", "initial", inventory(8)],
@@ -12067,16 +11401,12 @@ if __name__ == "__main__":
["poll"],
["resolve"],
["save", "inventory", inventory(9)],
- ...finish(
- progress("validation", 3, 8),
- progress("reporting", 4, 8),
- ),
],
- },
- {
- name: `copy retains admitted source scope (native copy=${control})`,
- expected: progress("reporting", 6, 6),
- steps: [
+ ),
+ progressWithWorkerFollowup(
+ `copy retains admitted source scope (native copy=${control})`,
+ progress("reporting", 2, 6),
+ [
["receipt", inventory(5)],
["live-message", "inventory", inventory(6)],
["live", "review", progress("discovery", 1, 6)],
@@ -12088,13 +11418,12 @@ if __name__ == "__main__":
],
["poll"],
["receipt", progress("reporting", 2, 6)],
- ...finish(progress("reporting", 2, 6), progress("reporting", 3, 6)),
],
- },
- {
- name: `later live inventory cannot rescope native history (persisted early=${control})`,
- expected: progress("reporting", 8, 8),
- steps: [
+ ),
+ progressWithWorkerFollowup(
+ `later live inventory cannot rescope native history (persisted early=${control})`,
+ progress("threat_model", 5, 8),
+ [
["save", "oversized", progress("validation", 9, 10)],
["live", "oversized", progress("validation", 9, 10)],
["poll"],
@@ -12124,16 +11453,12 @@ if __name__ == "__main__":
["live", "phase", progress("discovery", 3, 8)],
["save", "current", inventory(8)],
["save", "phase", progress("discovery", 3, 8)],
- ...finish(
- progress("threat_model", 5, 8),
- progress("reporting", 6, 8),
- ),
],
- },
- {
- name: `bounded omitted review closes inventory admission (exact review=${control})`,
- expected: progress("reporting", 5, 5),
- steps: [
+ ),
+ progressWithWorkerFollowup(
+ `bounded omitted review closes inventory admission (exact review=${control})`,
+ progress("validation", 3, 5),
+ [
["save", "initial", inventory(5)],
["live", "initial", inventory(5)],
["poll"],
@@ -12146,16 +11471,12 @@ if __name__ == "__main__":
["save", "review", persistedCommandOutput],
["resolve"],
["save", "later", [inventory(8), progress("reporting", 3, 8)]],
- ...finish(
- progress("validation", 3, 5),
- progress("reporting", 4, 5),
- ),
],
- },
- {
- name: `inventory after a known review is excluded through bounds (exact inventory=${control})`,
- expected: progress("reporting", 8, 8),
- steps: [
+ ),
+ progressWithWorkerFollowup(
+ `inventory after a known review is excluded through bounds (exact inventory=${control})`,
+ progress("attack_path", 6, 8),
+ [
["save", "initial", inventory(8)],
["live", "initial", inventory(8)],
["poll"],
@@ -12169,16 +11490,12 @@ if __name__ == "__main__":
["save", "after", "after"],
["resolve"],
["live", "after", "after"],
- ...finish(
- progress("attack_path", 6, 8),
- progress("reporting", 7, 8),
- ),
],
- },
- {
- name: `earlier eligible review precedes provisional scope (review polled early=${control})`,
- expected: progress("reporting", 8, 8),
- steps: [
+ ),
+ progressWithWorkerFollowup(
+ `earlier eligible review precedes provisional scope (review polled early=${control})`,
+ progress("validation", 3, 8),
+ [
["save", "initial", inventory(8)],
["poll"],
["worker", progress("attack_path", 2, 5)],
@@ -12199,31 +11516,11 @@ if __name__ == "__main__":
: []),
["resolve"],
["save", "inventory", inventory(6)],
- ...finish(
- progress("validation", 3, 8),
- progress("reporting", 4, 8),
- ),
],
- },
+ ),
);
}
- test.each(
- scenarios.flatMap((scenario) =>
- [false, true].flatMap((workerFirst) =>
- [false, true].flatMap((split) =>
- [false, true].map((stopBeforeResolution) => ({
- ...scenario,
- workerFirst,
- split,
- stopBeforeResolution,
- })),
- ),
- ),
- ),
- )(
- "retains $name (workers first=$workerFirst, stop=$stopBeforeResolution, split=$split)",
- (scenario) => runProgressScenario(scenario),
- );
+ testProgressSchedules(scenarios);
});
describe("source proof before inventory admission", () => {
@@ -13066,18 +12363,9 @@ if __name__ == "__main__":
: step,
)
: originalSteps.filter((step) => step[1] !== control);
- const anchor = steps.findLastIndex(
- (step) =>
- (step[0] === "save" || step[0] === "live") && step[1] === "anchor",
- );
return runProgressScenario(
{ steps, expected, workerFirst, stopBeforeResolution },
- {
- resolutionIndex:
- anchor === -1
- ? steps.findLastIndex((step) => step[0] === "save")
- : anchor,
- },
+ { resolutionIndex: lastAnchorOrSave(steps) },
);
},
);
@@ -13107,30 +12395,7 @@ if __name__ == "__main__":
async ({ scenario, changedDisplay, workerFirst }) => {
const { root, repository, codexHome, scanDir } = await scanDirectories();
const updates: ScanProgress[] = [];
- let tracker: ScanCostTracker | undefined;
- const originalStart = ScanCostTracker.prototype.start;
- const starting = spyOn(
- ScanCostTracker.prototype,
- "start",
- ).mockImplementation(function (this: ScanCostTracker, threadId: string) {
- originalStart.call(this, threadId);
- if (threadId === "thread-1") tracker = this;
- });
- let parentPath = "";
- let workerPath = "";
- const originalReaddir = fsPromises.readdir;
- const listing = spyOn(fsPromises, "readdir").mockImplementation((async (
- ...args: Parameters<typeof originalReaddir>
- ) => {
- const entries = await originalReaddir(...args);
- if (String(args[0]) !== dirname(parentPath)) return entries;
- const first = basename(workerFirst ? workerPath : parentPath);
- return entries.toSorted(
- (left, right) =>
- Number(String(right.name) === first) -
- Number(String(left.name) === first),
- );
- }) as typeof originalReaddir);
+ using rollouts = observeProgressRollouts(workerFirst);
const initial: ScanProgress = scenario.endsWith("phase")
? { phase: "validation", filesCompleted: 3, filesTotal: 8 }
: {
@@ -13152,28 +12417,24 @@ if __name__ == "__main__":
);
const display = (command: string) =>
changedDisplay ? `${command} displayed` : command;
- const client = TestClient.withDependencies({
+ await using client = TestClient.withDependencies({
...scanRuntimeDependencies(codexHome, scanDir),
runWorkbench: registerEightFileScan,
createCodex: codexFactory(async () => {
await copyCompletedScan(root);
- parentPath = await writeUsageSession(codexHome, "thread-1", {});
- workerPath = await writeUsageSession(
- codexHome,
- "worker-thread",
- {},
- { parent: "thread-1" },
- );
+ const { parentPath, workerPath } =
+ await rollouts.writeSessions(codexHome);
const save = (command: string, output: string) =>
appendFile(
parentPath,
jsonLines([nativeCommand(command, output, command)]) + "\n",
);
async function* events(): AsyncGenerator<ThreadEvent> {
+ const tracker = rollouts.tracker!;
expect(tracker).toBeDefined();
// Control polling through the real tracker while keeping the API's
// source, inventory, and count admission callback in the loop.
- await tracker!.stop();
+ await tracker.stop();
if (scenario.startsWith("pending")) {
yield completedCommand(
display("first"),
@@ -13181,20 +12442,20 @@ if __name__ == "__main__":
"live-first",
);
await save("first", text(initial));
- await tracker!.refresh();
+ await tracker.refresh();
yield completedCommand(
display("last"),
text(initial),
"live-last",
);
- await tracker!.refresh();
+ await tracker.refresh();
await save(
"middle",
scenario === "pending phase"
? progressMarker("reporting", 3, 8)
: progressMarker("discovery", 0, 9),
);
- await tracker!.refresh();
+ await tracker.refresh();
await save("last", text(initial));
if (scenario === "pending phase") {
await save("anchor", "Batch finished.");
@@ -13204,7 +12465,7 @@ if __name__ == "__main__":
"live-anchor",
);
}
- await tracker!.refresh();
+ await tracker.refresh();
} else {
await save(
"initial",
@@ -13213,7 +12474,7 @@ if __name__ == "__main__":
: text(initial),
);
await save("anchor", "Batch finished.");
- await tracker!.refresh();
+ await tracker.refresh();
yield completedCommand(
display("initial"),
scenario.startsWith("omitted")
@@ -13242,26 +12503,20 @@ if __name__ == "__main__":
},
]) + "\n",
);
- await tracker!.refresh();
- await tracker!.refresh();
- await tracker!.stop();
+ await tracker.refresh();
+ await tracker.refresh();
+ await tracker.stop();
}
return { events: completedEvents("thread-1", events()) };
}),
});
- try {
- await client.run(repository, {
- onProgress: (update) => updates.push(update),
- });
- expect(updates.at(-1)).toEqual(expected);
- await tracker!.refresh();
- await tracker!.stop();
- expect(updates.at(-1)).toEqual(expected);
- } finally {
- await client.close();
- starting.mockRestore();
- listing.mockRestore();
- }
+ await client.run(repository, {
+ onProgress: (update) => updates.push(update),
+ });
+ expect(updates.at(-1)).toEqual(expected);
+ await rollouts.tracker!.refresh();
+ await rollouts.tracker!.stop();
+ expect(updates.at(-1)).toEqual(expected);
},
);
diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts
index 7a66bc869..38f179b68 100644
--- a/sdk/typescript/tests-ts/external-findings-publish.test.ts
+++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts
@@ -17,6 +17,7 @@ import {
} from "../src/snyk-findings.js";
import {
semgrepJson,
+ semgrepPercentPathReports,
semgrepSarif,
} from "./support/semgrep-import-fixtures.js";
import { endorFinding, endorSarif } from "./support/endor-import-fixtures.js";
@@ -9293,23 +9294,7 @@ test("OpenGrep CLI Cloud preview forwards source provenance without uploading",
});
test("Semgrep native percent paths retain CE records through JSON-to-SARIF publication", async () => {
- const paths = [
- "src/my file.ts",
- "src/my%20file.ts",
- "src/100%.ts",
- "src/100%25.ts",
- ];
- const json: any = semgrepJson();
- const sarif: any = semgrepSarif();
- delete json.results[0].extra.fingerprint;
- delete sarif.runs[0].results[0].fingerprints;
- json.results = paths.map((path) => ({ ...json.results[0], path }));
- const run = sarif.runs[0];
- run.results = paths.map((path) => {
- const result = structuredClone(run.results[0]);
- result.locations[0].physicalLocation.artifactLocation.uri = path;
- return result;
- });
+ const { paths, json, sarif } = semgrepPercentPathReports();
const f = await fixture(json);
const settings = {
...options,
diff --git a/sdk/typescript/tests-ts/semgrep-findings.test.ts b/sdk/typescript/tests-ts/semgrep-findings.test.ts
index f4f3af57a..fcfa5fbdc 100644
--- a/sdk/typescript/tests-ts/semgrep-findings.test.ts
+++ b/sdk/typescript/tests-ts/semgrep-findings.test.ts
@@ -5,6 +5,7 @@ import {
} from "../src/index.js";
import {
semgrepJson,
+ semgrepPercentPathReports,
semgrepSarif,
} from "./support/semgrep-import-fixtures.js";
@@ -420,23 +421,8 @@ test.each([true, false])(
);
test("native SARIF keeps CE percent paths distinct after reordering", () => {
- const paths = [
- "src/my file.ts",
- "src/my%20file.ts",
- "src/100%.ts",
- "src/100%25.ts",
- ];
- const json: any = semgrepJson();
- const sarif: any = semgrepSarif();
- delete json.results[0].extra.fingerprint;
- delete sarif.runs[0].results[0].fingerprints;
- json.results = paths.map((path) => ({ ...json.results[0], path }));
- const run = sarif.runs[0];
- run.results = paths.map((path) => {
- const result = structuredClone(run.results[0]);
- result.locations[0].physicalLocation.artifactLocation.uri = path;
- return result;
- });
+ const { paths, json, sarif } = semgrepPercentPathReports();
+ const run = sarif.runs[0]!;
const expected = normalizeSemgrepFindings(json).findings;
expect(expected).toHaveLength(paths.length);
for (const results of [run.results, run.results.toReversed()]) {
diff --git a/sdk/typescript/tests-ts/support/api-client.ts b/sdk/typescript/tests-ts/support/api-client.ts
index 5bd30fcbc..30586a6ad 100644
--- a/sdk/typescript/tests-ts/support/api-client.ts
+++ b/sdk/typescript/tests-ts/support/api-client.ts
@@ -1,10 +1,39 @@
import { CodexSecurity } from "../../src/api.js";
-import type { JsonObject } from "../../src/config.js";
+import {
+ CHEAPERINFERENCE_CODEX_PROVIDER,
+ FIREWORKS_CODEX_PROVIDER,
+ OPENROUTER_CODEX_PROVIDER,
+ type JsonObject,
+} from "../../src/config.js";
import { throwing } from "./errors.js";
import { readFile } from "node:fs/promises";
import { parse } from "smol-toml";
import type { TurnOptions } from "@openai/codex-sdk";
+export const EXTERNAL_PROVIDER_CASES = [
+ [
+ "OpenRouter",
+ "openrouter",
+ "OPENROUTER_API_KEY",
+ "anthropic/claude-sonnet-4.5",
+ OPENROUTER_CODEX_PROVIDER,
+ ],
+ [
+ "Fireworks AI",
+ "fireworks",
+ "FIREWORKS_API_KEY",
+ "accounts/fireworks/models/qwen3-235b-a22b",
+ FIREWORKS_CODEX_PROVIDER,
+ ],
+ [
+ "Cheaper Inference",
+ "cheaperinference",
+ "CHEAPER_INFERENCE_API_KEY",
+ "gpt-5.4",
+ CHEAPERINFERENCE_CODEX_PROVIDER,
+ ],
+] as const;
+
type ClientArguments = ConstructorParameters<typeof CodexSecurity>;
type Dependencies = NonNullable<ClientArguments[1]>;
diff --git a/sdk/typescript/tests-ts/support/semgrep-import-fixtures.ts b/sdk/typescript/tests-ts/support/semgrep-import-fixtures.ts
index 772a2b038..6f1ac06bf 100644
--- a/sdk/typescript/tests-ts/support/semgrep-import-fixtures.ts
+++ b/sdk/typescript/tests-ts/support/semgrep-import-fixtures.ts
@@ -73,3 +73,31 @@ export function semgrepSarif() {
],
};
}
+
+export function semgrepPercentPathReports() {
+ const paths = [
+ "src/my file.ts",
+ "src/my%20file.ts",
+ "src/100%.ts",
+ "src/100%25.ts",
+ ];
+ const json = semgrepJson();
+ const jsonResult = json.results[0]!;
+ const { fingerprint: _fingerprint, ...extra } = jsonResult.extra;
+ const sarif = semgrepSarif();
+ const run = sarif.runs[0]!;
+ const { fingerprints: _fingerprints, ...sarifResult } = run.results[0]!;
+ const results = paths.map((path) => {
+ const result = structuredClone(sarifResult);
+ result.locations[0]!.physicalLocation.artifactLocation.uri = path;
+ return result;
+ });
+ return {
+ paths,
+ json: {
+ ...json,
+ results: paths.map((path) => ({ ...jsonResult, path, extra })),
+ },
+ sarif: { ...sarif, runs: [{ ...run, results }] },
+ };
+}