Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

test(egress): fix H2 CI observation coverage

leo pein623ad7adc07e
diff --git a/components/egress/tests/test_credential_bound_h2_runtime.py b/components/egress/tests/test_credential_bound_h2_runtime.py
index 9e03b854f..110061809 100644
--- a/components/egress/tests/test_credential_bound_h2_runtime.py
+++ b/components/egress/tests/test_credential_bound_h2_runtime.py
@@ -1397,7 +1397,7 @@ def run_case(name, artifacts, args):
def summarize_results(cases, selected):
- """Keep manual stress, real admission, delayed and GOAWAY cases distinct."""
+ """Summarize the full frozen matrix; omitted results remain NOT_RUN."""
groups = {
"manual_peer_late_goaway_stress": MANUAL_PEER_CASES,
"real_live_runtime_pre_admission_barrier": REAL_RUNTIME_CASES,
@@ -1408,18 +1408,49 @@ def summarize_results(cases, selected):
"go_early_notice_h2_origin": ("go_early_notice_h2_origin",),
}
vocabulary = ("PASS", "FAIL", "BLOCKED", "NOT_RUN")
+ all_cases = (*CASES, *GO_CASES)
+ known_cases = set(all_cases)
+ case_statuses = {}
+
+ for item in cases:
+ if not isinstance(item, dict):
+ raise ValueError("case results must be mappings")
+ name = item.get("case")
+ status = item.get("status")
+ if not isinstance(name, str) or name not in known_cases:
+ raise ValueError(f"unknown case: {name!r}")
+ if name in case_statuses:
+ raise ValueError(f"duplicate case result: {name!r}")
+ if status not in vocabulary:
+ raise ValueError(f"unknown case status for {name!r}: {status!r}")
+ case_statuses[name] = status
+
+ selected = tuple(selected)
+ if any(not isinstance(name, str) or name not in known_cases for name in selected):
+ raise ValueError("unknown selected case")
+ if len(set(selected)) != len(selected):
+ raise ValueError("duplicate selected case")
+
+ def result_status(statuses):
+ return ("FAIL" if "FAIL" in statuses else
+ "BLOCKED" if "BLOCKED" in statuses else
+ "NOT_RUN" if "NOT_RUN" in statuses or not statuses else
+ "PASS")
def group(names):
- statuses = [item["status"] for item in cases if item["case"] in names]
- status = ("FAIL" if "FAIL" in statuses else "BLOCKED" if "BLOCKED" in statuses
- else "NOT_RUN" if "NOT_RUN" in statuses or not statuses else "PASS")
- return {"denominator": len(names), "status": status,
- "counts": {value: statuses.count(value) for value in vocabulary}}
-
- return {"status": group(selected)["status"],
- "summary": {value: sum(item["status"] == value for item in cases)
- for value in vocabulary},
- "denominator_summaries": {name: group(names) for name, names in groups.items()}}
+ statuses = [case_statuses.get(name, "NOT_RUN") for name in names]
+ return {
+ "denominator": len(names),
+ "status": result_status(statuses),
+ "counts": {value: statuses.count(value) for value in vocabulary},
+ }
+
+ summary_statuses = [case_statuses.get(name, "NOT_RUN") for name in all_cases]
+ return {
+ "status": result_status([case_statuses.get(name, "NOT_RUN") for name in selected]),
+ "summary": {value: summary_statuses.count(value) for value in vocabulary},
+ "denominator_summaries": {name: group(names) for name, names in groups.items()},
+ }
def main(argv=None):
diff --git a/components/egress/tests/test_credential_bound_h2_runtime_selection.py b/components/egress/tests/test_credential_bound_h2_runtime_selection.py
index f7e3b5cb3..20b7f4f4f 100644
--- a/components/egress/tests/test_credential_bound_h2_runtime_selection.py
+++ b/components/egress/tests/test_credential_bound_h2_runtime_selection.py
@@ -68,6 +68,42 @@ class RuntimeSelectionTests(unittest.TestCase):
"FAIL",
)
+ def test_sparse_candidate_results_leave_missing_rows_not_run(self):
+ selected = [*runner.REAL_RUNTIME_CASES, *runner.GO_EARLY_CASES]
+ summary = runner.summarize_results(
+ [{"case": runner.REAL_RUNTIME_CASES[0], "status": "PASS"}],
+ selected,
+ )
+
+ candidate = summary["denominator_summaries"]["candidate_acceptance"]
+ self.assertEqual("NOT_RUN", summary["status"])
+ self.assertEqual("NOT_RUN", candidate["status"])
+ self.assertEqual(3, candidate["denominator"])
+ self.assertEqual(
+ {"PASS": 1, "FAIL": 0, "BLOCKED": 0, "NOT_RUN": 2},
+ candidate["counts"],
+ )
+ self.assertEqual(
+ {"PASS": 1, "FAIL": 0, "BLOCKED": 0, "NOT_RUN": 19},
+ summary["summary"],
+ )
+ self.assertEqual(20, sum(summary["summary"].values()))
+ for group in summary["denominator_summaries"].values():
+ self.assertEqual(group["denominator"], sum(group["counts"].values()))
+
+ def test_duplicate_cases_and_unknown_statuses_are_rejected(self):
+ name = runner.REAL_RUNTIME_CASES[0]
+ with self.assertRaisesRegex(ValueError, "duplicate"):
+ runner.summarize_results(
+ [{"case": name, "status": "PASS"},
+ {"case": name, "status": "NOT_RUN"}],
+ [name],
+ )
+ with self.assertRaisesRegex(ValueError, "status"):
+ runner.summarize_results([{"case": name, "status": "SKIPPED"}], [name])
+ with self.assertRaisesRegex(ValueError, "unknown selected case"):
+ runner.summarize_results([], ["unknown-case"])
+
def test_early_notice_failure_keeps_candidate_gate_nonzero(self):
selected = runner.select_cases(None, True)
statuses = {name: "PASS" for name in selected}
diff --git a/components/egress/tests/test_h2_retry_observation.py b/components/egress/tests/test_h2_retry_observation.py
index 5a7b11104..f2338a1d5 100644
--- a/components/egress/tests/test_h2_retry_observation.py
+++ b/components/egress/tests/test_h2_retry_observation.py
@@ -52,16 +52,30 @@ class RetryObservationTest(unittest.TestCase):
# Even a post-ACK event is stale when it precedes this new command.
self.assertIsNone(runner.go_fence_event([good], 7, 201))
- def test_manual_and_both_go_denominators_remain_separate_not_run(self):
+ def test_manual_barrier_and_go_denominators_remain_separate_not_run(self):
output = io.StringIO()
with contextlib.redirect_stdout(output):
self.assertEqual(0, runner.main(["--list"]))
value = json.loads(output.getvalue())
self.assertEqual("NOT_RUN", value["status"])
- self.assertEqual(15, len(value["manual_denominator"]))
- self.assertEqual(["go_retry", "go_retry_bounded_old"], value["optional_client_cases"])
- self.assertIn("old never finishes", value["go_scope"]["go_retry_bounded_old"])
- self.assertIn("outlasts", value["go_scope"]["go_retry_bounded_old"])
+ self.assertEqual(16, len(value["manual_denominator"]))
+ self.assertEqual(15, len(value["manual_peer_stress_denominator"]))
+ self.assertEqual(1, len(value["real_runtime_cases"]))
+ self.assertEqual(
+ ["go_retry", "go_retry_bounded_old",
+ "go_early_notice_h1_origin", "go_early_notice_h2_origin"],
+ value["optional_client_cases"],
+ )
+ self.assertEqual(
+ 20,
+ len(value["manual_peer_stress_denominator"])
+ + len(value["real_runtime_cases"])
+ + len(value["optional_client_cases"]),
+ )
+ self.assertIn("not a candidate gate after early GOAWAY",
+ value["go_scope"]["go_retry"])
+ self.assertIn("does not prove arbitrary held-old migration",
+ value["go_scope"]["go_retry_bounded_old"])
def test_bounded_success_cannot_promote_held_old_failure(self):
cases = [{"case": name, "status": "PASS"} for name in runner.CASES]
@@ -70,17 +84,40 @@ class RetryObservationTest(unittest.TestCase):
summary = runner.summarize_results(cases, [item["case"] for item in cases])
self.assertEqual("FAIL", summary["status"])
groups = summary["denominator_summaries"]
- self.assertEqual({"manual_peer", "go_held_old", "go_bounded_old"}, set(groups))
- self.assertEqual(15, groups["manual_peer"]["denominator"])
- self.assertEqual("PASS", groups["manual_peer"]["status"])
- self.assertEqual(1, groups["go_held_old"]["denominator"])
- self.assertEqual("FAIL", groups["go_held_old"]["status"])
- self.assertEqual(1, groups["go_held_old"]["counts"]["FAIL"])
- self.assertEqual("PASS", groups["go_bounded_old"]["status"])
+ self.assertEqual({
+ "manual_peer_late_goaway_stress",
+ "real_live_runtime_pre_admission_barrier",
+ "candidate_acceptance",
+ "go_delayed_held_old_diagnostic_not_candidate_gate",
+ "go_bounded_old_diagnostic_not_candidate_gate",
+ "go_early_notice_h1_origin",
+ "go_early_notice_h2_origin",
+ }, set(groups))
+ manual = groups["manual_peer_late_goaway_stress"]
+ self.assertEqual(15, manual["denominator"])
+ self.assertEqual("PASS", manual["status"])
+ barrier = groups["real_live_runtime_pre_admission_barrier"]
+ self.assertEqual(1, barrier["denominator"])
+ self.assertEqual("PASS", barrier["status"])
+ candidate = groups["candidate_acceptance"]
+ self.assertEqual(3, candidate["denominator"])
+ self.assertEqual("NOT_RUN", candidate["status"])
+ self.assertEqual(
+ {"PASS": 1, "FAIL": 0, "BLOCKED": 0, "NOT_RUN": 2},
+ candidate["counts"],
+ )
+ delayed_group = groups["go_delayed_held_old_diagnostic_not_candidate_gate"]
+ self.assertEqual("FAIL", delayed_group["status"])
+ self.assertEqual(1, delayed_group["counts"]["FAIL"])
+ bounded_group = groups["go_bounded_old_diagnostic_not_candidate_gate"]
+ self.assertEqual("PASS", bounded_group["status"])
+ self.assertEqual(1, bounded_group["counts"]["PASS"])
cases[-2]["status"] = "NOT_RUN"
summary = runner.summarize_results(cases, [item["case"] for item in cases])
self.assertEqual("NOT_RUN", summary["status"])
- self.assertEqual(1, summary["denominator_summaries"]["go_held_old"]["counts"]["NOT_RUN"])
+ delayed = summary["denominator_summaries"]["go_delayed_held_old_diagnostic_not_candidate_gate"]
+ self.assertEqual(1, delayed["counts"]["NOT_RUN"])
+ self.assertEqual(3, summary["denominator_summaries"]["candidate_acceptance"]["denominator"])
def exercise_go_observation(self, *, bounded, cleanup_fails=False):
"""Drive the real wrapper with memory-only child output and controls."""
diff --git a/components/egress/tests/test_h2_static_handoff.py b/components/egress/tests/test_h2_static_handoff.py
index ef693549b..f4fd9c432 100644
--- a/components/egress/tests/test_h2_static_handoff.py
+++ b/components/egress/tests/test_h2_static_handoff.py
@@ -24,6 +24,7 @@ class StaticHandoffTests(unittest.TestCase):
harness.protocol = "h2"
harness.scenario = "normal"
harness.drain_seconds = 8
+ harness.barrier_release_path = root / "release-pre-admission"
harness.ipc = object.__new__(LiveRuntimeHarness)
harness.ipc.socket_path = str(root / "receiver.sock")
harness.socket_path = harness.ipc.socket_path
@@ -41,6 +42,7 @@ class StaticHandoffTests(unittest.TestCase):
"OPENSANDBOX_UNRELATED": "untrusted",
"H2_PRODUCTION_OBSERVATIONS": "/untrusted",
"H2_PRODUCTION_SCENARIO": "untrusted",
+ "H2_PRODUCTION_BARRIER_RELEASE": "/untrusted/release",
"H2_PRODUCTION_UNRECOGNIZED": "untrusted",
"SSLKEYLOGFILE": "/untrusted/keys",
"MITMPROXY_SSLKEYLOGFILE": "/untrusted/keys",
@@ -68,11 +70,14 @@ class StaticHandoffTests(unittest.TestCase):
self.assertEqual(environment["OPENSANDBOX_EGRESS_REVISION_DRAIN_TIMEOUT_SECONDS"], "8")
self.assertEqual(environment["H2_PRODUCTION_OBSERVATIONS"], str(root))
self.assertEqual(environment["H2_PRODUCTION_SCENARIO"], "normal")
+ self.assertEqual(environment["H2_PRODUCTION_BARRIER_RELEASE"],
+ str(root / "release-pre-admission"))
self.assertEqual(environment["PATH"], "/synthetic/bin")
for key in inherited.keys() - {
"PATH", "OPENSANDBOX_EGRESS_MITMPROXY_PASSTHROUGH_HOSTS",
"OPENSANDBOX_EGRESS_REVISION_IPC_TOKEN",
"H2_PRODUCTION_OBSERVATIONS", "H2_PRODUCTION_SCENARIO",
+ "H2_PRODUCTION_BARRIER_RELEASE",
}:
self.assertNotIn(key, environment)
self.assertEqual(harness.combined_ca.read_bytes(), b"proxy-ca\norigin-ca\n")