Secure, Fast, and Extensible Sandbox runtime for AI agents.
test(egress): fix H2 CI observation coverage
leo pein623ad7adc07e
diff --git a/components/egress/tests/test_credential_bound_h2_runtime.py b/components/egress/tests/test_credential_bound_h2_runtime.pyindex 9e03b854f..110061809 100644--- a/components/egress/tests/test_credential_bound_h2_runtime.py+++ b/components/egress/tests/test_credential_bound_h2_runtime.py@@ -1397,7 +1397,7 @@ def run_case(name, artifacts, args):def summarize_results(cases, selected):- """Keep manual stress, real admission, delayed and GOAWAY cases distinct."""+ """Summarize the full frozen matrix; omitted results remain NOT_RUN."""groups = {"manual_peer_late_goaway_stress": MANUAL_PEER_CASES,"real_live_runtime_pre_admission_barrier": REAL_RUNTIME_CASES,@@ -1408,18 +1408,49 @@ def summarize_results(cases, selected):"go_early_notice_h2_origin": ("go_early_notice_h2_origin",),}vocabulary = ("PASS", "FAIL", "BLOCKED", "NOT_RUN")+ all_cases = (*CASES, *GO_CASES)+ known_cases = set(all_cases)+ case_statuses = {}++ for item in cases:+ if not isinstance(item, dict):+ raise ValueError("case results must be mappings")+ name = item.get("case")+ status = item.get("status")+ if not isinstance(name, str) or name not in known_cases:+ raise ValueError(f"unknown case: {name!r}")+ if name in case_statuses:+ raise ValueError(f"duplicate case result: {name!r}")+ if status not in vocabulary:+ raise ValueError(f"unknown case status for {name!r}: {status!r}")+ case_statuses[name] = status++ selected = tuple(selected)+ if any(not isinstance(name, str) or name not in known_cases for name in selected):+ raise ValueError("unknown selected case")+ if len(set(selected)) != len(selected):+ raise ValueError("duplicate selected case")++ def result_status(statuses):+ return ("FAIL" if "FAIL" in statuses else+ "BLOCKED" if "BLOCKED" in statuses else+ "NOT_RUN" if "NOT_RUN" in statuses or not statuses else+ "PASS")def group(names):- statuses = [item["status"] for item in cases if item["case"] in names]- status = ("FAIL" if "FAIL" in statuses else "BLOCKED" if "BLOCKED" in statuses- else "NOT_RUN" if "NOT_RUN" in statuses or not statuses else "PASS")- return {"denominator": len(names), "status": status,- "counts": {value: statuses.count(value) for value in vocabulary}}-- return {"status": group(selected)["status"],- "summary": {value: sum(item["status"] == value for item in cases)- for value in vocabulary},- "denominator_summaries": {name: group(names) for name, names in groups.items()}}+ statuses = [case_statuses.get(name, "NOT_RUN") for name in names]+ return {+ "denominator": len(names),+ "status": result_status(statuses),+ "counts": {value: statuses.count(value) for value in vocabulary},+ }++ summary_statuses = [case_statuses.get(name, "NOT_RUN") for name in all_cases]+ return {+ "status": result_status([case_statuses.get(name, "NOT_RUN") for name in selected]),+ "summary": {value: summary_statuses.count(value) for value in vocabulary},+ "denominator_summaries": {name: group(names) for name, names in groups.items()},+ }def main(argv=None):diff --git a/components/egress/tests/test_credential_bound_h2_runtime_selection.py b/components/egress/tests/test_credential_bound_h2_runtime_selection.pyindex f7e3b5cb3..20b7f4f4f 100644--- a/components/egress/tests/test_credential_bound_h2_runtime_selection.py+++ b/components/egress/tests/test_credential_bound_h2_runtime_selection.py@@ -68,6 +68,42 @@ class RuntimeSelectionTests(unittest.TestCase):"FAIL",)+ def test_sparse_candidate_results_leave_missing_rows_not_run(self):+ selected = [*runner.REAL_RUNTIME_CASES, *runner.GO_EARLY_CASES]+ summary = runner.summarize_results(+ [{"case": runner.REAL_RUNTIME_CASES[0], "status": "PASS"}],+ selected,+ )++ candidate = summary["denominator_summaries"]["candidate_acceptance"]+ self.assertEqual("NOT_RUN", summary["status"])+ self.assertEqual("NOT_RUN", candidate["status"])+ self.assertEqual(3, candidate["denominator"])+ self.assertEqual(+ {"PASS": 1, "FAIL": 0, "BLOCKED": 0, "NOT_RUN": 2},+ candidate["counts"],+ )+ self.assertEqual(+ {"PASS": 1, "FAIL": 0, "BLOCKED": 0, "NOT_RUN": 19},+ summary["summary"],+ )+ self.assertEqual(20, sum(summary["summary"].values()))+ for group in summary["denominator_summaries"].values():+ self.assertEqual(group["denominator"], sum(group["counts"].values()))++ def test_duplicate_cases_and_unknown_statuses_are_rejected(self):+ name = runner.REAL_RUNTIME_CASES[0]+ with self.assertRaisesRegex(ValueError, "duplicate"):+ runner.summarize_results(+ [{"case": name, "status": "PASS"},+ {"case": name, "status": "NOT_RUN"}],+ [name],+ )+ with self.assertRaisesRegex(ValueError, "status"):+ runner.summarize_results([{"case": name, "status": "SKIPPED"}], [name])+ with self.assertRaisesRegex(ValueError, "unknown selected case"):+ runner.summarize_results([], ["unknown-case"])+def test_early_notice_failure_keeps_candidate_gate_nonzero(self):selected = runner.select_cases(None, True)statuses = {name: "PASS" for name in selected}diff --git a/components/egress/tests/test_h2_retry_observation.py b/components/egress/tests/test_h2_retry_observation.pyindex 5a7b11104..f2338a1d5 100644--- a/components/egress/tests/test_h2_retry_observation.py+++ b/components/egress/tests/test_h2_retry_observation.py@@ -52,16 +52,30 @@ class RetryObservationTest(unittest.TestCase):# Even a post-ACK event is stale when it precedes this new command.self.assertIsNone(runner.go_fence_event([good], 7, 201))- def test_manual_and_both_go_denominators_remain_separate_not_run(self):+ def test_manual_barrier_and_go_denominators_remain_separate_not_run(self):output = io.StringIO()with contextlib.redirect_stdout(output):self.assertEqual(0, runner.main(["--list"]))value = json.loads(output.getvalue())self.assertEqual("NOT_RUN", value["status"])- self.assertEqual(15, len(value["manual_denominator"]))- self.assertEqual(["go_retry", "go_retry_bounded_old"], value["optional_client_cases"])- self.assertIn("old never finishes", value["go_scope"]["go_retry_bounded_old"])- self.assertIn("outlasts", value["go_scope"]["go_retry_bounded_old"])+ self.assertEqual(16, len(value["manual_denominator"]))+ self.assertEqual(15, len(value["manual_peer_stress_denominator"]))+ self.assertEqual(1, len(value["real_runtime_cases"]))+ self.assertEqual(+ ["go_retry", "go_retry_bounded_old",+ "go_early_notice_h1_origin", "go_early_notice_h2_origin"],+ value["optional_client_cases"],+ )+ self.assertEqual(+ 20,+ len(value["manual_peer_stress_denominator"])+ + len(value["real_runtime_cases"])+ + len(value["optional_client_cases"]),+ )+ self.assertIn("not a candidate gate after early GOAWAY",+ value["go_scope"]["go_retry"])+ self.assertIn("does not prove arbitrary held-old migration",+ value["go_scope"]["go_retry_bounded_old"])def test_bounded_success_cannot_promote_held_old_failure(self):cases = [{"case": name, "status": "PASS"} for name in runner.CASES]@@ -70,17 +84,40 @@ class RetryObservationTest(unittest.TestCase):summary = runner.summarize_results(cases, [item["case"] for item in cases])self.assertEqual("FAIL", summary["status"])groups = summary["denominator_summaries"]- self.assertEqual({"manual_peer", "go_held_old", "go_bounded_old"}, set(groups))- self.assertEqual(15, groups["manual_peer"]["denominator"])- self.assertEqual("PASS", groups["manual_peer"]["status"])- self.assertEqual(1, groups["go_held_old"]["denominator"])- self.assertEqual("FAIL", groups["go_held_old"]["status"])- self.assertEqual(1, groups["go_held_old"]["counts"]["FAIL"])- self.assertEqual("PASS", groups["go_bounded_old"]["status"])+ self.assertEqual({+ "manual_peer_late_goaway_stress",+ "real_live_runtime_pre_admission_barrier",+ "candidate_acceptance",+ "go_delayed_held_old_diagnostic_not_candidate_gate",+ "go_bounded_old_diagnostic_not_candidate_gate",+ "go_early_notice_h1_origin",+ "go_early_notice_h2_origin",+ }, set(groups))+ manual = groups["manual_peer_late_goaway_stress"]+ self.assertEqual(15, manual["denominator"])+ self.assertEqual("PASS", manual["status"])+ barrier = groups["real_live_runtime_pre_admission_barrier"]+ self.assertEqual(1, barrier["denominator"])+ self.assertEqual("PASS", barrier["status"])+ candidate = groups["candidate_acceptance"]+ self.assertEqual(3, candidate["denominator"])+ self.assertEqual("NOT_RUN", candidate["status"])+ self.assertEqual(+ {"PASS": 1, "FAIL": 0, "BLOCKED": 0, "NOT_RUN": 2},+ candidate["counts"],+ )+ delayed_group = groups["go_delayed_held_old_diagnostic_not_candidate_gate"]+ self.assertEqual("FAIL", delayed_group["status"])+ self.assertEqual(1, delayed_group["counts"]["FAIL"])+ bounded_group = groups["go_bounded_old_diagnostic_not_candidate_gate"]+ self.assertEqual("PASS", bounded_group["status"])+ self.assertEqual(1, bounded_group["counts"]["PASS"])cases[-2]["status"] = "NOT_RUN"summary = runner.summarize_results(cases, [item["case"] for item in cases])self.assertEqual("NOT_RUN", summary["status"])- self.assertEqual(1, summary["denominator_summaries"]["go_held_old"]["counts"]["NOT_RUN"])+ delayed = summary["denominator_summaries"]["go_delayed_held_old_diagnostic_not_candidate_gate"]+ self.assertEqual(1, delayed["counts"]["NOT_RUN"])+ self.assertEqual(3, summary["denominator_summaries"]["candidate_acceptance"]["denominator"])def exercise_go_observation(self, *, bounded, cleanup_fails=False):"""Drive the real wrapper with memory-only child output and controls."""diff --git a/components/egress/tests/test_h2_static_handoff.py b/components/egress/tests/test_h2_static_handoff.pyindex ef693549b..f4fd9c432 100644--- a/components/egress/tests/test_h2_static_handoff.py+++ b/components/egress/tests/test_h2_static_handoff.py@@ -24,6 +24,7 @@ class StaticHandoffTests(unittest.TestCase):harness.protocol = "h2"harness.scenario = "normal"harness.drain_seconds = 8+ harness.barrier_release_path = root / "release-pre-admission"harness.ipc = object.__new__(LiveRuntimeHarness)harness.ipc.socket_path = str(root / "receiver.sock")harness.socket_path = harness.ipc.socket_path@@ -41,6 +42,7 @@ class StaticHandoffTests(unittest.TestCase):"OPENSANDBOX_UNRELATED": "untrusted","H2_PRODUCTION_OBSERVATIONS": "/untrusted","H2_PRODUCTION_SCENARIO": "untrusted",+ "H2_PRODUCTION_BARRIER_RELEASE": "/untrusted/release","H2_PRODUCTION_UNRECOGNIZED": "untrusted","SSLKEYLOGFILE": "/untrusted/keys","MITMPROXY_SSLKEYLOGFILE": "/untrusted/keys",@@ -68,11 +70,14 @@ class StaticHandoffTests(unittest.TestCase):self.assertEqual(environment["OPENSANDBOX_EGRESS_REVISION_DRAIN_TIMEOUT_SECONDS"], "8")self.assertEqual(environment["H2_PRODUCTION_OBSERVATIONS"], str(root))self.assertEqual(environment["H2_PRODUCTION_SCENARIO"], "normal")+ self.assertEqual(environment["H2_PRODUCTION_BARRIER_RELEASE"],+ str(root / "release-pre-admission"))self.assertEqual(environment["PATH"], "/synthetic/bin")for key in inherited.keys() - {"PATH", "OPENSANDBOX_EGRESS_MITMPROXY_PASSTHROUGH_HOSTS","OPENSANDBOX_EGRESS_REVISION_IPC_TOKEN","H2_PRODUCTION_OBSERVATIONS", "H2_PRODUCTION_SCENARIO",+ "H2_PRODUCTION_BARRIER_RELEASE",}:self.assertNotIn(key, environment)self.assertEqual(harness.combined_ca.read_bytes(), b"proxy-ca\norigin-ca\n")