Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Fail real-recording metamorphic comparisons on coverage/completion changes (#405)

Cinna Davisf0ebc651c0ee
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 6ba7f34..61d4e1a 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -87,6 +87,8 @@ jobs:
assert hashlib.sha256(source.read_bytes()).hexdigest() == original
print(f"Recorded checkpoint progress passed: 256 receipts, {expected} receiver points; cached refresh reads no input JSON.")
PY
+ - name: Check metamorphic completion gates against real recorded outcomes
+ run: python -m pytest -q tests/test_real_metamorphic_runner.py::test_recorded_matrix_keeps_coverage_and_completion_separate_from_timing
- name: CLI smoke check
run: dopplergeist --help
- name: Check installed native validator and packaged browser assets
diff --git a/docs/REAL_METAMORPHIC.md b/docs/REAL_METAMORPHIC.md
index ee1c8e0..5d9b676 100644
--- a/docs/REAL_METAMORPHIC.md
+++ b/docs/REAL_METAMORPHIC.md
@@ -104,6 +104,13 @@ GEOS version, preparation refuses the incomplete runtime pin.
The runner preserves encoded inputs, exact transformation commands, actual
job artifacts and observations for every arm. It continues after an ordinary
arm failure. It preserves pending arms on interruption or changed inputs.
+
+Pairwise comparisons also fail if the material coverage state (search completion,
+data availability, missing-data identifiers or requested area) changes, or if
+verified pipeline/work-completion gates change. Measured elapsed time, peak memory
+and event proposal counts remain reported but are not themselves completeness
+failures. Raw coverage descriptions are retained separately. This check applies
+to newly generated reports; the recorded November 4 results are immutable.
An unusable encoded arm still reaches the HTTP pipeline. Encoding failures
remain explicit failures.
diff --git a/scripts/run_real_metamorphic.py b/scripts/run_real_metamorphic.py
index aa1a1b9..11763cb 100644
--- a/scripts/run_real_metamorphic.py
+++ b/scripts/run_real_metamorphic.py
@@ -623,11 +623,36 @@ def associations_equal(a, b, mapping):
return sorted((mapping[event], flight) for event, flight in first) == sorted(zip(b["event_ids"], b["flight_ids"]))
+def coverage_state(coverage):
+ """Comparable coverage evidence; preserve raw limitations in the report."""
+ if not isinstance(coverage, dict):
+ return None
+ return {key: coverage.get(key) for key in
+ ("search_complete", "data_status", "missing_data", "requested_area")}
+
+
+def completion_state(observation):
+ """Completion gates, excluding elapsed time and performance-only counters."""
+ work = observation.get("work") or {}
+ return {
+ "verification_passed": observation.get("verification_passed"),
+ "full_pipeline_observed": observation.get("full_pipeline_observed"),
+ "full_pipeline_requirements_met": observation.get("full_pipeline_requirements_met"),
+ **{key: work.get(key) for key in
+ ("status", "unsearched_tiles", "model_assessment_complete",
+ "model_support_complete", "data_scope_complete",
+ "all_event_groups_considered",
+ "alternative_modes_exhaustively_assessed")},
+ }
+
+
def compare_cases(left, right, protocol):
a, b = left["observation"], right["observation"]
tol = protocol["tolerances"]
correspondence = event_correspondence(a["mapped_events"], b["mapped_events"], protocol)
mapping = {m["left_event_id"]: m["right_event_id"] for m in correspondence["matches"]}
+ coverage_states = [coverage_state(a.get("coverage")), coverage_state(b.get("coverage"))]
+ completion_states = [completion_state(a), completion_state(b)]
result = {"left": left["id"], "right": right["id"], "event_correspondence": correspondence,
"representation_usable": bool(left.get("usability", {}).get("passed") and right.get("usability", {}).get("passed")),
"complete_pipelines": bool(a["full_pipeline_requirements_met"] and b["full_pipeline_requirements_met"]),
@@ -636,12 +661,16 @@ def compare_cases(left, right, protocol):
"group_count_delta": b["group_count"] - a["group_count"],
"ridge_count_delta": b["ridge_count"] - a["ridge_count"],
"coverage_equal": a["coverage"] == b["coverage"],
+ "coverage_state_transition": coverage_states,
+ "completion_state_transition": completion_states,
"coverage_transition": [a["coverage"], b["coverage"]],
"work_transition": [a["work"], b["work"]],
"decision_provenance_equal": a["decision_provenance"] == b["decision_provenance"],
"diagnostic_candidate_comparison": None, "accepted_location_displacement_m": None,
"serious_accepted_jump": False, "violations": []}
checks = {
+ "coverage_state_changed": coverage_states[0] != coverage_states[1],
+ "work_completion_changed": completion_states[0] != completion_states[1],
"status_changed": a["status"] != b["status"],
"reason_changed": a["reason_code"] != b["reason_code"],
"group_count_changed": a["group_count"] != b["group_count"],
diff --git a/tests/test_real_metamorphic_runner.py b/tests/test_real_metamorphic_runner.py
index e24c5fa..0ad9e94 100644
--- a/tests/test_real_metamorphic_runner.py
+++ b/tests/test_real_metamorphic_runner.py
@@ -115,3 +115,31 @@ def test_chunk_only_replay_compares_actual_baseline(tmp_path, case_id):
assert row["usability"]["chunk_input_matches_baseline"] is True
baseline = next((output / "baseline-encoding").glob("*.mp3"))
assert runner.identity(baseline)["sha256"] == row["encoded_input"]["sha256"]
+
+
+
+def test_recorded_matrix_keeps_coverage_and_completion_separate_from_timing():
+ """Actual nine-arm results share coverage/work gates, not measured CPU cost."""
+ archive = json.loads((ROOT / "docs/verification/real_metamorphic_reference_2026-11-04.json").read_text())
+ cases = archive["cases"]
+ assert len(cases) == 9 and archive["pairwise_comparison_count"] == 36
+
+ def observed(case):
+ return {"work": case["work"],
+ "verification_passed": case["verification_passed"],
+ "full_pipeline_observed": case["full_pipeline_observed"],
+ "full_pipeline_requirements_met": case["full_pipeline_requirements_met"]}
+
+ coverage = runner.coverage_state(cases[0]["coverage"])
+ completion = runner.completion_state(observed(cases[0]))
+ assert coverage["search_complete"] is False
+ assert coverage["data_status"] == "partial"
+ assert coverage["missing_data"] == ["historical_flight_scope_or_audible_border_region"]
+ assert completion["full_pipeline_requirements_met"] is True
+ assert completion["model_assessment_complete"] is False
+ assert completion["all_event_groups_considered"] is True
+ assert all(runner.coverage_state(case["coverage"]) == coverage for case in cases)
+ assert all(runner.completion_state(observed(case)) == completion for case in cases)
+ assert len({case["work"]["elapsed_s"] for case in cases}) > 1
+ assert len({case["work"]["event_groups_proposed"] for case in cases}) > 1
+ # These observed runtime and proposal-count differences are not completion changes.