Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Exclude unsupported tracks ending before the event clock (#299)
Cinna Davis03548be5fc04
diff --git a/docs/MODEL_SUPPORT_COVERAGE.md b/docs/MODEL_SUPPORT_COVERAGE.mdindex dccdd2a..51b4dd2 100644--- a/docs/MODEL_SUPPORT_COVERAGE.md+++ b/docs/MODEL_SUPPORT_COVERAGE.md@@ -11,7 +11,9 @@ partial. A completed tile does not establish complete evidence.For robust-ridge subset association with a bounded recording-start window and nosilence filter, an unsupported trajectory that starts strictly after every-possible event CPA arrival cannot participate in an association. Those tracks+possible event CPA arrival cannot participate in an association. Nor can one+whose latest emission plus the maximum sound-travel delay is strictly before+every possible event CPA arrival. Those tracksare reported separately as `clock_excluded_unsupported_tracks` in stagediagnostics and counted in `clock_excluded_unsupported_track_evaluations`; theiroriginal motion failures and conservative clock bounds remain inspectable. The@@ -21,7 +23,7 @@ fixed-clock mode, other scoring modes, and silence filtering retain the usualunsupported/partial behavior. This does not repair the source trajectory orestablish geographic coverage. The HTTP evidence verifier checks the recordedinequality against the search clock and stage totals; it does not independently-reload the original aircraft index to verify the recorded first emission time.+reload the original aircraft index to verify the recorded emission endpoints.The additive `model_support` report contains these integer work counts:diff --git a/scripts/verify_upload_http.py b/scripts/verify_upload_http.pyindex 0f7ad1f..e618110 100644--- a/scripts/verify_upload_http.py+++ b/scripts/verify_upload_http.py@@ -351,16 +351,30 @@ def valid_clock_exclusion(item, configuration):c = number(scoring.get("c", 343.0))if lo > hi or sweep < 0 or offset < 0 or c <= 0:return False- bounds = []+ lower_bounds, upper_bounds = [], []for event in configuration["events"]:event_tolerance = number(event.get("quality", {}).get("cpa_tolerance_s", 0))if event_tolerance < 0:return False- bounds.append(hi + sweep + offset / 2 + event_tolerance + number(event["cpa_receive_s"]))+ cpa = number(event["cpa_receive_s"])+ lower_bounds.append(lo - sweep - offset / 2 - event_tolerance + cpa)+ upper_bounds.append(hi + sweep + offset / 2 + event_tolerance + cpa)+ if item.get("exclusion_kind") == "before_events":+ radii = [number(event["slant_range_m"][1]) for event in configuration["events"]]+ if not radii or min(radii) <= 0 or max(radii) >= 1000000:+ return False+ first_event = number(item["earliest_event_receive_s"])+ last_emission = number(item["latest_emission_s"])+ last_receive = number(item["latest_possible_receive_s"])+ return bool(lower_bounds and first_event <= min(lower_bounds)+ and last_receive >= last_emission + max(radii) / c+ and last_receive < first_event)+ if "exclusion_kind" in item:+ return Falsesaved = number(item["latest_event_receive_s"])first = number(item["earliest_emission_s"])- return bool(bounds and saved >= max(bounds) and first > saved)- except (KeyError, TypeError, ValueError, OverflowError):+ return bool(upper_bounds and saved >= max(upper_bounds) and first > saved)+ except (KeyError, IndexError, TypeError, ValueError, OverflowError):return Falsediff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.pyindex de3eb23..81ddff2 100644--- a/src/dopplergeist/tiled_search.py+++ b/src/dopplergeist/tiled_search.py@@ -356,7 +356,7 @@ def _association_clock_window(start_window, sweep_s):return center, tolerance-def _latest_event_receive_time(events, parameters, filters, start_window):+def _event_receive_time_bounds(events, parameters, filters, start_window):"""Conservatively bound matchable CPA arrivals for the shared-clock path.None means that this optional exclusion is not justified. Silence filters@@ -372,22 +372,26 @@ def _latest_event_receive_time(events, parameters, filters, start_window):if not math.isfinite(c) or c <= 0 or not math.isfinite(offset) or offset < 0:return Nonecenter, clock_tolerance = _association_clock_window(start_window, parameters.get("time_sweep_s", 5.0))+ lower = math.nextafter(center - clock_tolerance, -math.inf)upper = math.nextafter(center + clock_tolerance, math.inf)- latest = []+ earliest, latest = [], []for event in events:cpa = float(event["cpa_receive_s"])event_tolerance = float(event.get("quality", {}).get("cpa_tolerance_s", 0))if not math.isfinite(cpa) or not math.isfinite(event_tolerance) or event_tolerance < 0:return Nonemargin = math.nextafter(event_tolerance + offset / 2, math.inf)+ first = math.nextafter(math.nextafter(lower - margin, -math.inf) + cpa, -math.inf)bound = math.nextafter(math.nextafter(upper + margin, math.inf) + cpa, math.inf)# Association calculates receive_time - cpa before comparing clock# intervals. Extra ULPs cover that inverse floating-point rounding.bound += 4 * max(math.ulp(bound), math.ulp(center), math.ulp(cpa))- if not math.isfinite(bound):+ first -= 4 * max(math.ulp(first), math.ulp(center), math.ulp(cpa))+ if not math.isfinite(bound) or not math.isfinite(first):return None+ earliest.append(first)latest.append(bound)- return max(latest) if latest else None+ return (min(earliest), max(latest)) if latest else Noneexcept (KeyError, TypeError, ValueError, OverflowError):return None@@ -525,7 +529,7 @@ def _score(points, tracks, events, measurements, budget, parameters, filters, stpoint_ids = points.point_id.to_numpy(dtype=int)cpa_batches, count = [], 0unsupported, clock_excluded = [], []- latest_receive = _latest_event_receive_time(events, parameters, filters, start_window)+ receive_bounds = _event_receive_time_bounds(events, parameters, filters, start_window)# Bound materialized CPA rows before pairing; one track at a time keeps the# national traffic count out of the CPA tensor/list allocation.for track in tracks:@@ -551,12 +555,25 @@ def _score(points, tracks, events, measurements, budget, parameters, filters, sttrack_batches.append(np.array(rows, dtype=_CPA_DTYPE))del rowsexcept AcousticSupportError as error:- if latest_receive is not None and track.time[0] > latest_receive:+ if receive_bounds is not None and track.time[0] > receive_bounds[1]:clock_excluded.append({"track_id": track.id, "reason": error.reason,"reported_intervals": _support_error(error),"earliest_emission_s": float(track.time[0]),- "latest_event_receive_s": latest_receive})+ "latest_event_receive_s": receive_bounds[1]})continue+ if receive_bounds is not None:+ c = float(parameters.get("c", 343.0))+ delay = math.nextafter(radius / c, math.inf)+ last_receive = math.nextafter(float(track.time[-1]) + delay, math.inf)+ last_receive += 4 * max(math.ulp(last_receive), math.ulp(float(track.time[-1])), math.ulp(delay))+ if math.isfinite(last_receive) and last_receive < receive_bounds[0]:+ clock_excluded.append({"track_id": track.id, "reason": error.reason,+ "reported_intervals": _support_error(error),+ "exclusion_kind": "before_events",+ "latest_emission_s": float(track.time[-1]),+ "latest_possible_receive_s": last_receive,+ "earliest_event_receive_s": receive_bounds[0]})+ continue# The typed failure does not identify an exact bad interval. Keep# the entire track/stage as explicitly unassessed, and retain rows# from other tracks. Never repair or clip the source payload here.diff --git a/tests/test_real_early_unsupported_clock.py b/tests/test_real_early_unsupported_clock.pynew file mode 100644index 0000000..5e5045a--- /dev/null+++ b/tests/test_real_early_unsupported_clock.py@@ -0,0 +1,111 @@+"""Optional early-track bounds using archived real recording and aircraft."""+import importlib.util+import json+import os+from pathlib import Path++import numpy as np+import pandas as pd+import pytest++from dopplergeist.data_access import FlightIndex+from dopplergeist.matching import prepare_events+from dopplergeist.geometry import Track, ecef+from dopplergeist.search import _CPA_DTYPE, _PreparedCPATrack, _ordered_cpa_rows, filter_association_event+from dopplergeist.terrain import Terrain+from dopplergeist.tiled_search import (SearchBudget, _association_clock_window, _event_receive_time_bounds, _expanded_bounds,+ _score, _tracks_for_points, tile_points)+++def test_real_early_track_and_nonempty_tile(tmp_path, monkeypatch):+ names = ("DOPPLERGEIST_REAL_EARLY_JOB", "DOPPLERGEIST_REAL_EARLY_INDEX",+ "DOPPLERGEIST_REAL_EARLY_DEM", "DOPPLERGEIST_REAL_EARLY_GEOID")+ values = [os.environ.get(name) for name in names]+ if not all(values):+ pytest.skip("set real completed job, aircraft index, DEM and geoid paths")+ job, index_path, dem, geoid = map(Path, values)+ cfg = json.loads((job / "search/manifest.json").read_text())["configuration"]+ raw = json.loads((job / "discovery/discovery.json").read_text())["events"]+ params = dict(cfg["scoring"])+ window = params.pop("recording_start_window_s")+ filters = cfg["filters"]+ radius = max(event["slant_range_m"][1] for event in cfg["events"])+ events, measurements = prepare_events(raw, c=params["c"], max_distance_m=radius)+ assert events == cfg["events"]+ budget = SearchBudget(**cfg["budget"])+ index, terrain = FlightIndex(index_path), Terrain([dem], geoid_path=geoid)+ spec = importlib.util.spec_from_file_location("verify_early_clock",+ Path(__file__).parents[1] / "scripts/verify_upload_http.py")+ verifier = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(verifier)+ first_event, _ = _event_receive_time_bounds(events, params, filters, window)+ early_stages = []+ try:+ for f in sorted((job / "search/support").glob("coarse-*.json")):+ if f.stat().st_size > 1048576:+ continue+ support = json.loads(f.read_text())+ selected = [x for x in support.get("unsupported_cpa_tracks", [])+ if x.get("emission_time_range_s") and+ x["emission_time_range_s"][1] + radius / params["c"] + 1 < first_event]+ if not selected:+ continue+ tile_name = support["stage"].removeprefix("coarse:")+ payload = json.loads((job / "search" / f"tile-{tile_name}.json").read_text())["payload"]+ points = tile_points(payload["cells"], budget.spacing_m, cfg["bounds"], terrain, cfg["crs"])+ tracks = list(index.lookup([x["track_id"] for x in selected]))+ assert tracks+ rows, counts = _score(points, tracks, events, measurements, budget, params, filters, window)+ assert not rows and not counts["unsupported_cpa_tracks"]+ for item in counts["clock_excluded_unsupported_tracks"]:+ assert item["exclusion_kind"] == "before_events"+ assert verifier.valid_clock_exclusion(item, cfg)+ assert not verifier.valid_clock_exclusion({**item, "latest_possible_receive_s": item["earliest_event_receive_s"]}, cfg)+ assert not verifier.valid_clock_exclusion({**item, "earliest_event_receive_s": item["earliest_event_receive_s"] + 1}, cfg)+ assert not verifier.valid_clock_exclusion({**item, "latest_possible_receive_s": item["latest_emission_s"]}, cfg)+ assert len(counts["clock_excluded_unsupported_tracks"]) == len(tracks)+ # Diagnostic only: use the unchanged CPA proposer with its whole-+ # track guard bypassed to check the existing event filter. This is+ # not used as physical support or a location claim.+ observers = ecef(points.lat.to_numpy(), points.lon.to_numpy(), points.elev_m.to_numpy())+ point_ids = points.point_id.to_numpy(dtype=int)+ center, tolerance = _association_clock_window(window, params.get("time_sweep_s", 5.0))+ for track in tracks:+ with monkeypatch.context() as patch:+ patch.setattr(Track, "validate_motion_support", lambda self, c: None)+ raw_rows = _ordered_cpa_rows(_PreparedCPATrack(track, params["c"], budget.cpa_step_s).raw_rows(+ observers, point_ids, radius), budget.cpa_step_s)+ if raw_rows:+ table = pd.DataFrame(np.array(raw_rows, dtype=_CPA_DTYPE))+ assert all(filter_association_event(table, event, filters.get("offset_tolerance_s", 5),+ center, tolerance).empty for event in events)+ # Moving the same real recording clock across the boundary must+ # retain the unsupported source, with and without silence enabled.+ item = counts["clock_excluded_unsupported_tracks"][0]+ shift = item["latest_possible_receive_s"] - item["earliest_event_receive_s"]+ overlap = [v + shift - 1 for v in window]+ _, uncertain = _score(points, tracks, events, measurements, budget, params, filters, overlap)+ assert any(x["track_id"] == tracks[0].id for x in uncertain["unsupported_cpa_tracks"])+ _, silent = _score(points, tracks, events, measurements, budget, params,+ {**filters, "silence": {"edge_margin_s": 60}}, window)+ assert any(x["track_id"] == tracks[0].id for x in silent["unsupported_cpa_tracks"])+ early_stages.append((payload, points))+ assert len(early_stages) == 2+ payload, points = max(early_stages, key=lambda pair: len(pair[0]["ranked"]))+ assert payload["ranked"]+ tracks, points = _tracks_for_points([index], _expanded_bounds(payload["cells"], budget.spacing_m, radius, cfg["crs"]),+ cfg["time_range_s"], budget, points, terrain, radius, params["c"])+ import dopplergeist.tiled_search as tiled+ with monkeypatch.context() as patch:+ patch.setattr(tiled, "_event_receive_time_bounds", lambda *args: None)+ old_rows, old_counts = _score(points, tracks, events, measurements, budget, params, filters, window)+ rows, counts = _score(points, tracks, events, measurements, budget, params, filters, window)+ assert rows and rows == old_rows+ assert counts["cpa_rows"] == old_counts["cpa_rows"]+ assert counts["candidates"] == old_counts["candidates"]+ assert counts["unclassified_candidate_assessments"] == old_counts["unclassified_candidate_assessments"]+ assert len(counts["clock_excluded_unsupported_tracks"]) > 0+ assert len(old_counts["unsupported_cpa_tracks"]) == len(counts["unsupported_cpa_tracks"]) + len(counts["clock_excluded_unsupported_tracks"])+ finally:+ for raster in terrain.rasters + terrain.geoid.rasters:+ raster.close()diff --git a/tests/test_real_unsupported_clock.py b/tests/test_real_unsupported_clock.pyindex 63f8e8f..a977e09 100644--- a/tests/test_real_unsupported_clock.py+++ b/tests/test_real_unsupported_clock.py@@ -8,7 +8,7 @@ import pytestfrom dopplergeist.data_access import CachedTerrain, FlightIndexfrom dopplergeist.matching import prepare_events-from dopplergeist.tiled_search import (SearchBudget, _expanded_bounds, _latest_event_receive_time,+from dopplergeist.tiled_search import (SearchBudget, _expanded_bounds, _event_receive_time_bounds,_score, _tracks_for_points, tile_points)@@ -46,7 +46,7 @@ def test_real_clock_excludes_late_unsupported_trajectories(tmp_path, monkeypatchpoints = tile_points(payload["cells"], budget.spacing_m, config["bounds"], terrain, config["crs"])tracks = list(index.lookup([item["track_id"] for item in failures]))assert tracks- latest = _latest_event_receive_time(events, params, filters, window)+ latest = _event_receive_time_bounds(events, params, filters, window)[1]assert latest is not None and all(track.time[0] > latest for track in tracks)rows, counts = _score(points, tracks, events, measurements, budget, params, filters, window)assert rows == [] and not counts["unsupported_cpa_tracks"]@@ -61,16 +61,16 @@ def test_real_clock_excludes_late_unsupported_trajectories(tmp_path, monkeypatchoverlapping = [window[0] + shift, window[1] + shift]_, unknown = _score(points, tracks, events, measurements, budget, params, filters, overlapping)assert any(item["track_id"] == tracks[0].id for item in unknown["unsupported_cpa_tracks"])- assert _latest_event_receive_time(events, params, {**filters, "silence": {"edge_margin_s": 60}}, window) is None- assert _latest_event_receive_time(events, params, filters, None) is None+ assert _event_receive_time_bounds(events, params, {**filters, "silence": {"edge_margin_s": 60}}, window) is None+ assert _event_receive_time_bounds(events, params, filters, None) is Noneif not tested:# Probe the actual track against clock windows on both sides of its# first recorded emission, including the inclusive boundary.boundary_shift = float(tracks[0].time[0] - latest)before = [value + boundary_shift - 0.01 for value in window]after = [value + boundary_shift + 0.01 for value in window]- assert tracks[0].time[0] > _latest_event_receive_time(events, params, filters, before)- assert tracks[0].time[0] <= _latest_event_receive_time(events, params, filters, after)+ assert tracks[0].time[0] > _event_receive_time_bounds(events, params, filters, before)[1]+ assert tracks[0].time[0] <= _event_receive_time_bounds(events, params, filters, after)[1]_, before_counts = _score(points, tracks, events, measurements, budget, params, filters, before)_, after_counts = _score(points, tracks, events, measurements, budget, params, filters, after)assert any(item["track_id"] == tracks[0].id for item in before_counts["clock_excluded_unsupported_tracks"])@@ -94,7 +94,7 @@ def test_real_clock_excludes_late_unsupported_trajectories(tmp_path, monkeypatchconfig["time_range_s"], budget, points, terrain, radius, params["c"])import dopplergeist.tiled_search as tiledwith monkeypatch.context() as patch:- patch.setattr(tiled, "_latest_event_receive_time", lambda *args: None)+ patch.setattr(tiled, "_event_receive_time_bounds", lambda *args: None)previous_rows, previous_counts = _score(points, tracks, events, measurements, budget, params, filters, window)rows, counts = _score(points, tracks, events, measurements, budget, params, filters, window)assert rows and counts["candidates"] > 0