Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Prune acoustic clock trials below retained score floor
Cinna Davis962aff002f19
diff --git a/docs/DATA_AND_SCALE.md b/docs/DATA_AND_SCALE.mdindex 6f3885e..89d8e9e 100644--- a/docs/DATA_AND_SCALE.md+++ b/docs/DATA_AND_SCALE.md@@ -153,6 +153,16 @@ the remaining supported clocks are fitted together. Representative batchedfits are checked against the scalar fitter, with fallback for mismatches orunsupported and numerically degenerate batches.+When an event's conservative loss bound alone makes the association unable to+reach the current Top-K floor, the scorer may also skip that clock and retain a+finite score ceiling. This applies only when the other matched groups cannot+make up the deficit even with perfect scores. Strict comparisons preserve ties;+the result verifier distinguishes these finite ceilings from outright acoustic+rejections. On one full-day point with 468 real associations, retained candidates+matched exactly and CPU time changed from 37.77 to 28.23 seconds. The persisted+bound ledger differs because it accounts for additional skipped trials. This+single measurement does not establish nationwide runtime or accuracy.+For association sets of at least 1,000 candidates, repeated acoustic batchesreuse exact track, receiver, time-array and sound-speed inputs in a per-worker32 MiB cache. Sparse searches keep the ordinary path. A complete real-data clockdiff --git a/docs/HANDOFF.md b/docs/HANDOFF.mdindex ca5740a..dec1f2a 100644--- a/docs/HANDOFF.md+++ b/docs/HANDOFF.md@@ -94,7 +94,10 @@ contains a newer harmonic-ambiguity confidence check from PR #192, which is notin the active search process. Its aligned-sample heuristic flagged no pair whenrun over all 17 discovered ridges from this upload; this does not establish thatthe groups are independent. Apply current confidence checks to the final-matched evidence before accepting a result.+matched evidence before accepting a result. PR #203 also corrects cache keys+for caller-supplied acoustic arrays; one complete 468-association point produced+the same full payload and retention ledger before and after that fix. That+point-level comparison does not prove equivalence across the national search.## Read firstdiff --git a/scripts/verify_upload_http.py b/scripts/verify_upload_http.pyindex 6eb224d..6fbeb78 100644--- a/scripts/verify_upload_http.py+++ b/scripts/verify_upload_http.py@@ -246,7 +246,8 @@ def bounded_finite_clock_assessment(detail, max_groups):or assessment.get("unknown_score_bound_kind") != kindor assessment.get("unknown_score_bound_method") not in("nonnegative_loss_and_validated_event_fits", "group_count_ceiling",- "nonnegative_loss_validated_fits_and_acoustic_dual")+ "nonnegative_loss_validated_fits_and_acoustic_dual",+ "nonnegative_loss_validated_fits_and_acoustic_floor_dual")or not isinstance(indices, list) or not indicesor any(type(index) is not int or not 0 <= index < max_groups for index in indices)or indices != sorted(set(indices))):@@ -263,9 +264,15 @@ def bounded_finite_clock_assessment(detail, max_groups):return Falsemethod = assessment["unknown_score_bound_method"]dual = assessment.get("acoustic_bound_rejected_trials", 0)- if type(dual) is not int:+ floor_dual = assessment.get("acoustic_floor_pruned_trials", 0)+ if type(dual) is not int or type(floor_dual) is not int:return False- if method == "nonnegative_loss_validated_fits_and_acoustic_dual":+ if method == "nonnegative_loss_validated_fits_and_acoustic_floor_dual":+ if not (0 < floor_dual <= bounded-rejected and 0 <= dual <= rejected):+ return False+ elif floor_dual != 0:+ return False+ elif method == "nonnegative_loss_validated_fits_and_acoustic_dual":if not 0 < dual <= rejected:return Falseelif dual != 0:diff --git a/src/dopplergeist/acoustic_screen.py b/src/dopplergeist/acoustic_screen.pyindex 570657c..54a3fca 100644--- a/src/dopplergeist/acoustic_screen.py+++ b/src/dopplergeist/acoustic_screen.py@@ -4,7 +4,7 @@ import numpy as npfrom .matching import _frequency_scale-def ridge_trial_rejections(measurement, event, factors, emissions):+def ridge_trial_rejections(measurement, event, factors, emissions, *, score_floor=None, group_count=None):"""Propose joint-trial rejections from a supported event's loss bound.The pseudo-Huber loss has a bounded convex dual. Any dual vector inside@@ -17,14 +17,20 @@ def ridge_trial_rejections(measurement, event, factors, emissions):``factors`` and ``emissions`` are the existing batched acoustic results,with one row per clock and one column per measured ridge sample. Incompletesupport, invalid settings, ill-conditioned fits and uncertain numerical- results return False so the ordinary physical fit handles them.+ results return False so the ordinary physical fit handles them. When a+ retention floor and group count are supplied, also return finite upper+ scores for supported trials that cannot reach that floor. Other entries+ in the second array are NaN."""factors = np.asarray(factors, dtype=float)emissions = np.asarray(emissions, dtype=float)trials = factors.shape[0] if factors.ndim else 0rejected = np.zeros(trials, dtype=bool)+ upper = np.full(trials, np.nan)+ def result():+ return (rejected, upper) if score_floor is not None else rejectedif factors.ndim != 2 or emissions.shape != factors.shape:- return rejected+ return result()try:sigma = float(_frequency_scale(event))fraction = float(event.get("quality", {}).get("source_drift_fraction", .02))@@ -32,26 +38,26 @@ def ridge_trial_rejections(measurement, event, factors, emissions):spec, ridge, weight, _ = measurementtimes, ridge, weight = (np.asarray(a, dtype=float) for a in (spec.time, ridge, weight))except (ValueError, TypeError, KeyError, AttributeError):- return rejected+ return result()if (not np.isfinite(sigma) or sigma <= 0 or not np.isfinite(fraction) or not 0 <= fraction <= .1or not np.isfinite(low) or not np.isfinite(high) or not 0 < low <= highor times.ndim != 1 or times.shape != ridge.shape or times.shape != weight.shapeor not np.all(np.isfinite(weight)) or np.any(weight < 0)):- return rejected+ return result()measured = np.isfinite(times) & np.isfinite(ridge) & (weight > 0)if measured.sum() < 10 or factors.shape[1] != measured.sum():- return rejected+ return result()times, ridge, weight = times[measured], ridge[measured], weight[measured]_, first = np.unique(times, return_index=True)first = np.sort(first)count = min(48, len(first))if count < 3:- return rejected+ return result()selected = first[np.linspace(0, len(first)-1, count, dtype=int)]active = np.flatnonzero(np.all(np.isfinite(factors), axis=1) & np.all(factors > 0, axis=1)& np.all(np.isfinite(emissions), axis=1))if not len(active):- return rejected+ return result()with np.errstate(over="ignore", invalid="ignore", divide="ignore"):x, full_emission = factors[active][:, selected], emissions[active]samples, y, w = full_emission[:, selected], ridge[selected], weight[selected]@@ -86,4 +92,13 @@ def ridge_trial_rejections(measurement, event, factors, emissions):lower = np.maximum(0., value-error)/np.sum(weight)rejected[active] = (valid & np.isfinite(lower)& (lower > (3*sigma)**2*(1+1e-8)+1e-9))- return rejected+ if (score_floor is not None and type(group_count) is int and group_count > 0+ and np.isfinite(score_floor) and 0 <= group_count-score_floor < 1):+ threshold = (3*sigma*(group_count-score_floor))**2*(1+1e-8)+1e-9+ ceiling = group_count-np.sqrt(lower)/(3*sigma)+ ceiling = np.maximum(0., np.nextafter(+ ceiling + 64*np.finfo(float).eps*(1+group_count+np.abs(ceiling)), np.inf))+ bounded = (valid & np.isfinite(lower) & (lower > threshold) & ~rejected[active]+ & np.isfinite(ceiling) & (ceiling < score_floor))+ upper[active] = np.where(bounded, ceiling, np.nan)+ return result()diff --git a/src/dopplergeist/finite_search.py b/src/dopplergeist/finite_search.pyindex d471613..8373f6b 100644--- a/src/dopplergeist/finite_search.py+++ b/src/dopplergeist/finite_search.py@@ -63,7 +63,8 @@ def bounded_assessment(row):or row.get("score_kind") != kindor assessment.get("unknown_score_bound_method") not in("nonnegative_loss_and_validated_event_fits", "group_count_ceiling",- "nonnegative_loss_validated_fits_and_acoustic_dual")+ "nonnegative_loss_validated_fits_and_acoustic_dual",+ "nonnegative_loss_validated_fits_and_acoustic_floor_dual")or not isinstance(indices, list) or not indicesor any(type(i) is not int or i < 0 for i in indices) or indices != sorted(set(indices))or len(row.get("flight_ids", [])) != len(indices)):@@ -83,9 +84,15 @@ def bounded_assessment(row):return Falsemethod = assessment["unknown_score_bound_method"]dual = assessment.get("acoustic_bound_rejected_trials", 0)- if type(dual) is not int:+ floor_dual = assessment.get("acoustic_floor_pruned_trials", 0)+ if type(dual) is not int or type(floor_dual) is not int:return False- if method == "nonnegative_loss_validated_fits_and_acoustic_dual":+ if method == "nonnegative_loss_validated_fits_and_acoustic_floor_dual":+ if not (0 < floor_dual <= bounded-rejected and 0 <= dual <= rejected):+ return False+ elif floor_dual != 0:+ return False+ elif method == "nonnegative_loss_validated_fits_and_acoustic_dual":if not 0 < dual <= rejected:return Falseelif dual != 0:diff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.pyindex 4900408..6606d25 100644--- a/src/dopplergeist/search.py+++ b/src/dopplergeist/search.py@@ -411,8 +411,10 @@ def _screen_association_trial(tracks, events, measurements, observer, start, c,diagnostics = []missing = {"score": -np.inf, "f0_hz": np.nan, "support_fraction": 0.0}for index, (track, event, measurement) in enumerate(zip(tracks, events, measurements)):- if dual_at is not None and dual_at(index):- return None, -np.inf+ if dual_at is not None:+ upper = dual_at(index)+ if upper is not None:+ return None, upperacoustic = None if acoustic_at is None else [acoustic_at(index)]if index == 0 and first_fit is not None:single = {"event_diagnostics": [first_fit]}@@ -521,6 +523,7 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="brightfirst_fits, first_batch_attempted = None, Falsedual_flags = {}dual_pruned = 0+ floor_dual_pruned = 0for trial, shift in enumerate(() if floor_pruned else shifts):if mode == "robust_ridge":from .matching import score_ridges@@ -537,23 +540,33 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="brightacoustic_at(index)batch = batches[index]if batch is None:- dual_flags[index] = np.zeros(len(shifts), dtype=bool)+ dual_flags[index] = (np.zeros(len(shifts), dtype=bool),+ np.full(len(shifts), np.nan))else:from .acoustic_screen import ridge_trial_rejections- dual_flags[index] = ridge_trial_rejections(measurements[index], events[index], *batch)+ dual_flags[index] = ridge_trial_rejections(measurements[index], events[index], *batch,+ score_floor=score_floor, group_count=len(events))return dual_flags[index]def dual_at(index):- nonlocal dual_pruned- rejected = bool(dual_flags_at(index)[trial])+ nonlocal dual_pruned, floor_dual_pruned+ rejections, bounds = dual_flags_at(index)+ rejected = bool(rejections[trial])dual_pruned += int(rejected)- return rejected+ if rejected:+ return -np.inf+ upper = bounds[trial]+ if np.isfinite(upper):+ floor_dual_pruned += 1+ return float(upper)+ return Nonetry:if association and score_floor is not None:if not first_batch_attempted and batches is not None and lazy_batches is not None and time_origin is None:first_batch_attempted = Trueacoustic_at(0)if batches[0] is not None:- eligible = np.flatnonzero(~dual_flags_at(0))+ rejected, upper = dual_flags_at(0)+ eligible = np.flatnonzero(~rejected & ~np.isfinite(upper))if len(eligible) >= 4:from .batch_ridge import first_event_fitsfactors, emissions = batches[0]@@ -611,6 +624,8 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="brightif mode == "robust_ridge":if dual_pruned:assessment["acoustic_bound_rejected_trials"] = dual_pruned+ if floor_dual_pruned:+ assessment["acoustic_floor_pruned_trials"] = floor_dual_prunedif partial_prunes and np.isfinite(best["score"]) and best["score"] >= score_floor:# Competitive modes retain their complete original diagnostics. Cached# fits are reused; skipped clocks are evaluated rather than relabelled.@@ -635,7 +650,9 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="brightbounded_unknown_rejections=sum(value == -np.inf for value in unknown_bounds if value is not None),unknown_score_upper_bound=max(finite_bounds) if finite_bounds else None,unknown_score_bound_kind="matched_group_support_minus_normalized_residual",- unknown_score_bound_method=("nonnegative_loss_validated_fits_and_acoustic_dual"+ unknown_score_bound_method=("nonnegative_loss_validated_fits_and_acoustic_floor_dual"+ if floor_dual_pruned else+ "nonnegative_loss_validated_fits_and_acoustic_dual"if dual_pruned else "nonnegative_loss_and_validated_event_fits"))if partial_prunes:assessment["retention_score_floor"] = score_floordiff --git a/tests/test_real_acoustic_screen.py b/tests/test_real_acoustic_screen.pyindex 50fc260..f2a6d37 100644--- a/tests/test_real_acoustic_screen.py+++ b/tests/test_real_acoustic_screen.py@@ -26,7 +26,7 @@ def test_real_ridge_rejections_have_supported_failing_witnesses():sorted(set(table.flight_id)), max_tracks=2000, max_samples=2000000)}point = fixture["point"]observer = ecef(point["lat"], point["lon"], point["elev_m"])- witnessed = tested = unsupported = 0+ witnessed = floor_witnessed = tested = unsupported = 0for event, measurement in zip(events, measurements):sigma = _frequency_scale(event)fraction = event.get("quality", {}).get("source_drift_fraction", .02)@@ -45,9 +45,17 @@ def test_real_ridge_rejections_have_supported_failing_witnesses():continuefactors, emissions = batchrejected = ridge_trial_rejections(measurement, event, factors, emissions)+ rejected_with_floor, ceiling = ridge_trial_rejections(+ measurement, event, factors, emissions, score_floor=.7, group_count=1)+ rejected_three, ceiling_three = ridge_trial_rejections(+ measurement, event, factors, emissions, score_floor=2.7, group_count=3)+ assert np.array_equal(rejected, rejected_with_floor)+ assert np.array_equal(rejected, rejected_three)+ assert np.array_equal(np.isfinite(ceiling), np.isfinite(ceiling_three))assert rejected.shape == starts.shapeincomplete = ~np.all(np.isfinite(factors) & np.isfinite(emissions), axis=1)assert not np.any(rejected & incomplete)+ assert not np.any(np.isfinite(ceiling) & incomplete)tested += int((~incomplete).sum())unsupported += int(incomplete.sum())for i in np.flatnonzero(rejected):@@ -58,7 +66,15 @@ def test_real_ridge_rejections_have_supported_failing_witnesses():assert np.isfinite(fit["score"])assert -fit["score"] > 3*sigmawitnessed += 1- assert tested > 100 and witnessed > 0 and unsupported > 0+ for i in np.flatnonzero(np.isfinite(ceiling)):+ fit = _fit_ridge_uncached(track, observer, measurement, starts[i],+ event["f0_range_hz"], 343., sigma, fraction,+ acoustic_result=(factors[i], emissions[i]))+ assert fit["support_fraction"] == 1. and np.isfinite(fit["score"])+ assert 1+fit["score"]/(3*sigma) <= ceiling[i] < .7+ assert 3+fit["score"]/(3*sigma) <= ceiling_three[i] < 2.7+ floor_witnessed += 1+ assert tested > 100 and witnessed > 0 and floor_witnessed > 0 and unsupported > 0def test_real_candidate_scores_match_full_fits_with_screening(monkeypatch):@@ -97,14 +113,15 @@ def test_real_candidate_scores_match_full_fits_with_screening(monkeypatch):original = screen.ridge_trial_rejectionswith monkeypatch.context() as patch:patch.setattr(screen, "ridge_trial_rejections",- lambda measurement, event, factors, emissions: np.zeros(len(factors), dtype=bool))+ lambda measurement, event, factors, emissions, **kwargs:+ (np.zeros(len(factors), dtype=bool), np.full(len(factors), np.nan)))baseline = run()witnessed = 0later_witnessed = 0- def checked(measurement, event, factors, emissions):+ def checked(measurement, event, factors, emissions, **kwargs):nonlocal witnessed, later_witnessed- rejected = original(measurement, event, factors, emissions)+ rejected, ceilings = original(measurement, event, factors, emissions, **kwargs)sigma = _frequency_scale(event)fraction = event.get("quality", {}).get("source_drift_fraction", .02)for i in np.flatnonzero(rejected):@@ -115,7 +132,7 @@ def test_real_candidate_scores_match_full_fits_with_screening(monkeypatch):witnessed += 1if measurement is not current_first_measurement:later_witnessed += 1- return rejected+ return rejected, ceilingswith monkeypatch.context() as patch:patch.setattr(screen, "ridge_trial_rejections", checked)diff --git a/tests/test_real_dual_provenance.py b/tests/test_real_dual_provenance.pyindex 86de8b0..d062a97 100644--- a/tests/test_real_dual_provenance.py+++ b/tests/test_real_dual_provenance.py@@ -51,3 +51,25 @@ def test_real_bound_methods_and_independent_verifier():invalid["scoring_assessment"][field] = valueassert not bounded_assessment(invalid)assert not verifier.bounded_finite_clock_assessment(dict(invalid, finite_best=False), len(events))++ singleton_candidates = [c for c in candidates if len(c["event_indices"]) == 1][:40]+ track_ids = sorted({entry["flight_id"] for c in singleton_candidates for entry in c["passes"]})+ singleton_tracks = {track.id: track for track in FlightIndex(fixture["real_index"]).lookup(track_ids)}+ for candidate in singleton_candidates:+ if any(entry["flight_id"] not in singleton_tracks for entry in candidate["passes"]):+ continue+ row = score_candidate(candidate, point, singleton_tracks, events, measurements,+ mode="robust_ridge", score_floor=.99)+ assessment = row["scoring_assessment"]+ if assessment.get("acoustic_floor_pruned_trials", 0):+ break+ else:+ pytest.fail("No retention-floor acoustic bound in the real candidate sample")+ assert assessment["unknown_score_bound_method"] == "nonnegative_loss_validated_fits_and_acoustic_floor_dual"+ assert bounded_assessment(row)+ assert verifier.bounded_finite_clock_assessment(dict(row, finite_best=False), len(events))+ for value in (True, 0, assessment["bounded_unknown_trials"]-assessment["bounded_unknown_rejections"]+1):+ invalid = copy.deepcopy(row)+ invalid["scoring_assessment"]["acoustic_floor_pruned_trials"] = value+ assert not bounded_assessment(invalid)+ assert not verifier.bounded_finite_clock_assessment(dict(invalid, finite_best=False), len(events))