Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Prune acoustic clock trials below retained score floor

Cinna Davis962aff002f19
diff --git a/docs/DATA_AND_SCALE.md b/docs/DATA_AND_SCALE.md
index 6f3885e..89d8e9e 100644
--- a/docs/DATA_AND_SCALE.md
+++ b/docs/DATA_AND_SCALE.md
@@ -153,6 +153,16 @@ the remaining supported clocks are fitted together. Representative batched
fits are checked against the scalar fitter, with fallback for mismatches or
unsupported and numerically degenerate batches.
+When an event's conservative loss bound alone makes the association unable to
+reach the current Top-K floor, the scorer may also skip that clock and retain a
+finite score ceiling. This applies only when the other matched groups cannot
+make up the deficit even with perfect scores. Strict comparisons preserve ties;
+the result verifier distinguishes these finite ceilings from outright acoustic
+rejections. On one full-day point with 468 real associations, retained candidates
+matched exactly and CPU time changed from 37.77 to 28.23 seconds. The persisted
+bound ledger differs because it accounts for additional skipped trials. This
+single measurement does not establish nationwide runtime or accuracy.
+
For association sets of at least 1,000 candidates, repeated acoustic batches
reuse exact track, receiver, time-array and sound-speed inputs in a per-worker
32 MiB cache. Sparse searches keep the ordinary path. A complete real-data clock
diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md
index ca5740a..dec1f2a 100644
--- a/docs/HANDOFF.md
+++ b/docs/HANDOFF.md
@@ -94,7 +94,10 @@ contains a newer harmonic-ambiguity confidence check from PR #192, which is not
in the active search process. Its aligned-sample heuristic flagged no pair when
run over all 17 discovered ridges from this upload; this does not establish that
the groups are independent. Apply current confidence checks to the final
-matched evidence before accepting a result.
+matched evidence before accepting a result. PR #203 also corrects cache keys
+for caller-supplied acoustic arrays; one complete 468-association point produced
+the same full payload and retention ledger before and after that fix. That
+point-level comparison does not prove equivalence across the national search.
## Read first
diff --git a/scripts/verify_upload_http.py b/scripts/verify_upload_http.py
index 6eb224d..6fbeb78 100644
--- a/scripts/verify_upload_http.py
+++ b/scripts/verify_upload_http.py
@@ -246,7 +246,8 @@ def bounded_finite_clock_assessment(detail, max_groups):
or assessment.get("unknown_score_bound_kind") != kind
or assessment.get("unknown_score_bound_method") not in
("nonnegative_loss_and_validated_event_fits", "group_count_ceiling",
- "nonnegative_loss_validated_fits_and_acoustic_dual")
+ "nonnegative_loss_validated_fits_and_acoustic_dual",
+ "nonnegative_loss_validated_fits_and_acoustic_floor_dual")
or not isinstance(indices, list) or not indices
or any(type(index) is not int or not 0 <= index < max_groups for index in indices)
or indices != sorted(set(indices))):
@@ -263,9 +264,15 @@ def bounded_finite_clock_assessment(detail, max_groups):
return False
method = assessment["unknown_score_bound_method"]
dual = assessment.get("acoustic_bound_rejected_trials", 0)
- if type(dual) is not int:
+ floor_dual = assessment.get("acoustic_floor_pruned_trials", 0)
+ if type(dual) is not int or type(floor_dual) is not int:
return False
- if method == "nonnegative_loss_validated_fits_and_acoustic_dual":
+ if method == "nonnegative_loss_validated_fits_and_acoustic_floor_dual":
+ if not (0 < floor_dual <= bounded-rejected and 0 <= dual <= rejected):
+ return False
+ elif floor_dual != 0:
+ return False
+ elif method == "nonnegative_loss_validated_fits_and_acoustic_dual":
if not 0 < dual <= rejected:
return False
elif dual != 0:
diff --git a/src/dopplergeist/acoustic_screen.py b/src/dopplergeist/acoustic_screen.py
index 570657c..54a3fca 100644
--- a/src/dopplergeist/acoustic_screen.py
+++ b/src/dopplergeist/acoustic_screen.py
@@ -4,7 +4,7 @@ import numpy as np
from .matching import _frequency_scale
-def ridge_trial_rejections(measurement, event, factors, emissions):
+def ridge_trial_rejections(measurement, event, factors, emissions, *, score_floor=None, group_count=None):
"""Propose joint-trial rejections from a supported event's loss bound.
The pseudo-Huber loss has a bounded convex dual. Any dual vector inside
@@ -17,14 +17,20 @@ def ridge_trial_rejections(measurement, event, factors, emissions):
``factors`` and ``emissions`` are the existing batched acoustic results,
with one row per clock and one column per measured ridge sample. Incomplete
support, invalid settings, ill-conditioned fits and uncertain numerical
- results return False so the ordinary physical fit handles them.
+ results return False so the ordinary physical fit handles them. When a
+ retention floor and group count are supplied, also return finite upper
+ scores for supported trials that cannot reach that floor. Other entries
+ in the second array are NaN.
"""
factors = np.asarray(factors, dtype=float)
emissions = np.asarray(emissions, dtype=float)
trials = factors.shape[0] if factors.ndim else 0
rejected = np.zeros(trials, dtype=bool)
+ upper = np.full(trials, np.nan)
+ def result():
+ return (rejected, upper) if score_floor is not None else rejected
if factors.ndim != 2 or emissions.shape != factors.shape:
- return rejected
+ return result()
try:
sigma = float(_frequency_scale(event))
fraction = float(event.get("quality", {}).get("source_drift_fraction", .02))
@@ -32,26 +38,26 @@ def ridge_trial_rejections(measurement, event, factors, emissions):
spec, ridge, weight, _ = measurement
times, ridge, weight = (np.asarray(a, dtype=float) for a in (spec.time, ridge, weight))
except (ValueError, TypeError, KeyError, AttributeError):
- return rejected
+ return result()
if (not np.isfinite(sigma) or sigma <= 0 or not np.isfinite(fraction) or not 0 <= fraction <= .1
or not np.isfinite(low) or not np.isfinite(high) or not 0 < low <= high
or times.ndim != 1 or times.shape != ridge.shape or times.shape != weight.shape
or not np.all(np.isfinite(weight)) or np.any(weight < 0)):
- return rejected
+ return result()
measured = np.isfinite(times) & np.isfinite(ridge) & (weight > 0)
if measured.sum() < 10 or factors.shape[1] != measured.sum():
- return rejected
+ return result()
times, ridge, weight = times[measured], ridge[measured], weight[measured]
_, first = np.unique(times, return_index=True)
first = np.sort(first)
count = min(48, len(first))
if count < 3:
- return rejected
+ return result()
selected = first[np.linspace(0, len(first)-1, count, dtype=int)]
active = np.flatnonzero(np.all(np.isfinite(factors), axis=1) & np.all(factors > 0, axis=1)
& np.all(np.isfinite(emissions), axis=1))
if not len(active):
- return rejected
+ return result()
with np.errstate(over="ignore", invalid="ignore", divide="ignore"):
x, full_emission = factors[active][:, selected], emissions[active]
samples, y, w = full_emission[:, selected], ridge[selected], weight[selected]
@@ -86,4 +92,13 @@ def ridge_trial_rejections(measurement, event, factors, emissions):
lower = np.maximum(0., value-error)/np.sum(weight)
rejected[active] = (valid & np.isfinite(lower)
& (lower > (3*sigma)**2*(1+1e-8)+1e-9))
- return rejected
+ if (score_floor is not None and type(group_count) is int and group_count > 0
+ and np.isfinite(score_floor) and 0 <= group_count-score_floor < 1):
+ threshold = (3*sigma*(group_count-score_floor))**2*(1+1e-8)+1e-9
+ ceiling = group_count-np.sqrt(lower)/(3*sigma)
+ ceiling = np.maximum(0., np.nextafter(
+ ceiling + 64*np.finfo(float).eps*(1+group_count+np.abs(ceiling)), np.inf))
+ bounded = (valid & np.isfinite(lower) & (lower > threshold) & ~rejected[active]
+ & np.isfinite(ceiling) & (ceiling < score_floor))
+ upper[active] = np.where(bounded, ceiling, np.nan)
+ return result()
diff --git a/src/dopplergeist/finite_search.py b/src/dopplergeist/finite_search.py
index d471613..8373f6b 100644
--- a/src/dopplergeist/finite_search.py
+++ b/src/dopplergeist/finite_search.py
@@ -63,7 +63,8 @@ def bounded_assessment(row):
or row.get("score_kind") != kind
or assessment.get("unknown_score_bound_method") not in
("nonnegative_loss_and_validated_event_fits", "group_count_ceiling",
- "nonnegative_loss_validated_fits_and_acoustic_dual")
+ "nonnegative_loss_validated_fits_and_acoustic_dual",
+ "nonnegative_loss_validated_fits_and_acoustic_floor_dual")
or not isinstance(indices, list) or not indices
or any(type(i) is not int or i < 0 for i in indices) or indices != sorted(set(indices))
or len(row.get("flight_ids", [])) != len(indices)):
@@ -83,9 +84,15 @@ def bounded_assessment(row):
return False
method = assessment["unknown_score_bound_method"]
dual = assessment.get("acoustic_bound_rejected_trials", 0)
- if type(dual) is not int:
+ floor_dual = assessment.get("acoustic_floor_pruned_trials", 0)
+ if type(dual) is not int or type(floor_dual) is not int:
return False
- if method == "nonnegative_loss_validated_fits_and_acoustic_dual":
+ if method == "nonnegative_loss_validated_fits_and_acoustic_floor_dual":
+ if not (0 < floor_dual <= bounded-rejected and 0 <= dual <= rejected):
+ return False
+ elif floor_dual != 0:
+ return False
+ elif method == "nonnegative_loss_validated_fits_and_acoustic_dual":
if not 0 < dual <= rejected:
return False
elif dual != 0:
diff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.py
index 4900408..6606d25 100644
--- a/src/dopplergeist/search.py
+++ b/src/dopplergeist/search.py
@@ -411,8 +411,10 @@ def _screen_association_trial(tracks, events, measurements, observer, start, c,
diagnostics = []
missing = {"score": -np.inf, "f0_hz": np.nan, "support_fraction": 0.0}
for index, (track, event, measurement) in enumerate(zip(tracks, events, measurements)):
- if dual_at is not None and dual_at(index):
- return None, -np.inf
+ if dual_at is not None:
+ upper = dual_at(index)
+ if upper is not None:
+ return None, upper
acoustic = None if acoustic_at is None else [acoustic_at(index)]
if index == 0 and first_fit is not None:
single = {"event_diagnostics": [first_fit]}
@@ -521,6 +523,7 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="bright
first_fits, first_batch_attempted = None, False
dual_flags = {}
dual_pruned = 0
+ floor_dual_pruned = 0
for trial, shift in enumerate(() if floor_pruned else shifts):
if mode == "robust_ridge":
from .matching import score_ridges
@@ -537,23 +540,33 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="bright
acoustic_at(index)
batch = batches[index]
if batch is None:
- dual_flags[index] = np.zeros(len(shifts), dtype=bool)
+ dual_flags[index] = (np.zeros(len(shifts), dtype=bool),
+ np.full(len(shifts), np.nan))
else:
from .acoustic_screen import ridge_trial_rejections
- dual_flags[index] = ridge_trial_rejections(measurements[index], events[index], *batch)
+ dual_flags[index] = ridge_trial_rejections(measurements[index], events[index], *batch,
+ score_floor=score_floor, group_count=len(events))
return dual_flags[index]
def dual_at(index):
- nonlocal dual_pruned
- rejected = bool(dual_flags_at(index)[trial])
+ nonlocal dual_pruned, floor_dual_pruned
+ rejections, bounds = dual_flags_at(index)
+ rejected = bool(rejections[trial])
dual_pruned += int(rejected)
- return rejected
+ if rejected:
+ return -np.inf
+ upper = bounds[trial]
+ if np.isfinite(upper):
+ floor_dual_pruned += 1
+ return float(upper)
+ return None
try:
if association and score_floor is not None:
if not first_batch_attempted and batches is not None and lazy_batches is not None and time_origin is None:
first_batch_attempted = True
acoustic_at(0)
if batches[0] is not None:
- eligible = np.flatnonzero(~dual_flags_at(0))
+ rejected, upper = dual_flags_at(0)
+ eligible = np.flatnonzero(~rejected & ~np.isfinite(upper))
if len(eligible) >= 4:
from .batch_ridge import first_event_fits
factors, emissions = batches[0]
@@ -611,6 +624,8 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="bright
if mode == "robust_ridge":
if dual_pruned:
assessment["acoustic_bound_rejected_trials"] = dual_pruned
+ if floor_dual_pruned:
+ assessment["acoustic_floor_pruned_trials"] = floor_dual_pruned
if partial_prunes and np.isfinite(best["score"]) and best["score"] >= score_floor:
# Competitive modes retain their complete original diagnostics. Cached
# fits are reused; skipped clocks are evaluated rather than relabelled.
@@ -635,7 +650,9 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="bright
bounded_unknown_rejections=sum(value == -np.inf for value in unknown_bounds if value is not None),
unknown_score_upper_bound=max(finite_bounds) if finite_bounds else None,
unknown_score_bound_kind="matched_group_support_minus_normalized_residual",
- unknown_score_bound_method=("nonnegative_loss_validated_fits_and_acoustic_dual"
+ unknown_score_bound_method=("nonnegative_loss_validated_fits_and_acoustic_floor_dual"
+ if floor_dual_pruned else
+ "nonnegative_loss_validated_fits_and_acoustic_dual"
if dual_pruned else "nonnegative_loss_and_validated_event_fits"))
if partial_prunes:
assessment["retention_score_floor"] = score_floor
diff --git a/tests/test_real_acoustic_screen.py b/tests/test_real_acoustic_screen.py
index 50fc260..f2a6d37 100644
--- a/tests/test_real_acoustic_screen.py
+++ b/tests/test_real_acoustic_screen.py
@@ -26,7 +26,7 @@ def test_real_ridge_rejections_have_supported_failing_witnesses():
sorted(set(table.flight_id)), max_tracks=2000, max_samples=2000000)}
point = fixture["point"]
observer = ecef(point["lat"], point["lon"], point["elev_m"])
- witnessed = tested = unsupported = 0
+ witnessed = floor_witnessed = tested = unsupported = 0
for event, measurement in zip(events, measurements):
sigma = _frequency_scale(event)
fraction = event.get("quality", {}).get("source_drift_fraction", .02)
@@ -45,9 +45,17 @@ def test_real_ridge_rejections_have_supported_failing_witnesses():
continue
factors, emissions = batch
rejected = ridge_trial_rejections(measurement, event, factors, emissions)
+ rejected_with_floor, ceiling = ridge_trial_rejections(
+ measurement, event, factors, emissions, score_floor=.7, group_count=1)
+ rejected_three, ceiling_three = ridge_trial_rejections(
+ measurement, event, factors, emissions, score_floor=2.7, group_count=3)
+ assert np.array_equal(rejected, rejected_with_floor)
+ assert np.array_equal(rejected, rejected_three)
+ assert np.array_equal(np.isfinite(ceiling), np.isfinite(ceiling_three))
assert rejected.shape == starts.shape
incomplete = ~np.all(np.isfinite(factors) & np.isfinite(emissions), axis=1)
assert not np.any(rejected & incomplete)
+ assert not np.any(np.isfinite(ceiling) & incomplete)
tested += int((~incomplete).sum())
unsupported += int(incomplete.sum())
for i in np.flatnonzero(rejected):
@@ -58,7 +66,15 @@ def test_real_ridge_rejections_have_supported_failing_witnesses():
assert np.isfinite(fit["score"])
assert -fit["score"] > 3*sigma
witnessed += 1
- assert tested > 100 and witnessed > 0 and unsupported > 0
+ for i in np.flatnonzero(np.isfinite(ceiling)):
+ fit = _fit_ridge_uncached(track, observer, measurement, starts[i],
+ event["f0_range_hz"], 343., sigma, fraction,
+ acoustic_result=(factors[i], emissions[i]))
+ assert fit["support_fraction"] == 1. and np.isfinite(fit["score"])
+ assert 1+fit["score"]/(3*sigma) <= ceiling[i] < .7
+ assert 3+fit["score"]/(3*sigma) <= ceiling_three[i] < 2.7
+ floor_witnessed += 1
+ assert tested > 100 and witnessed > 0 and floor_witnessed > 0 and unsupported > 0
def test_real_candidate_scores_match_full_fits_with_screening(monkeypatch):
@@ -97,14 +113,15 @@ def test_real_candidate_scores_match_full_fits_with_screening(monkeypatch):
original = screen.ridge_trial_rejections
with monkeypatch.context() as patch:
patch.setattr(screen, "ridge_trial_rejections",
- lambda measurement, event, factors, emissions: np.zeros(len(factors), dtype=bool))
+ lambda measurement, event, factors, emissions, **kwargs:
+ (np.zeros(len(factors), dtype=bool), np.full(len(factors), np.nan)))
baseline = run()
witnessed = 0
later_witnessed = 0
- def checked(measurement, event, factors, emissions):
+ def checked(measurement, event, factors, emissions, **kwargs):
nonlocal witnessed, later_witnessed
- rejected = original(measurement, event, factors, emissions)
+ rejected, ceilings = original(measurement, event, factors, emissions, **kwargs)
sigma = _frequency_scale(event)
fraction = event.get("quality", {}).get("source_drift_fraction", .02)
for i in np.flatnonzero(rejected):
@@ -115,7 +132,7 @@ def test_real_candidate_scores_match_full_fits_with_screening(monkeypatch):
witnessed += 1
if measurement is not current_first_measurement:
later_witnessed += 1
- return rejected
+ return rejected, ceilings
with monkeypatch.context() as patch:
patch.setattr(screen, "ridge_trial_rejections", checked)
diff --git a/tests/test_real_dual_provenance.py b/tests/test_real_dual_provenance.py
index 86de8b0..d062a97 100644
--- a/tests/test_real_dual_provenance.py
+++ b/tests/test_real_dual_provenance.py
@@ -51,3 +51,25 @@ def test_real_bound_methods_and_independent_verifier():
invalid["scoring_assessment"][field] = value
assert not bounded_assessment(invalid)
assert not verifier.bounded_finite_clock_assessment(dict(invalid, finite_best=False), len(events))
+
+ singleton_candidates = [c for c in candidates if len(c["event_indices"]) == 1][:40]
+ track_ids = sorted({entry["flight_id"] for c in singleton_candidates for entry in c["passes"]})
+ singleton_tracks = {track.id: track for track in FlightIndex(fixture["real_index"]).lookup(track_ids)}
+ for candidate in singleton_candidates:
+ if any(entry["flight_id"] not in singleton_tracks for entry in candidate["passes"]):
+ continue
+ row = score_candidate(candidate, point, singleton_tracks, events, measurements,
+ mode="robust_ridge", score_floor=.99)
+ assessment = row["scoring_assessment"]
+ if assessment.get("acoustic_floor_pruned_trials", 0):
+ break
+ else:
+ pytest.fail("No retention-floor acoustic bound in the real candidate sample")
+ assert assessment["unknown_score_bound_method"] == "nonnegative_loss_validated_fits_and_acoustic_floor_dual"
+ assert bounded_assessment(row)
+ assert verifier.bounded_finite_clock_assessment(dict(row, finite_best=False), len(events))
+ for value in (True, 0, assessment["bounded_unknown_trials"]-assessment["bounded_unknown_rejections"]+1):
+ invalid = copy.deepcopy(row)
+ invalid["scoring_assessment"]["acoustic_floor_pruned_trials"] = value
+ assert not bounded_assessment(invalid)
+ assert not verifier.bounded_finite_clock_assessment(dict(invalid, finite_best=False), len(events))