Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

[skip ci] Prepare bounded Top-K caller with point-local ordering

Cinna Davise8c1390bb414
diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.py
index ab20b9d..6b3dcbb 100644
--- a/src/dopplergeist/tiled_search.py
+++ b/src/dopplergeist/tiled_search.py
@@ -67,6 +67,40 @@ def _key(row):
tuple(seed["position"]), seed["recording_start_time_s"])
+class _RetainedScoreFloor:
+ """A lower bound on the kth score from distinct, assessed modes only."""
+
+ def __init__(self, top_k, initial=None):
+ self.top_k, self.initial = top_k, initial
+ self.scores = {}
+ self.value = initial
+
+ def observe(self, row):
+ if (row.get("core_assessment_complete") is not True
+ or row.get("score_kind") != "matched_group_support_minus_normalized_residual"
+ or not math.isfinite(row["score"])):
+ return
+ identity, score = identify_mode(row), row["score"]
+ if (identity not in self.scores and len(self.scores) == self.top_k
+ and score <= min(self.scores.values())):
+ return
+ self.scores[identity] = max(score, self.scores.get(identity, -math.inf))
+ if len(self.scores) > self.top_k:
+ del self.scores[min(self.scores, key=self.scores.get)]
+ if len(self.scores) == self.top_k:
+ local = min(self.scores.values())
+ self.value = local if self.initial is None else max(local, self.initial)
+
+
+def _retained_score_floor(rows, top_k):
+ # merge_candidates already returned distinct, ordered modes, at most top_k.
+ if len(rows) != top_k or any(row.get("core_assessment_complete") is not True
+ or row.get("score_kind") != "matched_group_support_minus_normalized_residual"
+ or not math.isfinite(row["score"]) for row in rows):
+ return None
+ return min(row["score"] for row in rows)
+
+
def merge_candidates(rows, top_k, *, ledger=None, stage=None, bounds=()):
unique, occurrences = {}, 0
for row in rows:
@@ -155,7 +189,7 @@ def _tracks(indexes, bounds, time_range, budget):
@_cache_ridge_fits
-def _score(points, tracks, events, measurements, budget, parameters, filters, start_window=None, *, ledger=None, stage=None):
+def _score(points, tracks, events, measurements, budget, parameters, filters, start_window=None, *, ledger=None, stage=None, score_floor=None):
radius = max(e["slant_range_m"][1] for e in events)
tables, count = [], 0
unsupported = []
@@ -213,8 +247,14 @@ def _score(points, tracks, events, measurements, budget, parameters, filters, st
if upper > budget.max_candidates_per_tile:
raise DataUnavailable("Tile candidate memory budget exceeded; reduce tile_cells")
candidates, counts = pair_candidates(table, events, **kwargs)
+ screened = subset and parameters.get("mode") == "robust_ridge"
+ cutoff = _RetainedScoreFloor(budget.top_k, score_floor) if screened else None
+ if screened:
+ # Find competitive multi-group modes before spending fits on singletons.
+ # All remaining modes still receive a score or a persisted score bound.
+ candidates.sort(key=lambda candidate: (candidate["point_id"], -len(candidate["event_indices"])))
# Attach approach identities before scoring, since legacy rank drops passes.
- ranked, unclassified, rejected = [], 0, 0
+ ranked, unclassified, rejected, retention_pruned = [], 0, 0, 0
bounds = []
trial_counts = dict(evaluated_trials=0, rejected_trials=0, unknown_trials=0, unreported_trial_envelopes=0)
unclassified_assessments = []
@@ -243,6 +283,8 @@ def _score(points, tracks, events, measurements, budget, parameters, filters, st
else:
settings.update(time_origin=(lo + hi) / 2, time_sweep_s=(hi - lo) / 2)
point = lookup[candidate["point_id"]]
+ if cutoff is not None and cutoff.value is not None:
+ settings["score_floor"] = cutoff.value
try:
row = score_candidate(scoring_candidate, point, track_lookup, events, measurements, **settings)
except AcousticSupportError as error:
@@ -253,11 +295,14 @@ def _score(points, tracks, events, measurements, budget, parameters, filters, st
event_indices=candidate.get("event_indices", list(range(len(events)))),
support_failure=_support_error(error))
row["pass_emission_times_s"] = [float(p["emission_time"]) for p in candidate["passes"]]
+ finite_score = row["score"] is not None and math.isfinite(row["score"])
known = (classified_assessment(row) if settings.get("mode") == "robust_ridge" else
- settings.get("time_origin") is not None and settings.get("time_sweep_s", 5) == 0 and math.isfinite(row["score"]))
- row["core_assessment_complete"] = bool(known and math.isfinite(row["score"]))
+ settings.get("time_origin") is not None and settings.get("time_sweep_s", 5) == 0 and finite_score)
+ row["core_assessment_complete"] = bool(known and finite_score)
if settings.get("mode") == "robust_ridge":
assessment = row.get("scoring_assessment")
+ if isinstance(assessment, dict) and assessment.get("unknown_score_bound_method") == "group_count_ceiling":
+ assessment["retention_score_floor"] = settings["score_floor"]
if isinstance(assessment, dict) and all(type(assessment.get(k)) is int and assessment[k] >= 0
for k in ("evaluated_trials", "rejected_trials", "unknown_trials")):
for key in ("evaluated_trials", "rejected_trials", "unknown_trials"):
@@ -276,18 +321,22 @@ def _score(points, tracks, events, measurements, budget, parameters, filters, st
"recording_start_bounds_s": scoring_candidate.get("recording_start_bounds_s"),
"clock_settings": {k: settings.get(k) for k in ("time_origin", "time_sweep_s", "time_step_s")},
"scoring_assessment": row.get("scoring_assessment"), "support_failure": row.get("support_failure"),
- "finite_best": math.isfinite(row["score"])}
+ "finite_best": finite_score}
unclassified_assessments.append(detail)
if bounded_assessment(row):
bounds.append({"bound_id": "bound-" + content_hash(detail), "detail": detail,
"groups_count": len(detail["event_indices"]),
"upper_score": detail["scoring_assessment"]["unknown_score_upper_bound"]})
- if math.isfinite(row["score"]):
+ retention_pruned += detail["scoring_assessment"]["unknown_score_bound_method"] == "group_count_ceiling"
+ if finite_score:
ranked.append(row)
+ if cutoff is not None:
+ cutoff.observe(row)
elif known:
rejected += 1
kept = merge_candidates(ranked, budget.top_k, ledger=ledger, stage=stage, bounds=bounds)
return kept, dict(counts, cpa_rows=count, candidates=len(candidates), unclassified_score_candidates=unclassified,
+ retention_score_pruned_candidates=retention_pruned,
bounded_score_candidates=len(bounds), unbounded_score_candidates=unclassified-len(bounds),
evaluated_rejections=rejected, unsupported_cpa_tracks=unsupported,
unclassified_candidate_assessments=unclassified_assessments, **trial_counts)
@@ -401,7 +450,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
points = tile_points(cells, budget.spacing_m, bounds, terrain, crs) if tracks else pd.DataFrame()
if len(points):
rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
- ledger=ledger, stage=stage)
+ ledger=ledger, stage=stage,
+ score_floor=_retained_score_floor(ranked, budget.top_k))
else:
rows, counts = [], {}
ledger.record(stage, [], [], 0)
@@ -454,7 +504,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
tracks = _tracks(indexes, _expanded_bounds(cells, fine, radius, crs), time_range_s, budget)
stage = f"fine:{seed_index}:{x0}:{y0}"
rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
- ledger=ledger, stage=stage)
+ ledger=ledger, stage=stage,
+ score_floor=_retained_score_floor(ranked, budget.top_k))
_record_support(output, stage, counts, support)
fine_stages += 1
refinement_points_searched += len(points)
diff --git a/tests/data/reference-retention-real.json b/tests/data/reference-retention-real.json
new file mode 100644
index 0000000..2b0eef3
--- /dev/null
+++ b/tests/data/reference-retention-real.json
@@ -0,0 +1,9 @@
+{"description": "Actual original-recording ridge fits against observed aircraft at12 previously retained comparison points; fitted clock and selected hypotheses, not independent geolocation truth. Projection of the existing70-association comparison, retaining computed assessment flags and original mode identities.",
+"rows":[
+{"score":1.1532652232436558,"score_kind":"matched_group_support_minus_normalized_residual","mode_seed":{"event_indices":[10,12],"flight_ids":["a70a89","ace391"],"pass_emission_times_s":[1788900814.93,1788901713.961054],"position":[1855000.0,2209000.0,40.91310994976599,-73.58962516278645,-31.25569725036621],"recording_start_time_s":1788898644.6620934},"mode_id":"mode-b943fc9d51b960cfd3f92a212469f5f026a29ad9e089a72a35bb463595bbe7fb","core_assessment_complete":true},
+{"score":0.8058617967239867,"score_kind":"matched_group_support_minus_normalized_residual","mode_seed":{"event_indices":[12],"flight_ids":["ace391"],"pass_emission_times_s":[1788901713.961054],"position":[1855000.0,2209000.0,40.91310994976599,-73.58962516278645,-31.25569725036621],"recording_start_time_s":1788898645.6620934},"mode_id":"mode-44ae71086619875ff419d5f8aad3271ce62f8d924841dee806beed9ea1fd4682","core_assessment_complete":true},
+{"score":0.7760558425118599,"score_kind":"matched_group_support_minus_normalized_residual","mode_seed":{"event_indices":[9,10,12],"flight_ids":["a1dfeb","a70a89","ace391"],"pass_emission_times_s":[1788900773.3204594,1788900812.93,1788901714.6453073],"position":[1856400.0,2208400.0,40.90497846984017,-73.57501133884975,-27.149030685424805],"recording_start_time_s":1788898644.6620934},"mode_id":"mode-29125b48e33d60c3d238b17a7b70fa30a1fdfe5b79f1415096959d5b7a8165cd","core_assessment_complete":true},
+{"score":0.7605361758866033,"score_kind":"matched_group_support_minus_normalized_residual","mode_seed":{"event_indices":[9,10],"flight_ids":["a1dfeb","a70a89"],"pass_emission_times_s":[1788900761.8929825,1788900814.93],"position":[1855800.0,2206800.0,40.89233862829982,-73.586467860397,-11.81562614440918],"recording_start_time_s":1788898645.6620934},"mode_id":"mode-7c3537ff3058e3f6b017d7164dba9422554ee935eb59f24268ad74823f9058e8","core_assessment_complete":true},
+{"score":0.6919344925189876,"score_kind":"matched_group_support_minus_normalized_residual","mode_seed":{"event_indices":[10],"flight_ids":["a70a89"],"pass_emission_times_s":[1788900814.93],"position":[1855800.0,2206800.0,40.89233862829982,-73.586467860397,-11.81562614440918],"recording_start_time_s":1788898645.6620934},"mode_id":"mode-6d6d3202691942f13de2f9151748294b03866cea18fc3951a81ead8f68bcc91f","core_assessment_complete":true},
+{"score":0.6841396642625563,"score_kind":"matched_group_support_minus_normalized_residual","mode_seed":{"event_indices":[10],"flight_ids":["a70a89"],"pass_emission_times_s":[1788900814.93],"position":[1856200.0,2205400.0,40.8793482944394,-73.58572862508859,-0.9863529205322266],"recording_start_time_s":1788898650.1620934},"mode_id":"mode-618e190ea09e8e7f53f239614440b3618be2263c25aa979b7162236cf86338a6","core_assessment_complete":true}
+]}
diff --git a/tests/test_real_retention_floor.py b/tests/test_real_retention_floor.py
new file mode 100644
index 0000000..0716863
--- /dev/null
+++ b/tests/test_real_retention_floor.py
@@ -0,0 +1,30 @@
+"""Retention-cutoff checks replaying actual fitted modes, not generated flyovers."""
+import json
+from pathlib import Path
+
+from dopplergeist.tiled_search import _RetainedScoreFloor, _retained_score_floor
+
+
+def test_real_modes_keep_the_kth_distinct_score_under_reordering_and_repetition():
+ rows = json.loads((Path(__file__).parent / 'data/reference-retention-real.json').read_text())['rows']
+ for k in (1, 5, 100):
+ for sequence in (rows, list(reversed(rows)), rows * 3):
+ floor = _RetainedScoreFloor(k)
+ seen = {}
+ for row in sequence:
+ floor.observe(dict(row))
+ seen[row['mode_id']] = max(row['score'], seen.get(row['mode_id'], -float('inf')))
+ expected = sorted(seen.values(), reverse=True)[k-1] if len(seen) >= k else None
+ assert floor.value == expected
+ assert len(floor.scores) <= k
+
+
+def test_global_floor_requires_the_retained_mode_count():
+ rows = json.loads((Path(__file__).parent / 'data/reference-retention-real.json').read_text())['rows']
+ assert _retained_score_floor(rows, len(rows)) == min(row['score'] for row in rows)
+ assert _retained_score_floor(rows, 100) is None
+ global_floor = _retained_score_floor(rows, len(rows))
+ local = _RetainedScoreFloor(len(rows), global_floor)
+ for row in rows:
+ local.observe(dict(row))
+ assert local.value >= global_floor