Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
[skip ci] Prepare bounded Top-K caller with point-local ordering
Cinna Davise8c1390bb414
diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.pyindex ab20b9d..6b3dcbb 100644--- a/src/dopplergeist/tiled_search.py+++ b/src/dopplergeist/tiled_search.py@@ -67,6 +67,40 @@ def _key(row):tuple(seed["position"]), seed["recording_start_time_s"])+class _RetainedScoreFloor:+ """A lower bound on the kth score from distinct, assessed modes only."""++ def __init__(self, top_k, initial=None):+ self.top_k, self.initial = top_k, initial+ self.scores = {}+ self.value = initial++ def observe(self, row):+ if (row.get("core_assessment_complete") is not True+ or row.get("score_kind") != "matched_group_support_minus_normalized_residual"+ or not math.isfinite(row["score"])):+ return+ identity, score = identify_mode(row), row["score"]+ if (identity not in self.scores and len(self.scores) == self.top_k+ and score <= min(self.scores.values())):+ return+ self.scores[identity] = max(score, self.scores.get(identity, -math.inf))+ if len(self.scores) > self.top_k:+ del self.scores[min(self.scores, key=self.scores.get)]+ if len(self.scores) == self.top_k:+ local = min(self.scores.values())+ self.value = local if self.initial is None else max(local, self.initial)+++def _retained_score_floor(rows, top_k):+ # merge_candidates already returned distinct, ordered modes, at most top_k.+ if len(rows) != top_k or any(row.get("core_assessment_complete") is not True+ or row.get("score_kind") != "matched_group_support_minus_normalized_residual"+ or not math.isfinite(row["score"]) for row in rows):+ return None+ return min(row["score"] for row in rows)++def merge_candidates(rows, top_k, *, ledger=None, stage=None, bounds=()):unique, occurrences = {}, 0for row in rows:@@ -155,7 +189,7 @@ def _tracks(indexes, bounds, time_range, budget):@_cache_ridge_fits-def _score(points, tracks, events, measurements, budget, parameters, filters, start_window=None, *, ledger=None, stage=None):+def _score(points, tracks, events, measurements, budget, parameters, filters, start_window=None, *, ledger=None, stage=None, score_floor=None):radius = max(e["slant_range_m"][1] for e in events)tables, count = [], 0unsupported = []@@ -213,8 +247,14 @@ def _score(points, tracks, events, measurements, budget, parameters, filters, stif upper > budget.max_candidates_per_tile:raise DataUnavailable("Tile candidate memory budget exceeded; reduce tile_cells")candidates, counts = pair_candidates(table, events, **kwargs)+ screened = subset and parameters.get("mode") == "robust_ridge"+ cutoff = _RetainedScoreFloor(budget.top_k, score_floor) if screened else None+ if screened:+ # Find competitive multi-group modes before spending fits on singletons.+ # All remaining modes still receive a score or a persisted score bound.+ candidates.sort(key=lambda candidate: (candidate["point_id"], -len(candidate["event_indices"])))# Attach approach identities before scoring, since legacy rank drops passes.- ranked, unclassified, rejected = [], 0, 0+ ranked, unclassified, rejected, retention_pruned = [], 0, 0, 0bounds = []trial_counts = dict(evaluated_trials=0, rejected_trials=0, unknown_trials=0, unreported_trial_envelopes=0)unclassified_assessments = []@@ -243,6 +283,8 @@ def _score(points, tracks, events, measurements, budget, parameters, filters, stelse:settings.update(time_origin=(lo + hi) / 2, time_sweep_s=(hi - lo) / 2)point = lookup[candidate["point_id"]]+ if cutoff is not None and cutoff.value is not None:+ settings["score_floor"] = cutoff.valuetry:row = score_candidate(scoring_candidate, point, track_lookup, events, measurements, **settings)except AcousticSupportError as error:@@ -253,11 +295,14 @@ def _score(points, tracks, events, measurements, budget, parameters, filters, stevent_indices=candidate.get("event_indices", list(range(len(events)))),support_failure=_support_error(error))row["pass_emission_times_s"] = [float(p["emission_time"]) for p in candidate["passes"]]+ finite_score = row["score"] is not None and math.isfinite(row["score"])known = (classified_assessment(row) if settings.get("mode") == "robust_ridge" else- settings.get("time_origin") is not None and settings.get("time_sweep_s", 5) == 0 and math.isfinite(row["score"]))- row["core_assessment_complete"] = bool(known and math.isfinite(row["score"]))+ settings.get("time_origin") is not None and settings.get("time_sweep_s", 5) == 0 and finite_score)+ row["core_assessment_complete"] = bool(known and finite_score)if settings.get("mode") == "robust_ridge":assessment = row.get("scoring_assessment")+ if isinstance(assessment, dict) and assessment.get("unknown_score_bound_method") == "group_count_ceiling":+ assessment["retention_score_floor"] = settings["score_floor"]if isinstance(assessment, dict) and all(type(assessment.get(k)) is int and assessment[k] >= 0for k in ("evaluated_trials", "rejected_trials", "unknown_trials")):for key in ("evaluated_trials", "rejected_trials", "unknown_trials"):@@ -276,18 +321,22 @@ def _score(points, tracks, events, measurements, budget, parameters, filters, st"recording_start_bounds_s": scoring_candidate.get("recording_start_bounds_s"),"clock_settings": {k: settings.get(k) for k in ("time_origin", "time_sweep_s", "time_step_s")},"scoring_assessment": row.get("scoring_assessment"), "support_failure": row.get("support_failure"),- "finite_best": math.isfinite(row["score"])}+ "finite_best": finite_score}unclassified_assessments.append(detail)if bounded_assessment(row):bounds.append({"bound_id": "bound-" + content_hash(detail), "detail": detail,"groups_count": len(detail["event_indices"]),"upper_score": detail["scoring_assessment"]["unknown_score_upper_bound"]})- if math.isfinite(row["score"]):+ retention_pruned += detail["scoring_assessment"]["unknown_score_bound_method"] == "group_count_ceiling"+ if finite_score:ranked.append(row)+ if cutoff is not None:+ cutoff.observe(row)elif known:rejected += 1kept = merge_candidates(ranked, budget.top_k, ledger=ledger, stage=stage, bounds=bounds)return kept, dict(counts, cpa_rows=count, candidates=len(candidates), unclassified_score_candidates=unclassified,+ retention_score_pruned_candidates=retention_pruned,bounded_score_candidates=len(bounds), unbounded_score_candidates=unclassified-len(bounds),evaluated_rejections=rejected, unsupported_cpa_tracks=unsupported,unclassified_candidate_assessments=unclassified_assessments, **trial_counts)@@ -401,7 +450,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outpoints = tile_points(cells, budget.spacing_m, bounds, terrain, crs) if tracks else pd.DataFrame()if len(points):rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,- ledger=ledger, stage=stage)+ ledger=ledger, stage=stage,+ score_floor=_retained_score_floor(ranked, budget.top_k))else:rows, counts = [], {}ledger.record(stage, [], [], 0)@@ -454,7 +504,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outtracks = _tracks(indexes, _expanded_bounds(cells, fine, radius, crs), time_range_s, budget)stage = f"fine:{seed_index}:{x0}:{y0}"rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,- ledger=ledger, stage=stage)+ ledger=ledger, stage=stage,+ score_floor=_retained_score_floor(ranked, budget.top_k))_record_support(output, stage, counts, support)fine_stages += 1refinement_points_searched += len(points)diff --git a/tests/data/reference-retention-real.json b/tests/data/reference-retention-real.jsonnew file mode 100644index 0000000..2b0eef3--- /dev/null+++ b/tests/data/reference-retention-real.json@@ -0,0 +1,9 @@+{"description": "Actual original-recording ridge fits against observed aircraft at12 previously retained comparison points; fitted clock and selected hypotheses, not independent geolocation truth. Projection of the existing70-association comparison, retaining computed assessment flags and original mode identities.",+"rows":[+{"score":1.1532652232436558,"score_kind":"matched_group_support_minus_normalized_residual","mode_seed":{"event_indices":[10,12],"flight_ids":["a70a89","ace391"],"pass_emission_times_s":[1788900814.93,1788901713.961054],"position":[1855000.0,2209000.0,40.91310994976599,-73.58962516278645,-31.25569725036621],"recording_start_time_s":1788898644.6620934},"mode_id":"mode-b943fc9d51b960cfd3f92a212469f5f026a29ad9e089a72a35bb463595bbe7fb","core_assessment_complete":true},+{"score":0.8058617967239867,"score_kind":"matched_group_support_minus_normalized_residual","mode_seed":{"event_indices":[12],"flight_ids":["ace391"],"pass_emission_times_s":[1788901713.961054],"position":[1855000.0,2209000.0,40.91310994976599,-73.58962516278645,-31.25569725036621],"recording_start_time_s":1788898645.6620934},"mode_id":"mode-44ae71086619875ff419d5f8aad3271ce62f8d924841dee806beed9ea1fd4682","core_assessment_complete":true},+{"score":0.7760558425118599,"score_kind":"matched_group_support_minus_normalized_residual","mode_seed":{"event_indices":[9,10,12],"flight_ids":["a1dfeb","a70a89","ace391"],"pass_emission_times_s":[1788900773.3204594,1788900812.93,1788901714.6453073],"position":[1856400.0,2208400.0,40.90497846984017,-73.57501133884975,-27.149030685424805],"recording_start_time_s":1788898644.6620934},"mode_id":"mode-29125b48e33d60c3d238b17a7b70fa30a1fdfe5b79f1415096959d5b7a8165cd","core_assessment_complete":true},+{"score":0.7605361758866033,"score_kind":"matched_group_support_minus_normalized_residual","mode_seed":{"event_indices":[9,10],"flight_ids":["a1dfeb","a70a89"],"pass_emission_times_s":[1788900761.8929825,1788900814.93],"position":[1855800.0,2206800.0,40.89233862829982,-73.586467860397,-11.81562614440918],"recording_start_time_s":1788898645.6620934},"mode_id":"mode-7c3537ff3058e3f6b017d7164dba9422554ee935eb59f24268ad74823f9058e8","core_assessment_complete":true},+{"score":0.6919344925189876,"score_kind":"matched_group_support_minus_normalized_residual","mode_seed":{"event_indices":[10],"flight_ids":["a70a89"],"pass_emission_times_s":[1788900814.93],"position":[1855800.0,2206800.0,40.89233862829982,-73.586467860397,-11.81562614440918],"recording_start_time_s":1788898645.6620934},"mode_id":"mode-6d6d3202691942f13de2f9151748294b03866cea18fc3951a81ead8f68bcc91f","core_assessment_complete":true},+{"score":0.6841396642625563,"score_kind":"matched_group_support_minus_normalized_residual","mode_seed":{"event_indices":[10],"flight_ids":["a70a89"],"pass_emission_times_s":[1788900814.93],"position":[1856200.0,2205400.0,40.8793482944394,-73.58572862508859,-0.9863529205322266],"recording_start_time_s":1788898650.1620934},"mode_id":"mode-618e190ea09e8e7f53f239614440b3618be2263c25aa979b7162236cf86338a6","core_assessment_complete":true}+]}diff --git a/tests/test_real_retention_floor.py b/tests/test_real_retention_floor.pynew file mode 100644index 0000000..0716863--- /dev/null+++ b/tests/test_real_retention_floor.py@@ -0,0 +1,30 @@+"""Retention-cutoff checks replaying actual fitted modes, not generated flyovers."""+import json+from pathlib import Path++from dopplergeist.tiled_search import _RetainedScoreFloor, _retained_score_floor+++def test_real_modes_keep_the_kth_distinct_score_under_reordering_and_repetition():+ rows = json.loads((Path(__file__).parent / 'data/reference-retention-real.json').read_text())['rows']+ for k in (1, 5, 100):+ for sequence in (rows, list(reversed(rows)), rows * 3):+ floor = _RetainedScoreFloor(k)+ seen = {}+ for row in sequence:+ floor.observe(dict(row))+ seen[row['mode_id']] = max(row['score'], seen.get(row['mode_id'], -float('inf')))+ expected = sorted(seen.values(), reverse=True)[k-1] if len(seen) >= k else None+ assert floor.value == expected+ assert len(floor.scores) <= k+++def test_global_floor_requires_the_retained_mode_count():+ rows = json.loads((Path(__file__).parent / 'data/reference-retention-real.json').read_text())['rows']+ assert _retained_score_floor(rows, len(rows)) == min(row['score'] for row in rows)+ assert _retained_score_floor(rows, 100) is None+ global_floor = _retained_score_floor(rows, len(rows))+ local = _RetainedScoreFloor(len(rows), global_floor)+ for row in rows:+ local.observe(dict(row))+ assert local.value >= global_floor