Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Reuse exact fits and finalist trajectories in the automatic pipeline

Cinna Davis90d95d7ea793
diff --git a/scripts/benchmark_real_discovery_plots.py b/scripts/benchmark_real_discovery_plots.py
new file mode 100644
index 0000000..0495c60
--- /dev/null
+++ b/scripts/benchmark_real_discovery_plots.py
@@ -0,0 +1,101 @@
+"""Compare discovery diagnostic rendering on an existing real-recording run.
+
+The input directory must contain intake/ and discovery/ from discover_audio.py
+or an automatic job. Both implementations render every measured ridge with the
+same full spectral normalization. Every PNG pixel must match. This measures
+diagnostic rendering only, not discovery or end-to-end localization speed.
+"""
+import argparse
+import importlib.util
+import json
+from pathlib import Path
+import statistics
+import subprocess
+import sys
+import time
+
+import matplotlib
+matplotlib.use("Agg")
+from matplotlib.image import imread
+import numpy as np
+
+from dopplergeist import discovery
+from dopplergeist.intake import IntakeRecording
+
+
+def _module(checkout):
+ name = "dopplergeist._baseline_discovery_plots"
+ spec = importlib.util.spec_from_file_location(name, checkout / "src/dopplergeist/discovery.py")
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("recording_directory", type=Path)
+ parser.add_argument("--baseline", type=Path, required=True, help="Unmodified baseline checkout")
+ parser.add_argument("--output", type=Path, required=True, help="New output directory")
+ parser.add_argument("--repeats", type=int, default=3)
+ args = parser.parse_args()
+ if args.repeats < 1:
+ parser.error("--repeats must be positive")
+ intake = args.recording_directory / "intake"
+ metadata = json.loads((intake / "recording.json").read_text())
+ diagnostics = json.loads((args.recording_directory / "discovery/discovery.json").read_text())
+ if metadata["source_sha256"] != diagnostics["source_sha256"]:
+ parser.error("Intake and discovery belong to different source recordings")
+ recording = IntakeRecording(intake / "recording.f32", metadata, {})
+ settings = discovery.DiscoverySettings(**diagnostics["settings"])
+ events = diagnostics["events"]
+ if not events:
+ parser.error("The real recording contains no event diagnostics to compare")
+ group_ids = list(dict.fromkeys(event["harmonic_group_id"] for event in events))
+ if len(group_ids) != len(diagnostics["harmonic_ridges"]):
+ parser.error("Discovery groups and measured harmonic ridges do not agree")
+ groups = {name: [np.asarray(points) for points in rows]
+ for name, rows in zip(group_ids, diagnostics["harmonic_ridges"])}
+ implementations = {"baseline": _module(args.baseline.resolve()), "current": discovery}
+ args.output.mkdir(parents=True, exist_ok=False)
+ samples = {name: [] for name in implementations}
+ reference_directory = None
+ # Load the backend, colormap, and fonts before either timed arm.
+ implementations["baseline"]._plot_event(recording, groups[events[0]["harmonic_group_id"]],
+ events[0], settings, args.output / "warmup.png")
+ for trial in range(args.repeats):
+ order = ("baseline", "current") if trial % 2 == 0 else ("current", "baseline")
+ for name in order:
+ directory = args.output / f"{name}-{trial + 1}"
+ directory.mkdir()
+ started_wall, started_cpu = time.perf_counter(), time.process_time()
+ for event in events:
+ implementations[name]._plot_event(recording, groups[event["harmonic_group_id"]],
+ event, settings, directory / f"{event['event_id']}.png")
+ timing = {"wall_s": time.perf_counter() - started_wall,
+ "cpu_s": time.process_time() - started_cpu}
+ samples[name].append(timing)
+ if reference_directory is None:
+ reference_directory = directory
+ for event in events:
+ filename = f"{event['event_id']}.png"
+ if not np.array_equal(imread(reference_directory / filename), imread(directory / filename)):
+ raise AssertionError(f"Diagnostic pixels changed: {name}, trial {trial + 1}, {filename}")
+ print(json.dumps({"implementation": name, "trial": trial + 1, **timing}), flush=True)
+ summary = {name: {"trials": rows,
+ "median_wall_s": statistics.median(row["wall_s"] for row in rows),
+ "median_cpu_s": statistics.median(row["cpu_s"] for row in rows)}
+ for name, rows in samples.items()}
+ summary.update({"source": metadata, "ridge_count": len(events), "group_count": len(groups),
+ "exact_png_pixels": True,
+ "wall_speedup": summary["baseline"]["median_wall_s"] / summary["current"]["median_wall_s"],
+ "cpu_speedup": summary["baseline"]["median_cpu_s"] / summary["current"]["median_cpu_s"],
+ "baseline_commit": subprocess.check_output(
+ ["git", "-C", str(args.baseline), "rev-parse", "HEAD"], text=True).strip(),
+ "scope": "All event PNGs from real measured ridges; excludes intake, discovery, and equality checks"})
+ (args.output / "summary.json").write_text(json.dumps(summary, indent=2, allow_nan=False) + "\n")
+ print(json.dumps(summary, indent=2, allow_nan=False))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/check_real_batch_cache.py b/scripts/check_real_batch_cache.py
new file mode 100644
index 0000000..39ce95d
--- /dev/null
+++ b/scripts/check_real_batch_cache.py
@@ -0,0 +1,204 @@
+"""Check exact batch-cache behavior using recorded ridges and flight tracks.
+
+This is a software regression check, not a location-accuracy experiment. Input
+mutation checks change copies by one ULP solely to test cache invalidation.
+No recording or trajectory is generated.
+"""
+import argparse
+from copy import deepcopy
+import json
+from pathlib import Path
+import sys
+from types import SimpleNamespace
+
+import numpy as np
+
+from dopplergeist.batch_ridge import first_event_fits
+from dopplergeist.geometry import AcousticSupportError, ecef
+from dopplergeist.matching import (
+ _cache_batch_ridge_fits, _cache_ridge_fits, _frequency_scale,
+ _iter_acoustic_trials, _ridge_fit_cache, prepare_events,
+)
+from dopplergeist.tracks import load_tracks
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--discovery", required=True, type=Path)
+ parser.add_argument("--tracks", required=True, type=Path)
+ parser.add_argument("--candidate", required=True, type=Path)
+ args = parser.parse_args()
+ events, measurements = prepare_events(json.loads(args.discovery.read_text())["events"])
+ saved = json.loads(args.candidate.read_text())
+ point = saved.get("best", saved)
+ start = point["recording_start_time_s"]
+ tracks, _ = load_tracks(args.tracks, time_range=(start - 120, start + max(e["end_s"] for e in events) + 120))
+ aircraft = {identifier.split(":")[0] for identifier in point["flight_ids"]}
+ chosen = [track for track in tracks if track.id.split(":")[0] in aircraft]
+ observer = ecef(point["lat"], point["lon"], point["elev_m"])
+ starts = start + np.linspace(-5, 5, 21)
+ uncached = first_event_fits.__wrapped__
+ inputs = None
+ for event, measurement in zip(events, measurements):
+ for track in chosen:
+ try:
+ batch = next(_iter_acoustic_trials([track], [measurement], observer, starts))
+ except AcousticSupportError:
+ continue
+ if batch is not None and uncached(measurement, event, *batch) is not None:
+ inputs = measurement, event, *batch
+ break
+ if inputs is not None:
+ break
+ if inputs is None:
+ raise RuntimeError("No supported batch exists for the supplied real receiver/flight hypotheses")
+
+ calls = 0
+ checks = []
+
+ def counted(*values):
+ nonlocal calls
+ calls += 1
+ return uncached(*values)
+
+ cached = _cache_batch_ridge_fits(counted)
+
+ def check(name, condition):
+ if not condition:
+ raise AssertionError(name)
+ checks.append(name)
+
+ expected = uncached(*inputs)
+ check("no_context_result", cached(*inputs) == expected)
+ check("no_context_bypass", calls == 1 and _ridge_fit_cache.get() is None)
+
+ @_cache_ridge_fits
+ def check_context():
+ outer = _ridge_fit_cache.get()
+ initial = calls
+ first = cached(*inputs)
+ second = cached(*deepcopy(inputs))
+ check("equal_copies_hit", calls == initial + 1 and first == second == expected)
+ first[0]["score"] = 12345.0
+ second[0]["f0_hz"] = 0.0
+ check("diagnostic_copies_are_independent", cached(*inputs) == expected)
+ check("memory_bound", 0 < outer.batch_bytes + sys.getsizeof(outer.batch_entries) <= outer.batch_limit)
+
+ def alternate(*values):
+ return counted(*values)
+
+ before = calls
+ check("different_function_result", _cache_batch_ridge_fits(alternate)(*inputs) == expected)
+ check("different_function_misses", calls == before + 1)
+
+ for index, name in enumerate(("times", "ridge", "weights", "factors", "emissions")):
+ changed = deepcopy(inputs)
+ array = ((changed[0][0].time, changed[0][1], changed[0][2], changed[2], changed[3])[index])
+ array.flat[0] = np.nextafter(array.flat[0], np.inf)
+ before = calls
+ check(name + "_mutation_result", cached(*changed) == uncached(*changed))
+ check(name + "_mutation_misses", calls == before + 1)
+
+ for name in ("bounds", "sigma", "fraction"):
+ changed = deepcopy(inputs)
+ event = changed[1]
+ if name == "bounds":
+ event["f0_range_hz"] = list(event["f0_range_hz"])
+ event["f0_range_hz"][0] = float(np.nextafter(event["f0_range_hz"][0], np.inf))
+ elif name == "sigma":
+ event.setdefault("quality", {})["frequency_loss_scale_hz"] = float(np.nextafter(_frequency_scale(event), np.inf))
+ else:
+ event.setdefault("quality", {})["source_drift_fraction"] = float(np.nextafter(event["quality"].get("source_drift_fraction", 0.02), np.inf))
+ before = calls
+ check(name + "_mutation_result", cached(*changed) == uncached(*changed))
+ check(name + "_mutation_misses", calls == before + 1)
+
+ changed = list(deepcopy(inputs))
+ changed[2] = changed[2].astype(changed[2].dtype.newbyteorder(">"))
+ before = calls
+ check("dtype_result", cached(*changed) == uncached(*changed))
+ check("dtype_misses", calls == before + 1)
+ changed = [*inputs[:2], inputs[2][::-1], inputs[3][::-1]]
+ before = calls
+ check("row_order_result", cached(*changed) == uncached(*changed))
+ check("row_order_misses", calls == before + 1)
+ changed = [*inputs[:2], np.asfortranarray(inputs[2]), np.asfortranarray(inputs[3])]
+ before = calls
+ check("fortran_layout_result", cached(*changed) == uncached(*changed))
+ check("fortran_layout_misses", calls == before + 1)
+ changed = [*inputs[:2], inputs[2].tolist(), inputs[3].tolist()]
+ before = calls
+ check("custom_representation_result", cached(*changed) == cached(*changed) == expected)
+ check("custom_representation_bypasses", calls == before + 2)
+ before = calls
+ check("measurement_iterator_result", cached(iter(inputs[0]), *inputs[1:]) == expected)
+ check("measurement_iterator_bypasses", calls == before + 1)
+ changed = list(deepcopy(inputs))
+ measurement = list(changed[0])
+ measurement[0] = SimpleNamespace(time=measurement[0].time)
+ changed[0] = measurement
+ before = calls
+ check("custom_time_container_result", cached(*changed) == cached(*changed) == expected)
+ check("custom_time_container_bypasses", calls == before + 2)
+
+ missing = [*inputs[:2], inputs[2][:0], inputs[3][:0]]
+ before = calls
+ entries = len(outer.batch_entries)
+ check("fallback_none", cached(*missing) is None and cached(*missing) is None)
+ check("fallback_not_cached", calls == before + 2 and len(outer.batch_entries) == entries)
+ malformed = deepcopy(inputs)
+ malformed[0][0].time = malformed[0][0].time[:2]
+ before = calls
+ for _ in range(2):
+ try:
+ cached(*malformed)
+ except ValueError:
+ pass
+ else:
+ raise AssertionError("Malformed measurement did not raise")
+ check("exceptions_not_cached", calls == before + 2 and len(outer.batch_entries) == entries)
+
+ @_cache_ridge_fits
+ def nested():
+ check("nested_context_is_independent", _ridge_fit_cache.get() is not outer)
+ before = calls
+ check("nested_fit_result", cached(*inputs) == expected)
+ check("nested_fit_misses", calls == before + 1)
+ raise RuntimeError("Check context restoration after an exception")
+
+ try:
+ nested()
+ except RuntimeError:
+ pass
+ check("nested_context_restored", _ridge_fit_cache.get() is outer)
+ before = calls
+ check("outer_fit_survives", cached(*inputs) == expected and calls == before)
+
+ check_context()
+ check("context_reset", _ridge_fit_cache.get() is None)
+
+ def eviction():
+ cache = _ridge_fit_cache.get()
+ first = deepcopy(inputs)
+ second = deepcopy(inputs)
+ second[2][0, 0] = np.nextafter(second[2][0, 0], np.inf)
+ check("eviction_first_result", cached(*first) == uncached(*first))
+ check("eviction_second_result", cached(*second) == uncached(*second))
+ check("entry_limit", len(cache.batch_entries) == 1)
+ before = calls
+ check("evicted_fit_recomputed", cached(*first) == uncached(*first) and calls == before + 1)
+ cache.batch_entries.clear()
+ cache.batch_bytes = 0
+ cache.batch_limit = 1
+ before = calls
+ check("oversized_fits_bypass", cached(*first) == cached(*first) == expected and calls == before + 2)
+ check("oversized_fits_not_stored", len(cache.batch_entries) == cache.batch_bytes == 0)
+
+ _cache_ridge_fits(eviction, max_entries=1)()
+ print(json.dumps({"checks": checks, "passed": len(checks), "recorded_groups": len(events),
+ "selected_event": inputs[1].get("event_id"), "clock_rows": len(inputs[2]),
+ "scope": "real-input cache regression; no location-accuracy claim"}, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/check_real_ceiling_cache.py b/scripts/check_real_ceiling_cache.py
new file mode 100644
index 0000000..ce224ac
--- /dev/null
+++ b/scripts/check_real_ceiling_cache.py
@@ -0,0 +1,161 @@
+"""Check ceiling reuse against the original validator using recorded ridge fits.
+
+Metadata mutations exercise cache invalidation and conservative unknown-support
+handling. They are software checks, not generated recordings or field evidence.
+"""
+import argparse
+from copy import deepcopy
+import json
+from pathlib import Path
+
+import numpy as np
+
+from dopplergeist import search
+from dopplergeist.batch_ridge import first_event_fits
+from dopplergeist.geometry import AcousticSupportError, ecef
+from dopplergeist.matching import (
+ _cache_ridge_fits, _frequency_scale, _iter_acoustic_trials, _ridge_fit_cache, prepare_events,
+)
+from dopplergeist.tracks import load_tracks
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--discovery", required=True, type=Path)
+ parser.add_argument("--tracks", required=True, type=Path)
+ parser.add_argument("--candidate", required=True, type=Path)
+ args = parser.parse_args()
+ events, measurements = prepare_events(json.loads(args.discovery.read_text())["events"])
+ saved = json.loads(args.candidate.read_text())
+ point = saved.get("best", saved)
+ start = point["recording_start_time_s"]
+ tracks, _ = load_tracks(args.tracks, time_range=(start - 120, start + max(e["end_s"] for e in events) + 120))
+ aircraft = {identifier.split(":")[0] for identifier in point["flight_ids"]}
+ observer = ecef(point["lat"], point["lon"], point["elev_m"])
+ starts = start + np.linspace(-5, 5, 21)
+ fits = None
+ for event, measurement in zip(events, measurements):
+ for track in (track for track in tracks if track.id.split(":")[0] in aircraft):
+ try:
+ batch = next(_iter_acoustic_trials([track], [measurement], observer, starts))
+ except AcousticSupportError:
+ continue
+ if batch is not None:
+ fits = first_event_fits(measurement, event, *batch)
+ if fits is not None:
+ break
+ if fits is not None:
+ break
+ if fits is None:
+ raise RuntimeError("No supported batch exists for the supplied real receiver/flight hypotheses")
+ limit = 3 * _frequency_scale(event)
+ base = (fits[0], fits[0]["score"], fits[0]["f0_hz"], event, limit)
+ checks = []
+ calls = 0
+ original_classify = search._ridge_trial_classification
+
+ def counted(*values):
+ nonlocal calls
+ calls += 1
+ return original_classify(*values)
+
+ def check(name, condition):
+ if not condition:
+ raise AssertionError(name)
+ checks.append(name)
+
+ def same(name, values, miss=None):
+ expected = search._association_event_ceiling_uncached(*values)
+ before = calls
+ actual = search._association_event_ceiling(*values)
+ check(name, actual == expected or (np.isnan(actual) and np.isnan(expected)))
+ if miss is not None:
+ check(name + "_cache_behavior", calls == before + int(miss))
+
+ @_cache_ridge_fits
+ def exercise():
+ outer = _ridge_fit_cache.get()
+ for index, fit in enumerate(fits):
+ values = (fit, fit["score"], fit["f0_hz"], event, limit)
+ same(f"real_clock_{index}", values, True)
+ same(f"copied_clock_{index}", deepcopy(values), False)
+ for field in search._CEILING_FLOAT_FIELDS:
+ changed = deepcopy(base)
+ changed[0][field] = float(np.nextafter(changed[0][field], np.inf))
+ same(field + "_mutation", changed, True)
+ for field in ("sample_count", "independent_sample_count"):
+ changed = deepcopy(base)
+ changed[0][field] += 1
+ same(field + "_mutation", changed, True)
+ for field, value in (("optimizer_success", False), ("support_fraction", 0.0),
+ ("source_span_s", 0.0), ("drift_bound_hz", -1.0),
+ ("rmse_hz", -1.0), ("outlier_fraction", 2.0)):
+ changed = deepcopy(base)
+ changed[0][field] = value
+ same(field + "_invalid", changed, True)
+ check(field + "_conservative", search._association_event_ceiling(*changed) == 1.0)
+ for index, name in ((1, "score_argument"), (2, "frequency_argument"), (4, "limit")):
+ changed = list(deepcopy(base))
+ changed[index] = float(np.nextafter(changed[index], np.inf))
+ same(name + "_mutation", changed, True)
+ for index in range(2):
+ changed = deepcopy(base)
+ changed[3]["f0_range_hz"][index] = float(np.nextafter(changed[3]["f0_range_hz"][index], np.inf))
+ same(f"frequency_bound_{index}_mutation", changed, True)
+ changed = deepcopy(base)
+ changed[3].setdefault("quality", {})["source_drift_fraction"] = float(np.nextafter(
+ changed[3]["quality"].get("source_drift_fraction", 0.02), np.inf))
+ same("drift_fraction_mutation", changed, True)
+ for value in (np.float32(base[1]), np.float64(base[1]), np.longdouble(base[1])):
+ changed = list(deepcopy(base))
+ changed[0]["score"] = changed[1] = value
+ same(type(value).__name__ + "_bypass", changed, True)
+ same(type(value).__name__ + "_repeated_bypass", changed, True)
+ for field in ("support_fraction", "sample_count", "source_span_s"):
+ changed = deepcopy(base)
+ del changed[0][field]
+ same(field + "_missing", changed, True)
+ same(field + "_missing_bypass", changed, True)
+ changed = list(deepcopy(base))
+ changed[4] = None
+ entries = len(outer.ceiling_entries)
+ for _ in range(2):
+ try:
+ search._association_event_ceiling(*changed)
+ except TypeError:
+ checks.append("invalid_limit_exception")
+ else:
+ raise AssertionError("Invalid limit did not raise the original error")
+ check("exceptions_not_cached", len(outer.ceiling_entries) == entries)
+
+ @_cache_ridge_fits
+ def nested():
+ check("nested_context_is_independent", _ridge_fit_cache.get() is not outer)
+ same("nested_first_lookup", base, True)
+ nested()
+ check("outer_context_restored", _ridge_fit_cache.get() is outer)
+ same("outer_cached_fit_survives", base, False)
+
+ def eviction():
+ outer = _ridge_fit_cache.get()
+ same("entry_limit_first", base, True)
+ changed = list(deepcopy(base))
+ changed[4] = float(np.nextafter(changed[4], np.inf))
+ same("entry_limit_second", changed, True)
+ check("entry_limit", len(outer.ceiling_entries) == 1)
+ same("evicted_entry_recomputed", base, True)
+
+ search._ridge_trial_classification = counted
+ try:
+ same("outside_context", base, True)
+ exercise()
+ check("context_reset", _ridge_fit_cache.get() is None)
+ _cache_ridge_fits(eviction, max_entries=1)()
+ finally:
+ search._ridge_trial_classification = original_classify
+ print(json.dumps({"passed": len(checks), "checks": checks, "event": event["event_id"],
+ "real_clock_fits": len(fits), "scope": "real-input ceiling cache regression"}, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/dopplergeist/automatic.py b/src/dopplergeist/automatic.py
index 0ed79b9..2cb7a6d 100644
--- a/src/dopplergeist/automatic.py
+++ b/src/dopplergeist/automatic.py
@@ -443,11 +443,40 @@ def _compare_retained_spectra(recording, rows, prepared_events, indexes, config,
return report
+def _refinement_tracks(indexes, needed, bounds, time_range, sample_limit):
+ """Load only a finalist's whole segments within the original query envelope."""
+ from .data_access import DataUnavailable, FlightIndex
+
+ tracks = {}
+ for index in indexes:
+ # Preserve custom query adapters and their filtering behavior. The native
+ # index can restrict names in SQL before decompressing unrelated flights.
+ named = type(index) is FlightIndex
+ source = (index.lookup(needed, bounds=bounds, time_range=time_range,
+ max_tracks=max(1, len(needed)), max_samples=sample_limit)
+ if named else index.query(bounds, time_range))
+ try:
+ for track in source:
+ if track.id in needed:
+ tracks[track.id] = track
+ if sum(len(item.time) for item in tracks.values()) > sample_limit:
+ raise ValueError("Refinement trajectory sample budget exceeded")
+ except DataUnavailable as error:
+ if named and str(error) == "Flight lookup sample budget exceeded":
+ raise ValueError("Refinement trajectory sample budget exceeded") from error
+ raise
+ return tracks
+
+
def _decide(search, events, prepared_events, measurements, indexes, terrain, state, output, config, finish, decision_progress, *, recording=None):
# The confidence producer supplies this adapter; missing decision machinery
# can never turn the highest raw score into a supported location.
from .confidence import decide_location
- from .matching import profile_candidate, refine_candidate
+ from .matching import _cache_ridge_fits, profile_candidate, refine_candidate
+ # Each finalist and the complete sensitivity profile own an exact-input
+ # cache. Profile's internal refinements share its scope; other calls do not.
+ refine_candidate = _cache_ridge_fits(refine_candidate)
+ profile_candidate = _cache_ridge_fits(profile_candidate)
raw = list(search.get("ranked", []))
refined, refinement_failures = [], []
window = state["recording"]["time_window"]
@@ -494,16 +523,10 @@ def _decide(search, events, prepared_events, measurements, indexes, terrain, sta
margin = max_range / 60000.0 + 0.3
bounds = (row["lon"]-margin, row["lat"]-margin, row["lon"]+margin, row["lat"]+margin)
needed = set(row["flight_ids"])
- tracks = {}
lo = start_window[0] + min(e["start_s"] for e in prepared_events) - max_range / 300 - 120
hi = start_window[1] + max(e["end_s"] for e in prepared_events) + 120
- for index in indexes:
- for track in index.query(bounds, (lo, hi)):
- if track.id in needed:
- tracks[track.id] = track
- sample_limit = config.get("search_budget", {}).get("max_track_samples_per_tile", 500000)
- if sum(len(item.time) for item in tracks.values()) > sample_limit:
- raise ValueError("Refinement trajectory sample budget exceeded")
+ sample_limit = config.get("search_budget", {}).get("max_track_samples_per_tile", 500000)
+ tracks = _refinement_tracks(indexes, needed, bounds, (lo, hi), sample_limit)
if needed - tracks.keys():
raise ValueError("Candidate flight segments are unavailable for refinement")
row = dict(row, altitude_basis={flight: sorted(set(getattr(track, "altitude_basis", ["unspecified"]))) for flight, track in tracks.items()})
diff --git a/src/dopplergeist/batch_ridge.py b/src/dopplergeist/batch_ridge.py
index 78995bd..010f715 100644
--- a/src/dopplergeist/batch_ridge.py
+++ b/src/dopplergeist/batch_ridge.py
@@ -1,9 +1,10 @@
"""Batched robust ridge fits with the scalar model's arithmetic order."""
import numpy as np
-from .matching import _frequency_scale, _ridge_fit_inputs
+from .matching import _cache_batch_ridge_fits, _frequency_scale, _ridge_fit_inputs
+@_cache_batch_ridge_fits
def first_event_fits(measurement, event, factors, emissions):
"""Fit supported clocks together, or return None for the ordinary fitter.
diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.py
index ca8c5e9..4c2353b 100644
--- a/src/dopplergeist/data_access.py
+++ b/src/dopplergeist/data_access.py
@@ -496,42 +496,51 @@ class FlightIndex:
except (OSError, ValueError, KeyError, TypeError) as error:
raise DataUnavailable(f"Invalid/truncated Parquet: {parquet}: {error}") from error
+ def _query_scope(self, bounds, time_range):
+ """Use the same conservative envelope for broad and named queries."""
+ west, south, east, north = bounds
+ start, end = time_range
+ current_normalizer = sha256(Path(__file__).with_name("tracks.py"))
+ compatible = (current_normalizer, *_EQUIVALENT_NORMALIZERS.get(current_normalizer, ()))
+ if (not self.report.get("continuous_geometry_bounds") or
+ self.report.get("normalizer_sha256") not in compatible):
+ # Old indexes used sample-only bounds or different derivatives.
+ # Disable unsafe spatial pruning, preserving explicit budgets.
+ return "", "s.start<=? AND s.end>=?", (end, start)
+ return (" JOIN spatial r ON s.id=r.id",
+ "r.west<=? AND r.east>=? AND r.south<=? AND r.north>=? AND s.start<=? AND s.end>=?",
+ (east, west, north, south, end, start))
+
def query(self, bounds, time_range):
"""Yield complete segments touching the query, retaining gaps and labels.
Callers expand receiver bounds for acoustic range and event duration.
An R-tree rectangle can return harmless extra tracks, never clipped ones.
"""
- west, south, east, north = bounds
- start, end = time_range
+ join, predicate, params = self._query_scope(bounds, time_range)
with self.connect() as db:
- sql = """SELECT s.name,s.payload,s.payload_hash FROM segments s JOIN spatial r ON s.id=r.id
- WHERE r.west<=? AND r.east>=? AND r.south<=? AND r.north>=?
- AND s.start<=? AND s.end>=? ORDER BY s.name"""
- params = (east, west, north, south, end, start)
- current_normalizer = sha256(Path(__file__).with_name("tracks.py"))
- compatible = (current_normalizer, *_EQUIVALENT_NORMALIZERS.get(current_normalizer, ()))
- if (not self.report.get("continuous_geometry_bounds") or
- self.report.get("normalizer_sha256") not in compatible):
- # Old indexes used sample-only bounds or different derivatives.
- # Disable unsafe spatial pruning, preserving explicit budgets.
- sql = "SELECT name,payload,payload_hash FROM segments WHERE start<=? AND end>=? ORDER BY name"
- params = (end, start)
+ sql = ("SELECT s.name,s.payload,s.payload_hash FROM segments s" + join
+ + " WHERE " + predicate + " ORDER BY s.name")
for name, payload, expected in db.execute(sql, params):
if hashlib.sha256(payload).hexdigest() != expected:
raise DataUnavailable(f"Corrupt flight payload: {name}")
frame = pd.DataFrame(json.loads(gzip.decompress(payload)))
yield _frame_track(name, frame)
- def lookup(self, identifiers, *, max_tracks=2000, max_samples=500000):
+ def lookup(self, identifiers, *, max_tracks=2000, max_samples=500000,
+ bounds=None, time_range=None):
"""Yield named whole segments without scanning unrelated trajectories.
Missing names are omitted; callers must compare requested and returned
IDs. Limits reject excess work explicitly, never truncate a trajectory.
+ Supply both bounds and time_range to apply the same envelope as query,
+ including its conservative fallback for older geometry indexes.
This lookup establishes no spatial, temporal or receiver completeness.
"""
if any(type(value) is not int or value <= 0 for value in (max_tracks, max_samples)):
raise ValueError("Flight lookup limits must be positive integers")
+ if (bounds is None) != (time_range is None):
+ raise ValueError("Flight lookup bounds and time_range must be supplied together")
names = set()
for name in identifiers:
if not isinstance(name, str) or not name:
@@ -540,14 +549,18 @@ class FlightIndex:
if len(names) > max_tracks:
raise DataUnavailable("Flight lookup identifier budget exceeded")
names, samples = sorted(names), 0
+ join, predicate, scope_params = ("", "", ())
+ if bounds is not None:
+ join, predicate, scope_params = self._query_scope(bounds, time_range)
with self.connect() as db:
# Stay below SQLite's conservative bound on bound SQL parameters.
for first in range(0, len(names), 500):
batch = names[first:first+500]
placeholders = ",".join("?" for _ in batch)
- sql = ("SELECT name,payload,payload_hash FROM segments WHERE name IN ("
- + placeholders + ") ORDER BY name")
- for name, payload, expected in db.execute(sql, batch):
+ sql = ("SELECT s.name,s.payload,s.payload_hash FROM segments s" + join
+ + " WHERE s.name IN (" + placeholders + ")"
+ + (" AND " + predicate if predicate else "") + " ORDER BY s.name")
+ for name, payload, expected in db.execute(sql, (*batch, *scope_params)):
if hashlib.sha256(payload).hexdigest() != expected:
raise DataUnavailable(f"Corrupt flight payload: {name}")
frame = pd.DataFrame(json.loads(gzip.decompress(payload)))
diff --git a/src/dopplergeist/discovery.py b/src/dopplergeist/discovery.py
index 0905839..04dde53 100644
--- a/src/dopplergeist/discovery.py
+++ b/src/dopplergeist/discovery.py
@@ -362,15 +362,22 @@ def _plot_event(recording, group, event, settings, path):
sr = recording.metadata["sample_rate_hz"]
data = np.memmap(recording.samples_path, dtype="<f4", mode="r")
spec = _spectrum(np.array(data[round(first * sr):round(last * sr)]), sr, first, settings)
+ low = min(points[:, 1].min() for points in group)
+ high = max(points[:, 1].max() for points in group)
+ limits = (max(settings.min_frequency_hz, low * 0.9), high * 1.1)
+ # Only visible cells can affect the image. Keep an extra frequency bin on
+ # each side so pcolormesh reconstructs the same boundary-cell coordinates.
+ # Whitening still uses the full spectrum and all measured ridges are kept.
+ lower = max(0, np.searchsorted(spec.frequency, limits[0]) - 1)
+ upper = min(len(spec.frequency), np.searchsorted(spec.frequency, limits[1], side="right") + 1)
+ visible = slice(lower, upper)
fig, ax = plt.subplots(figsize=(10, 4), layout="constrained")
- ax.pcolormesh(spec.time, spec.frequency, spec.contrast, shading="auto", cmap="magma", vmin=0, vmax=6,
+ ax.pcolormesh(spec.time, spec.frequency[visible], spec.contrast[visible], shading="auto", cmap="magma", vmin=0, vmax=6,
rasterized=True)
for points in group:
for segment in np.split(points, np.flatnonzero(np.diff(points[:, 0]) > settings.max_gap_s) + 1):
ax.plot(segment[:, 0], segment[:, 1], color="cyan", lw=0.9)
- low = min(points[:, 1].min() for points in group)
- high = max(points[:, 1].max() for points in group)
- ax.set(xlim=(first, last), ylim=(max(settings.min_frequency_hz, low * 0.9), high * 1.1),
+ ax.set(xlim=(first, last), ylim=limits,
xlabel="Recording-relative time (s)", ylabel="Frequency (Hz)",
title=f"{event['event_id']}: measured ridges; acoustic proposal, identity unconfirmed")
fig.savefig(path, dpi=130)
diff --git a/src/dopplergeist/matching.py b/src/dopplergeist/matching.py
index 88a7ec9..d6f38ab 100644
--- a/src/dopplergeist/matching.py
+++ b/src/dopplergeist/matching.py
@@ -10,11 +10,13 @@ from dataclasses import dataclass
from functools import wraps
import hashlib
from itertools import chain
+import sys
import numpy as np
from pyproj import Transformer
from scipy.optimize import minimize
+from .audio import Spectrum
from .geometry import Track, _EMISSION_ITERATION_TOLERANCE_S, ecef
@@ -53,6 +55,10 @@ class _RidgeFitCache:
self.measurement_entries = OrderedDict()
self.measurement_bytes = 0
self.measurement_limit = 4 * 1024 * 1024
+ self.batch_entries = OrderedDict()
+ self.batch_bytes = 0
+ self.batch_limit = 16 * 1024 * 1024
+ self.ceiling_entries = OrderedDict()
def prepared_measurement(self, measurement, sigma_hz, drift_fraction, key):
"""Reuse selected samples only after checking their current content key."""
@@ -163,6 +169,76 @@ def _cache_ridge_fits(function, *, max_entries=4096):
return run
+def _cache_batch_ridge_fits(function):
+ """Reuse identical source fits across associations within one scoring call.
+
+ Every lookup checks current input bytes. The cache cannot turn changed
+ measurements, propagation arrays or model parameters into an old result.
+ Successful diagnostics contain immutable scalars; independent dictionary
+ copies prevent callers from modifying the saved fit. Custom representations
+ retain the ordinary fitter and its validation behavior.
+ """
+ @wraps(function)
+ def run(measurement, event, factors, emissions):
+ cache = _ridge_fit_cache.get()
+ if cache is None or cache.batch_limit <= 0 or cache.max_entries <= 0:
+ return function(measurement, event, factors, emissions)
+ if (type(measurement) not in (list, tuple) or len(measurement) != 4
+ or type(event) is not dict or type(event.get("quality", {})) is not dict):
+ return function(measurement, event, factors, emissions)
+ spec, ridge, weight, _ = measurement
+ if type(spec) not in (RidgeTimes, Spectrum):
+ return function(measurement, event, factors, emissions)
+ arrays = (spec.time, ridge, weight, factors, emissions)
+ if any(type(value) is not np.ndarray or value.dtype.kind not in "biuf" for value in arrays):
+ return function(measurement, event, factors, emissions)
+ bounds = event.get("f0_range_hz")
+ sigma = _frequency_scale(event)
+ fraction = event.get("quality", {}).get("source_drift_fraction", 0.02)
+ if (type(bounds) not in (list, tuple) or len(bounds) != 2
+ or any(type(value) not in (int, float) for value in (sigma, fraction, *bounds))):
+ return function(measurement, event, factors, emissions)
+ parameters = tuple(_fit_argument_key(value) for value in (sigma, fraction, bounds))
+ # Equal values can have different NumPy reduction orders when their
+ # strides differ. Preserve layout as well as every numeric input byte.
+ content = tuple((value.dtype.str, value.shape, value.strides, value.flags.aligned,
+ hashlib.sha256(value.tobytes()).digest())
+ for value in arrays)
+ key = function, parameters, content
+ if key in cache.batch_entries:
+ cache.batch_entries.move_to_end(key)
+ return [diagnostic.copy() for diagnostic in cache.batch_entries[key][0]]
+ result = function(measurement, event, factors, emissions)
+ if result is not None:
+ # Builtin type tags and literal diagnostic keys already live in the
+ # module. Charge every other key/scalar object to each entry, even
+ # when shared. Check the actual OrderedDict size after insertion
+ # too, since its backing table can grow or retain capacity.
+ def key_size(value):
+ if isinstance(value, type):
+ return 0
+ return sys.getsizeof(value) + (sum(map(key_size, value)) if isinstance(value, tuple) else 0)
+
+ size = (key_size(key) + sys.getsizeof((None, None)) + sys.getsizeof(0) + sys.getsizeof(result)
+ + sum(sys.getsizeof(diagnostic)
+ + sum(sys.getsizeof(value) for value in diagnostic.values())
+ for diagnostic in result))
+ if size <= cache.batch_limit:
+ while cache.batch_entries and (
+ cache.batch_bytes + size + sys.getsizeof(cache.batch_entries) > cache.batch_limit
+ or len(cache.batch_entries) >= cache.max_entries):
+ _, (_, removed) = cache.batch_entries.popitem(last=False)
+ cache.batch_bytes -= removed
+ cache.batch_entries[key] = ([diagnostic.copy() for diagnostic in result], size)
+ cache.batch_bytes += size
+ while cache.batch_entries and cache.batch_bytes + sys.getsizeof(cache.batch_entries) > cache.batch_limit:
+ _, (_, removed) = cache.batch_entries.popitem(last=False)
+ cache.batch_bytes -= removed
+ return result
+
+ return run
+
+
def _frequency_scale(event):
quality = event.get("quality", {})
return quality.get("frequency_loss_scale_hz", quality.get("frequency_sigma_hz",
diff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.py
index fb717cb..6b65d10 100644
--- a/src/dopplergeist/search.py
+++ b/src/dopplergeist/search.py
@@ -335,7 +335,49 @@ def _ridge_trial_classification(result, events, association):
+_CEILING_FLOAT_FIELDS = (
+ "support_fraction", "score", "f0_hz", "source_drift_hz", "source_drift_hz_s",
+ "source_reference_emission_time_s", "source_span_s", "drift_bound_hz", "rmse_hz", "outlier_fraction",
+)
+
+
def _association_event_ceiling(diagnostic, score, frequency, event, limit):
+ """Reuse a validated ceiling only for the same complete scalar evidence.
+
+ Fit dictionaries are copied for each association. Content keys preserve
+ mutation checks while avoiding repeated validation of the same clock fit.
+ Noncanonical numeric types retain the original validation and arithmetic.
+ """
+ from .matching import _ridge_fit_cache
+ cache = _ridge_fit_cache.get()
+ if (cache is None or cache.max_entries <= 0 or type(diagnostic) is not dict
+ or type(event) is not dict or type(event.get("quality", {})) is not dict):
+ return _association_event_ceiling_uncached(diagnostic, score, frequency, event, limit)
+ bounds = event.get("f0_range_hz")
+ if type(bounds) not in (list, tuple) or len(bounds) != 2:
+ return _association_event_ceiling_uncached(diagnostic, score, frequency, event, limit)
+ values = tuple(diagnostic.get(name) for name in _CEILING_FLOAT_FIELDS)
+ fraction = event.get("quality", {}).get("source_drift_fraction", 0.02)
+ counts = (diagnostic.get("sample_count"), diagnostic.get("independent_sample_count"))
+ optimizer = diagnostic.get("optimizer_success", True)
+ if (not all(type(value) is float for value in (*values, score, frequency, limit, *bounds, fraction))
+ or not all(type(value) is int for value in counts) or type(optimizer) is not bool):
+ return _association_event_ceiling_uncached(diagnostic, score, frequency, event, limit)
+ key = (_ridge_trial_classification, values, counts, optimizer, score, frequency,
+ tuple(bounds), fraction, limit)
+ entries = cache.ceiling_entries
+ previous = entries.get(key)
+ if previous is not None:
+ entries.move_to_end(key)
+ return previous
+ result = _association_event_ceiling_uncached(diagnostic, score, frequency, event, limit)
+ entries[key] = result
+ if len(entries) > cache.max_entries:
+ entries.popitem(last=False)
+ return result
+
+
+def _association_event_ceiling_uncached(diagnostic, score, frequency, event, limit):
"""Validate one fitted group's contribution before tightening a ceiling."""
single = {"score_kind": "negative_robust_ridge_loss_hz", "score": score,
"event_scores": [score], "f0_hz": [frequency], "event_diagnostics": [diagnostic]}
diff --git a/tests/test_real_batch_ridge.py b/tests/test_real_batch_ridge.py
index 5d9b403..47711eb 100644
--- a/tests/test_real_batch_ridge.py
+++ b/tests/test_real_batch_ridge.py
@@ -71,9 +71,18 @@ def test_real_association_results_match_scalar_fits(recorded, monkeypatch):
import dopplergeist.batch_ridge as batch_ridge
fixture, events, measurements, table, tracks = recorded
- candidates, _ = association_candidates(table, events, max_candidates=20000)
+ options = {}
+ if "recording_start_window_s" in fixture:
+ from dopplergeist.tiled_search import _association_clock_window
+ # Preserve the frozen workload's declared clock domain. Enumerating
+ # every possible clock from a broad CPA table changes that workload
+ # and can exhaust its association budget before this fit check runs.
+ center, tolerance = _association_clock_window(
+ fixture["recording_start_window_s"], fixture.get("scoring", {}).get("time_sweep_s", 5.0))
+ options = {"time_origin": center, "time_tolerance_s": tolerance}
+ candidates, _ = association_candidates(table, events, max_candidates=20000, **options)
selected = sorted(random.Random(20261103).sample(range(len(candidates)), min(100, len(candidates))))
- point = SimpleNamespace(**fixture["point"], x=0., y=0.)
+ point = SimpleNamespace(**{"x": 0., "y": 0., **fixture["point"]})
@_cache_ridge_fits
def run():
diff --git a/tests/test_real_refinement_lookup.py b/tests/test_real_refinement_lookup.py
new file mode 100644
index 0000000..265190e
--- /dev/null
+++ b/tests/test_real_refinement_lookup.py
@@ -0,0 +1,116 @@
+"""Named refinement reads against actual indexed aircraft observations."""
+import json
+import os
+from pathlib import Path
+import shutil
+import sqlite3
+
+import numpy as np
+import pytest
+
+from dopplergeist.automatic import _refinement_tracks
+from dopplergeist.data_access import DataUnavailable, FlightIndex
+
+
[email protected](scope="module")
+def recorded_index():
+ source = os.environ.get("DOPPLERGEIST_REAL_CPA_FIXTURE")
+ if not source:
+ pytest.skip("set DOPPLERGEIST_REAL_CPA_FIXTURE to a recorded-data diagnostic")
+ fixture = json.loads(Path(source).read_text())
+ index = FlightIndex(fixture["real_index"])
+ assert index.report["input_hash"] == fixture["index_input_hash"]
+ bounds, times = fixture["query_bounds"], fixture["query_time_s"]
+ observed = list(index.query(bounds, times))
+ assert observed, "The recorded workload contains no trajectories"
+ assert [track.id for track in observed] == sorted(fixture["track_ids"])
+ return index, bounds, times, observed
+
+
+def same_tracks(actual, expected):
+ assert [track.id for track in actual] == [track.id for track in expected]
+ for left, right in zip(actual, expected):
+ for name in ("time", "lat", "lon", "altitude", "xyz", "relative_time", "velocity"):
+ a, b = getattr(left, name), getattr(right, name)
+ if a is None or b is None:
+ assert a is b
+ else:
+ np.testing.assert_array_equal(a, b, strict=True)
+ np.testing.assert_array_equal(left.spline.c, right.spline.c, strict=True)
+ np.testing.assert_array_equal(left.derivative.c, right.derivative.c, strict=True)
+ assert left.origin == right.origin
+ for name in ("typecode", "altitude_basis", "aircraft_class", "velocity_inferred_components"):
+ assert getattr(left, name) == getattr(right, name)
+
+
+def test_named_envelope_matches_broad_query_across_sql_batches(recorded_index):
+ index, bounds, times, observed = recorded_index
+ with index.connect() as database:
+ names = [row[0] for row in database.execute("SELECT name FROM segments ORDER BY name LIMIT 650")]
+ # The real index contains enough names to cross the 500-parameter batch.
+ assert len(names) > 500
+ names = sorted(set(names) | {track.id for track in observed[:6]})
+ requested = names[::-1] + names[:3] + ["missing-recorded-segment"]
+ selected = set(requested)
+ expected = [track for track in observed if track.id in selected]
+ actual = list(index.lookup(requested, bounds=bounds, time_range=times))
+ same_tracks(actual, expected)
+
+
+def test_named_envelope_preserves_spatial_time_filters_and_old_index_fallback(recorded_index):
+ original, bounds, times, observed = recorded_index
+ index = FlightIndex(original.path)
+ names = [track.id for track in observed[:6]]
+ # A disjoint geographic envelope and out-of-day time query expose accidental
+ # removal of either predicate without manufacturing trajectory observations.
+ disjoint = (bounds[0] + 20, bounds[1], bounds[2] + 20, bounds[3])
+ outside_time = (index.report["coverage"]["time_range_s"][1] + 86400,) * 2
+ for box, window in ((disjoint, times), (bounds, outside_time),
+ (bounds, (observed[0].time[0], observed[0].time[0]))):
+ expected = [track for track in index.query(box, window) if track.id in names]
+ if window == outside_time:
+ assert not expected
+ if window == (observed[0].time[0], observed[0].time[0]):
+ assert observed[0].id in {track.id for track in expected}
+ same_tracks(list(index.lookup(names, bounds=box, time_range=window)), expected)
+ # Older index metadata must disable spatial pruning in both entry points.
+ index.report["continuous_geometry_bounds"] = False
+ expected = [track for track in index.query(disjoint, times) if track.id in names]
+ assert [track.id for track in expected] == names
+ same_tracks(list(index.lookup(names, bounds=disjoint, time_range=times)), expected)
+ index.report["continuous_geometry_bounds"] = True
+ index.report["normalizer_sha256"] = "different-normalizer-version"
+ same_tracks(list(index.lookup(names, bounds=disjoint, time_range=times)), expected)
+
+
+def test_refinement_adapter_duplicates_and_sample_budget(recorded_index):
+ index, bounds, times, observed = recorded_index
+ needed = {track.id for track in observed[:6]}
+ expected = [track for track in observed if track.id in needed]
+ sample_limit = sum(len(track.time) for track in expected)
+
+ class QueryAdapter:
+ def query(self, box, window):
+ assert tuple(box) == tuple(bounds) and tuple(window) == tuple(times)
+ yield from observed
+
+ for indexes in ([index], [index, index], [QueryAdapter()], [index, QueryAdapter()]):
+ actual = _refinement_tracks(indexes, needed, bounds, times, sample_limit)
+ same_tracks(list(actual.values()), expected)
+ with pytest.raises(ValueError, match="^Refinement trajectory sample budget exceeded$"):
+ _refinement_tracks(indexes, needed, bounds, times, sample_limit - 1)
+ assert _refinement_tracks([index], {"missing-recorded-segment"}, bounds, times, 1) == {}
+
+
+def test_named_envelope_requires_complete_scope_and_checks_payload(recorded_index, tmp_path):
+ index, bounds, times, observed = recorded_index
+ name = observed[0].id
+ for settings in ({"bounds": bounds}, {"time_range": times}):
+ with pytest.raises(ValueError, match="must be supplied together"):
+ list(index.lookup([name], **settings))
+ path = tmp_path / "corrupt-index.sqlite"
+ shutil.copyfile(index.path, path)
+ with sqlite3.connect(path) as database:
+ database.execute("UPDATE segments SET payload_hash=? WHERE name=?", ("corrupt", name))
+ with pytest.raises(DataUnavailable, match="Corrupt flight payload"):
+ list(FlightIndex(path).lookup([name], bounds=bounds, time_range=times))