Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Reuse exact fits and finalist trajectories in the automatic pipeline
Cinna Davis90d95d7ea793
diff --git a/scripts/benchmark_real_discovery_plots.py b/scripts/benchmark_real_discovery_plots.pynew file mode 100644index 0000000..0495c60--- /dev/null+++ b/scripts/benchmark_real_discovery_plots.py@@ -0,0 +1,101 @@+"""Compare discovery diagnostic rendering on an existing real-recording run.++The input directory must contain intake/ and discovery/ from discover_audio.py+or an automatic job. Both implementations render every measured ridge with the+same full spectral normalization. Every PNG pixel must match. This measures+diagnostic rendering only, not discovery or end-to-end localization speed.+"""+import argparse+import importlib.util+import json+from pathlib import Path+import statistics+import subprocess+import sys+import time++import matplotlib+matplotlib.use("Agg")+from matplotlib.image import imread+import numpy as np++from dopplergeist import discovery+from dopplergeist.intake import IntakeRecording+++def _module(checkout):+ name = "dopplergeist._baseline_discovery_plots"+ spec = importlib.util.spec_from_file_location(name, checkout / "src/dopplergeist/discovery.py")+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ return module+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("recording_directory", type=Path)+ parser.add_argument("--baseline", type=Path, required=True, help="Unmodified baseline checkout")+ parser.add_argument("--output", type=Path, required=True, help="New output directory")+ parser.add_argument("--repeats", type=int, default=3)+ args = parser.parse_args()+ if args.repeats < 1:+ parser.error("--repeats must be positive")+ intake = args.recording_directory / "intake"+ metadata = json.loads((intake / "recording.json").read_text())+ diagnostics = json.loads((args.recording_directory / "discovery/discovery.json").read_text())+ if metadata["source_sha256"] != diagnostics["source_sha256"]:+ parser.error("Intake and discovery belong to different source recordings")+ recording = IntakeRecording(intake / "recording.f32", metadata, {})+ settings = discovery.DiscoverySettings(**diagnostics["settings"])+ events = diagnostics["events"]+ if not events:+ parser.error("The real recording contains no event diagnostics to compare")+ group_ids = list(dict.fromkeys(event["harmonic_group_id"] for event in events))+ if len(group_ids) != len(diagnostics["harmonic_ridges"]):+ parser.error("Discovery groups and measured harmonic ridges do not agree")+ groups = {name: [np.asarray(points) for points in rows]+ for name, rows in zip(group_ids, diagnostics["harmonic_ridges"])}+ implementations = {"baseline": _module(args.baseline.resolve()), "current": discovery}+ args.output.mkdir(parents=True, exist_ok=False)+ samples = {name: [] for name in implementations}+ reference_directory = None+ # Load the backend, colormap, and fonts before either timed arm.+ implementations["baseline"]._plot_event(recording, groups[events[0]["harmonic_group_id"]],+ events[0], settings, args.output / "warmup.png")+ for trial in range(args.repeats):+ order = ("baseline", "current") if trial % 2 == 0 else ("current", "baseline")+ for name in order:+ directory = args.output / f"{name}-{trial + 1}"+ directory.mkdir()+ started_wall, started_cpu = time.perf_counter(), time.process_time()+ for event in events:+ implementations[name]._plot_event(recording, groups[event["harmonic_group_id"]],+ event, settings, directory / f"{event['event_id']}.png")+ timing = {"wall_s": time.perf_counter() - started_wall,+ "cpu_s": time.process_time() - started_cpu}+ samples[name].append(timing)+ if reference_directory is None:+ reference_directory = directory+ for event in events:+ filename = f"{event['event_id']}.png"+ if not np.array_equal(imread(reference_directory / filename), imread(directory / filename)):+ raise AssertionError(f"Diagnostic pixels changed: {name}, trial {trial + 1}, {filename}")+ print(json.dumps({"implementation": name, "trial": trial + 1, **timing}), flush=True)+ summary = {name: {"trials": rows,+ "median_wall_s": statistics.median(row["wall_s"] for row in rows),+ "median_cpu_s": statistics.median(row["cpu_s"] for row in rows)}+ for name, rows in samples.items()}+ summary.update({"source": metadata, "ridge_count": len(events), "group_count": len(groups),+ "exact_png_pixels": True,+ "wall_speedup": summary["baseline"]["median_wall_s"] / summary["current"]["median_wall_s"],+ "cpu_speedup": summary["baseline"]["median_cpu_s"] / summary["current"]["median_cpu_s"],+ "baseline_commit": subprocess.check_output(+ ["git", "-C", str(args.baseline), "rev-parse", "HEAD"], text=True).strip(),+ "scope": "All event PNGs from real measured ridges; excludes intake, discovery, and equality checks"})+ (args.output / "summary.json").write_text(json.dumps(summary, indent=2, allow_nan=False) + "\n")+ print(json.dumps(summary, indent=2, allow_nan=False))+++if __name__ == "__main__":+ main()diff --git a/scripts/check_real_batch_cache.py b/scripts/check_real_batch_cache.pynew file mode 100644index 0000000..39ce95d--- /dev/null+++ b/scripts/check_real_batch_cache.py@@ -0,0 +1,204 @@+"""Check exact batch-cache behavior using recorded ridges and flight tracks.++This is a software regression check, not a location-accuracy experiment. Input+mutation checks change copies by one ULP solely to test cache invalidation.+No recording or trajectory is generated.+"""+import argparse+from copy import deepcopy+import json+from pathlib import Path+import sys+from types import SimpleNamespace++import numpy as np++from dopplergeist.batch_ridge import first_event_fits+from dopplergeist.geometry import AcousticSupportError, ecef+from dopplergeist.matching import (+ _cache_batch_ridge_fits, _cache_ridge_fits, _frequency_scale,+ _iter_acoustic_trials, _ridge_fit_cache, prepare_events,+)+from dopplergeist.tracks import load_tracks+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--discovery", required=True, type=Path)+ parser.add_argument("--tracks", required=True, type=Path)+ parser.add_argument("--candidate", required=True, type=Path)+ args = parser.parse_args()+ events, measurements = prepare_events(json.loads(args.discovery.read_text())["events"])+ saved = json.loads(args.candidate.read_text())+ point = saved.get("best", saved)+ start = point["recording_start_time_s"]+ tracks, _ = load_tracks(args.tracks, time_range=(start - 120, start + max(e["end_s"] for e in events) + 120))+ aircraft = {identifier.split(":")[0] for identifier in point["flight_ids"]}+ chosen = [track for track in tracks if track.id.split(":")[0] in aircraft]+ observer = ecef(point["lat"], point["lon"], point["elev_m"])+ starts = start + np.linspace(-5, 5, 21)+ uncached = first_event_fits.__wrapped__+ inputs = None+ for event, measurement in zip(events, measurements):+ for track in chosen:+ try:+ batch = next(_iter_acoustic_trials([track], [measurement], observer, starts))+ except AcousticSupportError:+ continue+ if batch is not None and uncached(measurement, event, *batch) is not None:+ inputs = measurement, event, *batch+ break+ if inputs is not None:+ break+ if inputs is None:+ raise RuntimeError("No supported batch exists for the supplied real receiver/flight hypotheses")++ calls = 0+ checks = []++ def counted(*values):+ nonlocal calls+ calls += 1+ return uncached(*values)++ cached = _cache_batch_ridge_fits(counted)++ def check(name, condition):+ if not condition:+ raise AssertionError(name)+ checks.append(name)++ expected = uncached(*inputs)+ check("no_context_result", cached(*inputs) == expected)+ check("no_context_bypass", calls == 1 and _ridge_fit_cache.get() is None)++ @_cache_ridge_fits+ def check_context():+ outer = _ridge_fit_cache.get()+ initial = calls+ first = cached(*inputs)+ second = cached(*deepcopy(inputs))+ check("equal_copies_hit", calls == initial + 1 and first == second == expected)+ first[0]["score"] = 12345.0+ second[0]["f0_hz"] = 0.0+ check("diagnostic_copies_are_independent", cached(*inputs) == expected)+ check("memory_bound", 0 < outer.batch_bytes + sys.getsizeof(outer.batch_entries) <= outer.batch_limit)++ def alternate(*values):+ return counted(*values)++ before = calls+ check("different_function_result", _cache_batch_ridge_fits(alternate)(*inputs) == expected)+ check("different_function_misses", calls == before + 1)++ for index, name in enumerate(("times", "ridge", "weights", "factors", "emissions")):+ changed = deepcopy(inputs)+ array = ((changed[0][0].time, changed[0][1], changed[0][2], changed[2], changed[3])[index])+ array.flat[0] = np.nextafter(array.flat[0], np.inf)+ before = calls+ check(name + "_mutation_result", cached(*changed) == uncached(*changed))+ check(name + "_mutation_misses", calls == before + 1)++ for name in ("bounds", "sigma", "fraction"):+ changed = deepcopy(inputs)+ event = changed[1]+ if name == "bounds":+ event["f0_range_hz"] = list(event["f0_range_hz"])+ event["f0_range_hz"][0] = float(np.nextafter(event["f0_range_hz"][0], np.inf))+ elif name == "sigma":+ event.setdefault("quality", {})["frequency_loss_scale_hz"] = float(np.nextafter(_frequency_scale(event), np.inf))+ else:+ event.setdefault("quality", {})["source_drift_fraction"] = float(np.nextafter(event["quality"].get("source_drift_fraction", 0.02), np.inf))+ before = calls+ check(name + "_mutation_result", cached(*changed) == uncached(*changed))+ check(name + "_mutation_misses", calls == before + 1)++ changed = list(deepcopy(inputs))+ changed[2] = changed[2].astype(changed[2].dtype.newbyteorder(">"))+ before = calls+ check("dtype_result", cached(*changed) == uncached(*changed))+ check("dtype_misses", calls == before + 1)+ changed = [*inputs[:2], inputs[2][::-1], inputs[3][::-1]]+ before = calls+ check("row_order_result", cached(*changed) == uncached(*changed))+ check("row_order_misses", calls == before + 1)+ changed = [*inputs[:2], np.asfortranarray(inputs[2]), np.asfortranarray(inputs[3])]+ before = calls+ check("fortran_layout_result", cached(*changed) == uncached(*changed))+ check("fortran_layout_misses", calls == before + 1)+ changed = [*inputs[:2], inputs[2].tolist(), inputs[3].tolist()]+ before = calls+ check("custom_representation_result", cached(*changed) == cached(*changed) == expected)+ check("custom_representation_bypasses", calls == before + 2)+ before = calls+ check("measurement_iterator_result", cached(iter(inputs[0]), *inputs[1:]) == expected)+ check("measurement_iterator_bypasses", calls == before + 1)+ changed = list(deepcopy(inputs))+ measurement = list(changed[0])+ measurement[0] = SimpleNamespace(time=measurement[0].time)+ changed[0] = measurement+ before = calls+ check("custom_time_container_result", cached(*changed) == cached(*changed) == expected)+ check("custom_time_container_bypasses", calls == before + 2)++ missing = [*inputs[:2], inputs[2][:0], inputs[3][:0]]+ before = calls+ entries = len(outer.batch_entries)+ check("fallback_none", cached(*missing) is None and cached(*missing) is None)+ check("fallback_not_cached", calls == before + 2 and len(outer.batch_entries) == entries)+ malformed = deepcopy(inputs)+ malformed[0][0].time = malformed[0][0].time[:2]+ before = calls+ for _ in range(2):+ try:+ cached(*malformed)+ except ValueError:+ pass+ else:+ raise AssertionError("Malformed measurement did not raise")+ check("exceptions_not_cached", calls == before + 2 and len(outer.batch_entries) == entries)++ @_cache_ridge_fits+ def nested():+ check("nested_context_is_independent", _ridge_fit_cache.get() is not outer)+ before = calls+ check("nested_fit_result", cached(*inputs) == expected)+ check("nested_fit_misses", calls == before + 1)+ raise RuntimeError("Check context restoration after an exception")++ try:+ nested()+ except RuntimeError:+ pass+ check("nested_context_restored", _ridge_fit_cache.get() is outer)+ before = calls+ check("outer_fit_survives", cached(*inputs) == expected and calls == before)++ check_context()+ check("context_reset", _ridge_fit_cache.get() is None)++ def eviction():+ cache = _ridge_fit_cache.get()+ first = deepcopy(inputs)+ second = deepcopy(inputs)+ second[2][0, 0] = np.nextafter(second[2][0, 0], np.inf)+ check("eviction_first_result", cached(*first) == uncached(*first))+ check("eviction_second_result", cached(*second) == uncached(*second))+ check("entry_limit", len(cache.batch_entries) == 1)+ before = calls+ check("evicted_fit_recomputed", cached(*first) == uncached(*first) and calls == before + 1)+ cache.batch_entries.clear()+ cache.batch_bytes = 0+ cache.batch_limit = 1+ before = calls+ check("oversized_fits_bypass", cached(*first) == cached(*first) == expected and calls == before + 2)+ check("oversized_fits_not_stored", len(cache.batch_entries) == cache.batch_bytes == 0)++ _cache_ridge_fits(eviction, max_entries=1)()+ print(json.dumps({"checks": checks, "passed": len(checks), "recorded_groups": len(events),+ "selected_event": inputs[1].get("event_id"), "clock_rows": len(inputs[2]),+ "scope": "real-input cache regression; no location-accuracy claim"}, indent=2))+++if __name__ == "__main__":+ main()diff --git a/scripts/check_real_ceiling_cache.py b/scripts/check_real_ceiling_cache.pynew file mode 100644index 0000000..ce224ac--- /dev/null+++ b/scripts/check_real_ceiling_cache.py@@ -0,0 +1,161 @@+"""Check ceiling reuse against the original validator using recorded ridge fits.++Metadata mutations exercise cache invalidation and conservative unknown-support+handling. They are software checks, not generated recordings or field evidence.+"""+import argparse+from copy import deepcopy+import json+from pathlib import Path++import numpy as np++from dopplergeist import search+from dopplergeist.batch_ridge import first_event_fits+from dopplergeist.geometry import AcousticSupportError, ecef+from dopplergeist.matching import (+ _cache_ridge_fits, _frequency_scale, _iter_acoustic_trials, _ridge_fit_cache, prepare_events,+)+from dopplergeist.tracks import load_tracks+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--discovery", required=True, type=Path)+ parser.add_argument("--tracks", required=True, type=Path)+ parser.add_argument("--candidate", required=True, type=Path)+ args = parser.parse_args()+ events, measurements = prepare_events(json.loads(args.discovery.read_text())["events"])+ saved = json.loads(args.candidate.read_text())+ point = saved.get("best", saved)+ start = point["recording_start_time_s"]+ tracks, _ = load_tracks(args.tracks, time_range=(start - 120, start + max(e["end_s"] for e in events) + 120))+ aircraft = {identifier.split(":")[0] for identifier in point["flight_ids"]}+ observer = ecef(point["lat"], point["lon"], point["elev_m"])+ starts = start + np.linspace(-5, 5, 21)+ fits = None+ for event, measurement in zip(events, measurements):+ for track in (track for track in tracks if track.id.split(":")[0] in aircraft):+ try:+ batch = next(_iter_acoustic_trials([track], [measurement], observer, starts))+ except AcousticSupportError:+ continue+ if batch is not None:+ fits = first_event_fits(measurement, event, *batch)+ if fits is not None:+ break+ if fits is not None:+ break+ if fits is None:+ raise RuntimeError("No supported batch exists for the supplied real receiver/flight hypotheses")+ limit = 3 * _frequency_scale(event)+ base = (fits[0], fits[0]["score"], fits[0]["f0_hz"], event, limit)+ checks = []+ calls = 0+ original_classify = search._ridge_trial_classification++ def counted(*values):+ nonlocal calls+ calls += 1+ return original_classify(*values)++ def check(name, condition):+ if not condition:+ raise AssertionError(name)+ checks.append(name)++ def same(name, values, miss=None):+ expected = search._association_event_ceiling_uncached(*values)+ before = calls+ actual = search._association_event_ceiling(*values)+ check(name, actual == expected or (np.isnan(actual) and np.isnan(expected)))+ if miss is not None:+ check(name + "_cache_behavior", calls == before + int(miss))++ @_cache_ridge_fits+ def exercise():+ outer = _ridge_fit_cache.get()+ for index, fit in enumerate(fits):+ values = (fit, fit["score"], fit["f0_hz"], event, limit)+ same(f"real_clock_{index}", values, True)+ same(f"copied_clock_{index}", deepcopy(values), False)+ for field in search._CEILING_FLOAT_FIELDS:+ changed = deepcopy(base)+ changed[0][field] = float(np.nextafter(changed[0][field], np.inf))+ same(field + "_mutation", changed, True)+ for field in ("sample_count", "independent_sample_count"):+ changed = deepcopy(base)+ changed[0][field] += 1+ same(field + "_mutation", changed, True)+ for field, value in (("optimizer_success", False), ("support_fraction", 0.0),+ ("source_span_s", 0.0), ("drift_bound_hz", -1.0),+ ("rmse_hz", -1.0), ("outlier_fraction", 2.0)):+ changed = deepcopy(base)+ changed[0][field] = value+ same(field + "_invalid", changed, True)+ check(field + "_conservative", search._association_event_ceiling(*changed) == 1.0)+ for index, name in ((1, "score_argument"), (2, "frequency_argument"), (4, "limit")):+ changed = list(deepcopy(base))+ changed[index] = float(np.nextafter(changed[index], np.inf))+ same(name + "_mutation", changed, True)+ for index in range(2):+ changed = deepcopy(base)+ changed[3]["f0_range_hz"][index] = float(np.nextafter(changed[3]["f0_range_hz"][index], np.inf))+ same(f"frequency_bound_{index}_mutation", changed, True)+ changed = deepcopy(base)+ changed[3].setdefault("quality", {})["source_drift_fraction"] = float(np.nextafter(+ changed[3]["quality"].get("source_drift_fraction", 0.02), np.inf))+ same("drift_fraction_mutation", changed, True)+ for value in (np.float32(base[1]), np.float64(base[1]), np.longdouble(base[1])):+ changed = list(deepcopy(base))+ changed[0]["score"] = changed[1] = value+ same(type(value).__name__ + "_bypass", changed, True)+ same(type(value).__name__ + "_repeated_bypass", changed, True)+ for field in ("support_fraction", "sample_count", "source_span_s"):+ changed = deepcopy(base)+ del changed[0][field]+ same(field + "_missing", changed, True)+ same(field + "_missing_bypass", changed, True)+ changed = list(deepcopy(base))+ changed[4] = None+ entries = len(outer.ceiling_entries)+ for _ in range(2):+ try:+ search._association_event_ceiling(*changed)+ except TypeError:+ checks.append("invalid_limit_exception")+ else:+ raise AssertionError("Invalid limit did not raise the original error")+ check("exceptions_not_cached", len(outer.ceiling_entries) == entries)++ @_cache_ridge_fits+ def nested():+ check("nested_context_is_independent", _ridge_fit_cache.get() is not outer)+ same("nested_first_lookup", base, True)+ nested()+ check("outer_context_restored", _ridge_fit_cache.get() is outer)+ same("outer_cached_fit_survives", base, False)++ def eviction():+ outer = _ridge_fit_cache.get()+ same("entry_limit_first", base, True)+ changed = list(deepcopy(base))+ changed[4] = float(np.nextafter(changed[4], np.inf))+ same("entry_limit_second", changed, True)+ check("entry_limit", len(outer.ceiling_entries) == 1)+ same("evicted_entry_recomputed", base, True)++ search._ridge_trial_classification = counted+ try:+ same("outside_context", base, True)+ exercise()+ check("context_reset", _ridge_fit_cache.get() is None)+ _cache_ridge_fits(eviction, max_entries=1)()+ finally:+ search._ridge_trial_classification = original_classify+ print(json.dumps({"passed": len(checks), "checks": checks, "event": event["event_id"],+ "real_clock_fits": len(fits), "scope": "real-input ceiling cache regression"}, indent=2))+++if __name__ == "__main__":+ main()diff --git a/src/dopplergeist/automatic.py b/src/dopplergeist/automatic.pyindex 0ed79b9..2cb7a6d 100644--- a/src/dopplergeist/automatic.py+++ b/src/dopplergeist/automatic.py@@ -443,11 +443,40 @@ def _compare_retained_spectra(recording, rows, prepared_events, indexes, config,return report+def _refinement_tracks(indexes, needed, bounds, time_range, sample_limit):+ """Load only a finalist's whole segments within the original query envelope."""+ from .data_access import DataUnavailable, FlightIndex++ tracks = {}+ for index in indexes:+ # Preserve custom query adapters and their filtering behavior. The native+ # index can restrict names in SQL before decompressing unrelated flights.+ named = type(index) is FlightIndex+ source = (index.lookup(needed, bounds=bounds, time_range=time_range,+ max_tracks=max(1, len(needed)), max_samples=sample_limit)+ if named else index.query(bounds, time_range))+ try:+ for track in source:+ if track.id in needed:+ tracks[track.id] = track+ if sum(len(item.time) for item in tracks.values()) > sample_limit:+ raise ValueError("Refinement trajectory sample budget exceeded")+ except DataUnavailable as error:+ if named and str(error) == "Flight lookup sample budget exceeded":+ raise ValueError("Refinement trajectory sample budget exceeded") from error+ raise+ return tracks++def _decide(search, events, prepared_events, measurements, indexes, terrain, state, output, config, finish, decision_progress, *, recording=None):# The confidence producer supplies this adapter; missing decision machinery# can never turn the highest raw score into a supported location.from .confidence import decide_location- from .matching import profile_candidate, refine_candidate+ from .matching import _cache_ridge_fits, profile_candidate, refine_candidate+ # Each finalist and the complete sensitivity profile own an exact-input+ # cache. Profile's internal refinements share its scope; other calls do not.+ refine_candidate = _cache_ridge_fits(refine_candidate)+ profile_candidate = _cache_ridge_fits(profile_candidate)raw = list(search.get("ranked", []))refined, refinement_failures = [], []window = state["recording"]["time_window"]@@ -494,16 +523,10 @@ def _decide(search, events, prepared_events, measurements, indexes, terrain, stamargin = max_range / 60000.0 + 0.3bounds = (row["lon"]-margin, row["lat"]-margin, row["lon"]+margin, row["lat"]+margin)needed = set(row["flight_ids"])- tracks = {}lo = start_window[0] + min(e["start_s"] for e in prepared_events) - max_range / 300 - 120hi = start_window[1] + max(e["end_s"] for e in prepared_events) + 120- for index in indexes:- for track in index.query(bounds, (lo, hi)):- if track.id in needed:- tracks[track.id] = track- sample_limit = config.get("search_budget", {}).get("max_track_samples_per_tile", 500000)- if sum(len(item.time) for item in tracks.values()) > sample_limit:- raise ValueError("Refinement trajectory sample budget exceeded")+ sample_limit = config.get("search_budget", {}).get("max_track_samples_per_tile", 500000)+ tracks = _refinement_tracks(indexes, needed, bounds, (lo, hi), sample_limit)if needed - tracks.keys():raise ValueError("Candidate flight segments are unavailable for refinement")row = dict(row, altitude_basis={flight: sorted(set(getattr(track, "altitude_basis", ["unspecified"]))) for flight, track in tracks.items()})diff --git a/src/dopplergeist/batch_ridge.py b/src/dopplergeist/batch_ridge.pyindex 78995bd..010f715 100644--- a/src/dopplergeist/batch_ridge.py+++ b/src/dopplergeist/batch_ridge.py@@ -1,9 +1,10 @@"""Batched robust ridge fits with the scalar model's arithmetic order."""import numpy as np-from .matching import _frequency_scale, _ridge_fit_inputs+from .matching import _cache_batch_ridge_fits, _frequency_scale, _ridge_fit_inputs+@_cache_batch_ridge_fitsdef first_event_fits(measurement, event, factors, emissions):"""Fit supported clocks together, or return None for the ordinary fitter.diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.pyindex ca8c5e9..4c2353b 100644--- a/src/dopplergeist/data_access.py+++ b/src/dopplergeist/data_access.py@@ -496,42 +496,51 @@ class FlightIndex:except (OSError, ValueError, KeyError, TypeError) as error:raise DataUnavailable(f"Invalid/truncated Parquet: {parquet}: {error}") from error+ def _query_scope(self, bounds, time_range):+ """Use the same conservative envelope for broad and named queries."""+ west, south, east, north = bounds+ start, end = time_range+ current_normalizer = sha256(Path(__file__).with_name("tracks.py"))+ compatible = (current_normalizer, *_EQUIVALENT_NORMALIZERS.get(current_normalizer, ()))+ if (not self.report.get("continuous_geometry_bounds") or+ self.report.get("normalizer_sha256") not in compatible):+ # Old indexes used sample-only bounds or different derivatives.+ # Disable unsafe spatial pruning, preserving explicit budgets.+ return "", "s.start<=? AND s.end>=?", (end, start)+ return (" JOIN spatial r ON s.id=r.id",+ "r.west<=? AND r.east>=? AND r.south<=? AND r.north>=? AND s.start<=? AND s.end>=?",+ (east, west, north, south, end, start))+def query(self, bounds, time_range):"""Yield complete segments touching the query, retaining gaps and labels.Callers expand receiver bounds for acoustic range and event duration.An R-tree rectangle can return harmless extra tracks, never clipped ones."""- west, south, east, north = bounds- start, end = time_range+ join, predicate, params = self._query_scope(bounds, time_range)with self.connect() as db:- sql = """SELECT s.name,s.payload,s.payload_hash FROM segments s JOIN spatial r ON s.id=r.id- WHERE r.west<=? AND r.east>=? AND r.south<=? AND r.north>=?- AND s.start<=? AND s.end>=? ORDER BY s.name"""- params = (east, west, north, south, end, start)- current_normalizer = sha256(Path(__file__).with_name("tracks.py"))- compatible = (current_normalizer, *_EQUIVALENT_NORMALIZERS.get(current_normalizer, ()))- if (not self.report.get("continuous_geometry_bounds") or- self.report.get("normalizer_sha256") not in compatible):- # Old indexes used sample-only bounds or different derivatives.- # Disable unsafe spatial pruning, preserving explicit budgets.- sql = "SELECT name,payload,payload_hash FROM segments WHERE start<=? AND end>=? ORDER BY name"- params = (end, start)+ sql = ("SELECT s.name,s.payload,s.payload_hash FROM segments s" + join+ + " WHERE " + predicate + " ORDER BY s.name")for name, payload, expected in db.execute(sql, params):if hashlib.sha256(payload).hexdigest() != expected:raise DataUnavailable(f"Corrupt flight payload: {name}")frame = pd.DataFrame(json.loads(gzip.decompress(payload)))yield _frame_track(name, frame)- def lookup(self, identifiers, *, max_tracks=2000, max_samples=500000):+ def lookup(self, identifiers, *, max_tracks=2000, max_samples=500000,+ bounds=None, time_range=None):"""Yield named whole segments without scanning unrelated trajectories.Missing names are omitted; callers must compare requested and returnedIDs. Limits reject excess work explicitly, never truncate a trajectory.+ Supply both bounds and time_range to apply the same envelope as query,+ including its conservative fallback for older geometry indexes.This lookup establishes no spatial, temporal or receiver completeness."""if any(type(value) is not int or value <= 0 for value in (max_tracks, max_samples)):raise ValueError("Flight lookup limits must be positive integers")+ if (bounds is None) != (time_range is None):+ raise ValueError("Flight lookup bounds and time_range must be supplied together")names = set()for name in identifiers:if not isinstance(name, str) or not name:@@ -540,14 +549,18 @@ class FlightIndex:if len(names) > max_tracks:raise DataUnavailable("Flight lookup identifier budget exceeded")names, samples = sorted(names), 0+ join, predicate, scope_params = ("", "", ())+ if bounds is not None:+ join, predicate, scope_params = self._query_scope(bounds, time_range)with self.connect() as db:# Stay below SQLite's conservative bound on bound SQL parameters.for first in range(0, len(names), 500):batch = names[first:first+500]placeholders = ",".join("?" for _ in batch)- sql = ("SELECT name,payload,payload_hash FROM segments WHERE name IN ("- + placeholders + ") ORDER BY name")- for name, payload, expected in db.execute(sql, batch):+ sql = ("SELECT s.name,s.payload,s.payload_hash FROM segments s" + join+ + " WHERE s.name IN (" + placeholders + ")"+ + (" AND " + predicate if predicate else "") + " ORDER BY s.name")+ for name, payload, expected in db.execute(sql, (*batch, *scope_params)):if hashlib.sha256(payload).hexdigest() != expected:raise DataUnavailable(f"Corrupt flight payload: {name}")frame = pd.DataFrame(json.loads(gzip.decompress(payload)))diff --git a/src/dopplergeist/discovery.py b/src/dopplergeist/discovery.pyindex 0905839..04dde53 100644--- a/src/dopplergeist/discovery.py+++ b/src/dopplergeist/discovery.py@@ -362,15 +362,22 @@ def _plot_event(recording, group, event, settings, path):sr = recording.metadata["sample_rate_hz"]data = np.memmap(recording.samples_path, dtype="<f4", mode="r")spec = _spectrum(np.array(data[round(first * sr):round(last * sr)]), sr, first, settings)+ low = min(points[:, 1].min() for points in group)+ high = max(points[:, 1].max() for points in group)+ limits = (max(settings.min_frequency_hz, low * 0.9), high * 1.1)+ # Only visible cells can affect the image. Keep an extra frequency bin on+ # each side so pcolormesh reconstructs the same boundary-cell coordinates.+ # Whitening still uses the full spectrum and all measured ridges are kept.+ lower = max(0, np.searchsorted(spec.frequency, limits[0]) - 1)+ upper = min(len(spec.frequency), np.searchsorted(spec.frequency, limits[1], side="right") + 1)+ visible = slice(lower, upper)fig, ax = plt.subplots(figsize=(10, 4), layout="constrained")- ax.pcolormesh(spec.time, spec.frequency, spec.contrast, shading="auto", cmap="magma", vmin=0, vmax=6,+ ax.pcolormesh(spec.time, spec.frequency[visible], spec.contrast[visible], shading="auto", cmap="magma", vmin=0, vmax=6,rasterized=True)for points in group:for segment in np.split(points, np.flatnonzero(np.diff(points[:, 0]) > settings.max_gap_s) + 1):ax.plot(segment[:, 0], segment[:, 1], color="cyan", lw=0.9)- low = min(points[:, 1].min() for points in group)- high = max(points[:, 1].max() for points in group)- ax.set(xlim=(first, last), ylim=(max(settings.min_frequency_hz, low * 0.9), high * 1.1),+ ax.set(xlim=(first, last), ylim=limits,xlabel="Recording-relative time (s)", ylabel="Frequency (Hz)",title=f"{event['event_id']}: measured ridges; acoustic proposal, identity unconfirmed")fig.savefig(path, dpi=130)diff --git a/src/dopplergeist/matching.py b/src/dopplergeist/matching.pyindex 88a7ec9..d6f38ab 100644--- a/src/dopplergeist/matching.py+++ b/src/dopplergeist/matching.py@@ -10,11 +10,13 @@ from dataclasses import dataclassfrom functools import wrapsimport hashlibfrom itertools import chain+import sysimport numpy as npfrom pyproj import Transformerfrom scipy.optimize import minimize+from .audio import Spectrumfrom .geometry import Track, _EMISSION_ITERATION_TOLERANCE_S, ecef@@ -53,6 +55,10 @@ class _RidgeFitCache:self.measurement_entries = OrderedDict()self.measurement_bytes = 0self.measurement_limit = 4 * 1024 * 1024+ self.batch_entries = OrderedDict()+ self.batch_bytes = 0+ self.batch_limit = 16 * 1024 * 1024+ self.ceiling_entries = OrderedDict()def prepared_measurement(self, measurement, sigma_hz, drift_fraction, key):"""Reuse selected samples only after checking their current content key."""@@ -163,6 +169,76 @@ def _cache_ridge_fits(function, *, max_entries=4096):return run+def _cache_batch_ridge_fits(function):+ """Reuse identical source fits across associations within one scoring call.++ Every lookup checks current input bytes. The cache cannot turn changed+ measurements, propagation arrays or model parameters into an old result.+ Successful diagnostics contain immutable scalars; independent dictionary+ copies prevent callers from modifying the saved fit. Custom representations+ retain the ordinary fitter and its validation behavior.+ """+ @wraps(function)+ def run(measurement, event, factors, emissions):+ cache = _ridge_fit_cache.get()+ if cache is None or cache.batch_limit <= 0 or cache.max_entries <= 0:+ return function(measurement, event, factors, emissions)+ if (type(measurement) not in (list, tuple) or len(measurement) != 4+ or type(event) is not dict or type(event.get("quality", {})) is not dict):+ return function(measurement, event, factors, emissions)+ spec, ridge, weight, _ = measurement+ if type(spec) not in (RidgeTimes, Spectrum):+ return function(measurement, event, factors, emissions)+ arrays = (spec.time, ridge, weight, factors, emissions)+ if any(type(value) is not np.ndarray or value.dtype.kind not in "biuf" for value in arrays):+ return function(measurement, event, factors, emissions)+ bounds = event.get("f0_range_hz")+ sigma = _frequency_scale(event)+ fraction = event.get("quality", {}).get("source_drift_fraction", 0.02)+ if (type(bounds) not in (list, tuple) or len(bounds) != 2+ or any(type(value) not in (int, float) for value in (sigma, fraction, *bounds))):+ return function(measurement, event, factors, emissions)+ parameters = tuple(_fit_argument_key(value) for value in (sigma, fraction, bounds))+ # Equal values can have different NumPy reduction orders when their+ # strides differ. Preserve layout as well as every numeric input byte.+ content = tuple((value.dtype.str, value.shape, value.strides, value.flags.aligned,+ hashlib.sha256(value.tobytes()).digest())+ for value in arrays)+ key = function, parameters, content+ if key in cache.batch_entries:+ cache.batch_entries.move_to_end(key)+ return [diagnostic.copy() for diagnostic in cache.batch_entries[key][0]]+ result = function(measurement, event, factors, emissions)+ if result is not None:+ # Builtin type tags and literal diagnostic keys already live in the+ # module. Charge every other key/scalar object to each entry, even+ # when shared. Check the actual OrderedDict size after insertion+ # too, since its backing table can grow or retain capacity.+ def key_size(value):+ if isinstance(value, type):+ return 0+ return sys.getsizeof(value) + (sum(map(key_size, value)) if isinstance(value, tuple) else 0)++ size = (key_size(key) + sys.getsizeof((None, None)) + sys.getsizeof(0) + sys.getsizeof(result)+ + sum(sys.getsizeof(diagnostic)+ + sum(sys.getsizeof(value) for value in diagnostic.values())+ for diagnostic in result))+ if size <= cache.batch_limit:+ while cache.batch_entries and (+ cache.batch_bytes + size + sys.getsizeof(cache.batch_entries) > cache.batch_limit+ or len(cache.batch_entries) >= cache.max_entries):+ _, (_, removed) = cache.batch_entries.popitem(last=False)+ cache.batch_bytes -= removed+ cache.batch_entries[key] = ([diagnostic.copy() for diagnostic in result], size)+ cache.batch_bytes += size+ while cache.batch_entries and cache.batch_bytes + sys.getsizeof(cache.batch_entries) > cache.batch_limit:+ _, (_, removed) = cache.batch_entries.popitem(last=False)+ cache.batch_bytes -= removed+ return result++ return run++def _frequency_scale(event):quality = event.get("quality", {})return quality.get("frequency_loss_scale_hz", quality.get("frequency_sigma_hz",diff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.pyindex fb717cb..6b65d10 100644--- a/src/dopplergeist/search.py+++ b/src/dopplergeist/search.py@@ -335,7 +335,49 @@ def _ridge_trial_classification(result, events, association):+_CEILING_FLOAT_FIELDS = (+ "support_fraction", "score", "f0_hz", "source_drift_hz", "source_drift_hz_s",+ "source_reference_emission_time_s", "source_span_s", "drift_bound_hz", "rmse_hz", "outlier_fraction",+)++def _association_event_ceiling(diagnostic, score, frequency, event, limit):+ """Reuse a validated ceiling only for the same complete scalar evidence.++ Fit dictionaries are copied for each association. Content keys preserve+ mutation checks while avoiding repeated validation of the same clock fit.+ Noncanonical numeric types retain the original validation and arithmetic.+ """+ from .matching import _ridge_fit_cache+ cache = _ridge_fit_cache.get()+ if (cache is None or cache.max_entries <= 0 or type(diagnostic) is not dict+ or type(event) is not dict or type(event.get("quality", {})) is not dict):+ return _association_event_ceiling_uncached(diagnostic, score, frequency, event, limit)+ bounds = event.get("f0_range_hz")+ if type(bounds) not in (list, tuple) or len(bounds) != 2:+ return _association_event_ceiling_uncached(diagnostic, score, frequency, event, limit)+ values = tuple(diagnostic.get(name) for name in _CEILING_FLOAT_FIELDS)+ fraction = event.get("quality", {}).get("source_drift_fraction", 0.02)+ counts = (diagnostic.get("sample_count"), diagnostic.get("independent_sample_count"))+ optimizer = diagnostic.get("optimizer_success", True)+ if (not all(type(value) is float for value in (*values, score, frequency, limit, *bounds, fraction))+ or not all(type(value) is int for value in counts) or type(optimizer) is not bool):+ return _association_event_ceiling_uncached(diagnostic, score, frequency, event, limit)+ key = (_ridge_trial_classification, values, counts, optimizer, score, frequency,+ tuple(bounds), fraction, limit)+ entries = cache.ceiling_entries+ previous = entries.get(key)+ if previous is not None:+ entries.move_to_end(key)+ return previous+ result = _association_event_ceiling_uncached(diagnostic, score, frequency, event, limit)+ entries[key] = result+ if len(entries) > cache.max_entries:+ entries.popitem(last=False)+ return result+++def _association_event_ceiling_uncached(diagnostic, score, frequency, event, limit):"""Validate one fitted group's contribution before tightening a ceiling."""single = {"score_kind": "negative_robust_ridge_loss_hz", "score": score,"event_scores": [score], "f0_hz": [frequency], "event_diagnostics": [diagnostic]}diff --git a/tests/test_real_batch_ridge.py b/tests/test_real_batch_ridge.pyindex 5d9b403..47711eb 100644--- a/tests/test_real_batch_ridge.py+++ b/tests/test_real_batch_ridge.py@@ -71,9 +71,18 @@ def test_real_association_results_match_scalar_fits(recorded, monkeypatch):import dopplergeist.batch_ridge as batch_ridgefixture, events, measurements, table, tracks = recorded- candidates, _ = association_candidates(table, events, max_candidates=20000)+ options = {}+ if "recording_start_window_s" in fixture:+ from dopplergeist.tiled_search import _association_clock_window+ # Preserve the frozen workload's declared clock domain. Enumerating+ # every possible clock from a broad CPA table changes that workload+ # and can exhaust its association budget before this fit check runs.+ center, tolerance = _association_clock_window(+ fixture["recording_start_window_s"], fixture.get("scoring", {}).get("time_sweep_s", 5.0))+ options = {"time_origin": center, "time_tolerance_s": tolerance}+ candidates, _ = association_candidates(table, events, max_candidates=20000, **options)selected = sorted(random.Random(20261103).sample(range(len(candidates)), min(100, len(candidates))))- point = SimpleNamespace(**fixture["point"], x=0., y=0.)+ point = SimpleNamespace(**{"x": 0., "y": 0., **fixture["point"]})@_cache_ridge_fitsdef run():diff --git a/tests/test_real_refinement_lookup.py b/tests/test_real_refinement_lookup.pynew file mode 100644index 0000000..265190e--- /dev/null+++ b/tests/test_real_refinement_lookup.py@@ -0,0 +1,116 @@+"""Named refinement reads against actual indexed aircraft observations."""+import json+import os+from pathlib import Path+import shutil+import sqlite3++import numpy as np+import pytest++from dopplergeist.automatic import _refinement_tracks+from dopplergeist.data_access import DataUnavailable, FlightIndex++[email protected](scope="module")+def recorded_index():+ source = os.environ.get("DOPPLERGEIST_REAL_CPA_FIXTURE")+ if not source:+ pytest.skip("set DOPPLERGEIST_REAL_CPA_FIXTURE to a recorded-data diagnostic")+ fixture = json.loads(Path(source).read_text())+ index = FlightIndex(fixture["real_index"])+ assert index.report["input_hash"] == fixture["index_input_hash"]+ bounds, times = fixture["query_bounds"], fixture["query_time_s"]+ observed = list(index.query(bounds, times))+ assert observed, "The recorded workload contains no trajectories"+ assert [track.id for track in observed] == sorted(fixture["track_ids"])+ return index, bounds, times, observed+++def same_tracks(actual, expected):+ assert [track.id for track in actual] == [track.id for track in expected]+ for left, right in zip(actual, expected):+ for name in ("time", "lat", "lon", "altitude", "xyz", "relative_time", "velocity"):+ a, b = getattr(left, name), getattr(right, name)+ if a is None or b is None:+ assert a is b+ else:+ np.testing.assert_array_equal(a, b, strict=True)+ np.testing.assert_array_equal(left.spline.c, right.spline.c, strict=True)+ np.testing.assert_array_equal(left.derivative.c, right.derivative.c, strict=True)+ assert left.origin == right.origin+ for name in ("typecode", "altitude_basis", "aircraft_class", "velocity_inferred_components"):+ assert getattr(left, name) == getattr(right, name)+++def test_named_envelope_matches_broad_query_across_sql_batches(recorded_index):+ index, bounds, times, observed = recorded_index+ with index.connect() as database:+ names = [row[0] for row in database.execute("SELECT name FROM segments ORDER BY name LIMIT 650")]+ # The real index contains enough names to cross the 500-parameter batch.+ assert len(names) > 500+ names = sorted(set(names) | {track.id for track in observed[:6]})+ requested = names[::-1] + names[:3] + ["missing-recorded-segment"]+ selected = set(requested)+ expected = [track for track in observed if track.id in selected]+ actual = list(index.lookup(requested, bounds=bounds, time_range=times))+ same_tracks(actual, expected)+++def test_named_envelope_preserves_spatial_time_filters_and_old_index_fallback(recorded_index):+ original, bounds, times, observed = recorded_index+ index = FlightIndex(original.path)+ names = [track.id for track in observed[:6]]+ # A disjoint geographic envelope and out-of-day time query expose accidental+ # removal of either predicate without manufacturing trajectory observations.+ disjoint = (bounds[0] + 20, bounds[1], bounds[2] + 20, bounds[3])+ outside_time = (index.report["coverage"]["time_range_s"][1] + 86400,) * 2+ for box, window in ((disjoint, times), (bounds, outside_time),+ (bounds, (observed[0].time[0], observed[0].time[0]))):+ expected = [track for track in index.query(box, window) if track.id in names]+ if window == outside_time:+ assert not expected+ if window == (observed[0].time[0], observed[0].time[0]):+ assert observed[0].id in {track.id for track in expected}+ same_tracks(list(index.lookup(names, bounds=box, time_range=window)), expected)+ # Older index metadata must disable spatial pruning in both entry points.+ index.report["continuous_geometry_bounds"] = False+ expected = [track for track in index.query(disjoint, times) if track.id in names]+ assert [track.id for track in expected] == names+ same_tracks(list(index.lookup(names, bounds=disjoint, time_range=times)), expected)+ index.report["continuous_geometry_bounds"] = True+ index.report["normalizer_sha256"] = "different-normalizer-version"+ same_tracks(list(index.lookup(names, bounds=disjoint, time_range=times)), expected)+++def test_refinement_adapter_duplicates_and_sample_budget(recorded_index):+ index, bounds, times, observed = recorded_index+ needed = {track.id for track in observed[:6]}+ expected = [track for track in observed if track.id in needed]+ sample_limit = sum(len(track.time) for track in expected)++ class QueryAdapter:+ def query(self, box, window):+ assert tuple(box) == tuple(bounds) and tuple(window) == tuple(times)+ yield from observed++ for indexes in ([index], [index, index], [QueryAdapter()], [index, QueryAdapter()]):+ actual = _refinement_tracks(indexes, needed, bounds, times, sample_limit)+ same_tracks(list(actual.values()), expected)+ with pytest.raises(ValueError, match="^Refinement trajectory sample budget exceeded$"):+ _refinement_tracks(indexes, needed, bounds, times, sample_limit - 1)+ assert _refinement_tracks([index], {"missing-recorded-segment"}, bounds, times, 1) == {}+++def test_named_envelope_requires_complete_scope_and_checks_payload(recorded_index, tmp_path):+ index, bounds, times, observed = recorded_index+ name = observed[0].id+ for settings in ({"bounds": bounds}, {"time_range": times}):+ with pytest.raises(ValueError, match="must be supplied together"):+ list(index.lookup([name], **settings))+ path = tmp_path / "corrupt-index.sqlite"+ shutil.copyfile(index.path, path)+ with sqlite3.connect(path) as database:+ database.execute("UPDATE segments SET payload_hash=? WHERE name=?", ("corrupt", name))+ with pytest.raises(DataUnavailable, match="Corrupt flight payload"):+ list(FlightIndex(path).lookup([name], bounds=bounds, time_range=times))