Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Prepare verified trajectory inputs and reduce exact search work
Cinna Davisecb1726fb987
diff --git a/.github/workflows/real-performance.yml b/.github/workflows/real-performance.ymlindex d727775..19d2d43 100644--- a/.github/workflows/real-performance.yml+++ b/.github/workflows/real-performance.yml@@ -4,18 +4,20 @@ on:inputs:baseline:description: Exact baseline commit for both real-input comparisons- default: 27da08a878ac2ec90cd7e2c07f43765ceeea609b+ default: 994d1ee48e25df6d30b23a356bba8f6502d8b7f6required: trueregional:description: Also compare the complete regional HTTP workflowtype: booleandefault: truepush:- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103]+ branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103, perf/prepared-flight-search-20261103]paths:- .github/workflows/real-performance.yml- src/dopplergeist/**- scripts/benchmark_prepared_measurements.py+ - scripts/benchmark_prepared_flight_search.py+ - scripts/prepare_flight_index.py- scripts/benchmark_real_cpa_rows.py- scripts/export_real_acoustic_controls.py- scripts/benchmark_real_regional_upload.py@@ -38,7 +40,7 @@ env:NUMEXPR_NUM_THREADS: '1'CC: gccPYTHONPATH: src- BASELINE_REF: ${{ inputs.baseline || '27da08a878ac2ec90cd7e2c07f43765ceeea609b' }}+ BASELINE_REF: ${{ inputs.baseline || '994d1ee48e25df6d30b23a356bba8f6502d8b7f6' }}jobs:compare:if: github.event_name != 'push' || github.ref != 'refs/heads/perf/real-pipeline-throughput-20261103'@@ -92,6 +94,12 @@ jobs:--full-day --max-tracks 2000 --max-samples 500000 --max-candidates 20000 \--anchor-window-s 1788864888.7151732 1788865236.7214916 \--include-track-query --output benchmark-inputs/dense-query.json+ - name: Prepare reusable numeric trajectory inputs+ run: |+ python scripts/benchmark_prepared_flight_search.py prepare-index \+ --source src --index benchmark-inputs/reference-region.sqlite \+ --destination benchmark-inputs/reference-region-prepared.sqlite \+ --output results/flight-preparation.json- name: Export real pipeline objective controlsenv:DOPPLERGEIST_RIDGE_BACKEND: python@@ -102,7 +110,7 @@ jobs:DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.jsonDOPPLERGEIST_REAL_REFERENCE_ROOT: benchmark-inputs/reference-checkoutDOPPLERGEIST_REAL_REFERENCE_DISCOVERY: evidence/audio/w1-sensitivity-20261102/reference/discovery.json- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_native_ridge.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py+ run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_native_ridge.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py tests/test_real_native_convergence.py tests/test_real_flight_payload.py tests/test_real_prepared_index.py tests/test_real_trial_vectors.py- name: Verify the explicit Python fallback on recorded arithmeticenv:DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json@@ -129,6 +137,14 @@ jobs:--baseline-source "$RUNNER_TEMP/baseline/src" --current-source src \--repeats 3 --timeout-s 600 --output results/performance-dense-query \"${native_option[@]}"+ taskset -c "$benchmark_cpu" python scripts/benchmark_prepared_flight_search.py compare \+ --fixture benchmark-inputs/dense-query.json \+ --baseline-source "$RUNNER_TEMP/baseline/src" --current-source src \+ --baseline-index benchmark-inputs/reference-region-prepared.sqlite \+ --current-index benchmark-inputs/reference-region-prepared.sqlite \+ --preparation-report results/flight-preparation.json \+ --repeats 3 --timeout-s 600 --output results/performance-prepared-flight-search \+ --require-native --require-native-acoustic --require-current-prepared- name: Profile remaining point work outside the timing comparisonrun: |python -m cProfile -o results/current-point.prof scripts/benchmark_prepared_measurements.py _worker --source src --fixture benchmark-inputs/point-120.json --output results/current-point-profile --require-native --require-native-acousticdiff --git a/scripts/benchmark_prepared_flight_search.py b/scripts/benchmark_prepared_flight_search.pynew file mode 100644index 0000000..b70e45e--- /dev/null+++ b/scripts/benchmark_prepared_flight_search.py@@ -0,0 +1,484 @@+"""Measure real flight-query, overload-sieve and scoring work with exact parity.++Use a frozen recorded-data fixture and two pinned src/ directories. Each arm runs+in a fresh interpreter. An optional prepared index changes representation only;+its one-time construction is measured separately by the prepare-index command.+The source-informed regional receiver and anchor partition are performance+workloads, not independent localization or complete national-search evidence.+"""+from __future__ import annotations++import argparse+import hashlib+import json+import math+import os+from pathlib import Path+import statistics+import subprocess+import sys+import time++try:+ import resource+except ImportError:+ resource = None++ROOT = Path(__file__).resolve().parents[1]++# Query and scientific-output capture adapted from the project's PR #258+# benchmark_prepared_measurements.py; keep the original scoring harness intact.+++def json_scalar(value):+ if hasattr(value, "tolist"):+ return value.tolist()+ raise TypeError(f"Cannot serialize {type(value).__name__}")++def write_new(path, value):+ path = Path(path)+ path.parent.mkdir(parents=True, exist_ok=True)+ with path.open("x") as output:+ json.dump(value, output, indent=2, allow_nan=True, default=json_scalar)+ output.write("\n")++def canonical(value):+ return json.dumps(value, sort_keys=True, separators=(",", ":"),+ allow_nan=True, default=json_scalar)++def query_tracks(index, points, events, budget, scoring, bounds, time_range):+ """Count actual index traversal while exercising the production overload path."""+ from dopplergeist.tiled_search import _tracks_for_points++ if "elev_m" not in points:+ raise ValueError("The recorded receiver hypothesis must include its elevation")+ operations = {"query_calls": 0, "decoded_track_yields": 0, "decoded_sample_yields": 0}+ inventory = {}+ ordered_inventory = []++ class CountedIndex:+ def query(self, query_bounds, query_time):+ operations["query_calls"] += 1+ for track in index.query(query_bounds, query_time):+ operations["decoded_track_yields"] += 1+ operations["decoded_sample_yields"] += len(track.time)+ inventory[track.id] = len(track.time)+ ordered_inventory.append([track.id, len(track.time)])+ yield track++ radius = max(event["slant_range_m"][1] for event in events)+ tracks, points = _tracks_for_points(+ [CountedIndex()], bounds, time_range, budget, points, None, radius,+ scoring.get("c", 343.0))+ accounting = {+ "queried_tracks_in_order": ordered_inventory,+ "queried_track_ids": sorted(inventory),+ "queried_trajectory_samples": sum(inventory.values()),+ "selected_track_ids": [track.id for track in tracks],+ "selected_trajectory_samples": sum(len(track.time) for track in tracks),+ }+ return tracks, points, operations, accounting++def worker(args):+ source = args.source.resolve()+ sys.path.insert(0, str(source))+ import dopplergeist+ import pandas as pd+ from dopplergeist import geometry, matching, tiled_search+ from dopplergeist.data_access import FlightIndex+ from dopplergeist.finite_search import RetentionLedger+ from dopplergeist.geometry import AcousticSupportError, Track++ if Path(dopplergeist.__file__).resolve().parent != source / "dopplergeist":+ raise RuntimeError("Comparison imported a different source tree")+ native_fit = getattr(matching, "_native_fit_irls", None)+ ridge_backend = getattr(matching, "RIDGE_FIT_BACKEND", "python-irls-v1")+ acoustic_backend = getattr(geometry, "ACOUSTIC_BACKEND", "python-acoustic-v1")+ try:+ from dopplergeist.native_acoustic import _native as native_acoustic+ except ImportError:+ native_acoustic = None+ if args.require_native and not callable(native_fit):+ raise RuntimeError("The selected source's native ridge fitter is unavailable")+ if args.require_native_acoustic and native_acoustic is None:+ raise RuntimeError("The selected source's native acoustic solver is unavailable")+ fixture = json.loads(args.fixture.read_text())+ index = FlightIndex(args.index or fixture["real_index"],+ track_cache_bytes=args.track_cache_bytes)+ if index.report["input_hash"] != fixture["index_input_hash"]:+ raise ValueError("The real trajectory index no longer matches the fixture")+ if args.require_prepared and not getattr(index, "_prepared_tracks", False):+ raise RuntimeError("The selected source cannot use this prepared index")+ budget = tiled_search.SearchBudget(**fixture["budget"])+ loaded_at = time.perf_counter()+ tracks = None+ if not args.include_track_query:+ tracks = list(index.lookup(fixture["track_ids"], max_tracks=budget.max_tracks_per_tile,+ max_samples=budget.max_track_samples_per_tile))+ if sorted(track.id for track in tracks) != sorted(fixture["track_ids"]):+ raise ValueError("The index is missing a recorded trajectory")+ events, measurements = matching.prepare_events(fixture["events"])+ points = pd.DataFrame([fixture["point"]])+ load_seconds = time.perf_counter() - loaded_at+ args.output.mkdir(parents=True, exist_ok=False)+ ledger = RetentionLedger(args.output / "retention.sqlite", "recorded-point-benchmark",+ "matched_group_support_minus_normalized_residual")+ stage = "coarse:recorded-point"+ operations = {"scalar_fit_calls": 0, "acoustic_calls": 0, "scored_candidates": 0,+ "native_fit_calls": 0, "native_fit_successes": 0,+ "native_acoustic_successes": 0, "prepared_decode_calls": 0,+ "prepared_decode_successes": 0}+ all_rows = []+ original_fit, original_acoustic = matching._fit_ridge_uncached, Track.acoustic+ original_score = tiled_search.score_candidate+ phase_seconds, phase_patches = {}, []+ original_decoder, payload_module = None, None+ if getattr(index, "_prepared_tracks", False):+ from dopplergeist import flight_payload as payload_module+ original_decoder = payload_module.decode_track++ def counted_decode(*values, **options):+ operations["prepared_decode_calls"] += 1+ track = original_decoder(*values, **options)+ operations["prepared_decode_successes"] += 1+ return track++ payload_module.decode_track = counted_decode++ def timed_phase(name, function):+ def run(*values, **options):+ began, cpu = time.perf_counter(), time.process_time()+ try:+ return function(*values, **options)+ finally:+ phase = phase_seconds.setdefault(name, {"wall_seconds": 0.0, "cpu_seconds": 0.0, "calls": 0})+ phase["wall_seconds"] += time.perf_counter() - began+ phase["cpu_seconds"] += time.process_time() - cpu+ phase["calls"] += 1+ return run++ if args.stage_timing:+ phase_targets = [+ (Track, "outside_observer_radius", "distance_sieve"),+ (matching, "association_candidates", "association_enumeration"),+ (tiled_search, "merge_candidates", "retention"),+ ]+ if callable(getattr(tiled_search, "cpa_table", None)):+ phase_targets.append((tiled_search, "cpa_table", "closest_approach"))+ else:+ prepared_cpa = getattr(tiled_search, "_PreparedCPATrack", None)+ if prepared_cpa is not None:+ phase_targets.extend([+ (prepared_cpa, "__init__", "closest_approach_preparation"),+ (prepared_cpa, "raw_rows", "closest_approach"),+ ])+ for owner, attribute, name in phase_targets:+ function = getattr(owner, attribute, None)+ if function is not None:+ phase_patches.append((owner, attribute, function))+ setattr(owner, attribute, timed_phase(name, function))+ score_function = timed_phase("candidate_scoring", original_score) if args.stage_timing else original_score++ def counted_fit(*values, **options):+ operations["scalar_fit_calls"] += 1+ return original_fit(*values, **options)++ def counted_acoustic(*values, **options):+ operations["acoustic_calls"] += 1+ return original_acoustic(*values, **options)++ def counted_native(*values, **options):+ operations["native_fit_calls"] += 1+ result = native_fit(*values, **options)+ if result is not None and all(math.isfinite(value) for value in result):+ operations["native_fit_successes"] += 1+ return result++ def captured_score(*values, **options):+ operations["scored_candidates"] += 1+ try:+ result = score_function(*values, **options)+ except AcousticSupportError as error:+ all_rows.append({"support_failure": tiled_search._support_error(error)})+ raise+ all_rows.append(result)+ return result++ matching._fit_ridge_uncached, Track.acoustic = counted_fit, counted_acoustic+ tiled_search.score_candidate = captured_score+ if callable(native_fit):+ matching._native_fit_irls = counted_native+ native_acoustic_before = native_acoustic.successful_calls() if native_acoustic is not None else 0+ query_accounting, query_operations, execution_stages = None, None, {}+ began, cpu = time.perf_counter(), time.process_time()+ try:+ if args.include_track_query:+ query_began, query_cpu = time.perf_counter(), time.process_time()+ tracks, points, query_operations, query_accounting = query_tracks(+ index, points, events, budget, fixture["scoring"],+ fixture["query_bounds"], fixture["query_time_s"])+ execution_stages["track_query_and_distance_sieve"] = {+ "wall_seconds": time.perf_counter() - query_began,+ "cpu_seconds": time.process_time() - query_cpu,+ }+ if query_accounting["selected_track_ids"] != fixture["track_ids"]:+ raise AssertionError("The real query changed the ordered selected-track inventory")+ if query_accounting["selected_trajectory_samples"] != fixture["counts"]["trajectory_samples"]:+ raise AssertionError("The real query changed the selected trajectory sample count")+ prepared_query = fixture.get("track_query_preparation")+ if prepared_query is not None:+ if (len(query_accounting["queried_track_ids"]) != prepared_query["queried_trajectories"]+ or query_accounting["queried_trajectory_samples"] != prepared_query["queried_trajectory_samples"]):+ raise AssertionError("The real query changed the complete observed-track inventory")+ scoring_began, scoring_cpu = time.perf_counter(), time.process_time()+ partition = ({"anchor_window": fixture["anchor_window_s"]}+ if "anchor_window_s" in fixture else {})+ ranked, counts = tiled_search._score(+ points, tracks, events, measurements, budget, fixture["scoring"], {},+ start_window=fixture["recording_start_window_s"], ledger=ledger, stage=stage, **partition)+ if args.include_track_query:+ execution_stages["cpa_association_scoring_and_retention"] = {+ "wall_seconds": time.perf_counter() - scoring_began,+ "cpu_seconds": time.process_time() - scoring_cpu,+ }+ finally:+ elapsed, cpu_seconds = time.perf_counter() - began, time.process_time() - cpu+ matching._fit_ridge_uncached, Track.acoustic = original_fit, original_acoustic+ tiled_search.score_candidate = original_score+ if callable(native_fit):+ matching._native_fit_irls = native_fit+ if native_acoustic is not None:+ operations["native_acoustic_successes"] = native_acoustic.successful_calls() - native_acoustic_before+ for owner, attribute, function in reversed(phase_patches):+ setattr(owner, attribute, function)+ if payload_module is not None:+ payload_module.decode_track = original_decoder+ expected_counts = fixture.get("expected_counts")+ if expected_counts is not None:+ actual_counts = {"cpa_rows": counts["cpa_rows"],+ "associations": counts["candidates"], "retained": len(ranked)}+ if actual_counts != expected_counts:+ raise AssertionError(f"Recorded workload counts changed: {actual_counts}")+ # macOS reports bytes and Linux reports KiB. Keep scoring usable where+ # resource or its unit convention is unavailable, without guessing a value.+ peak_rss_mib = None+ if resource is not None and (sys.platform == "darwin" or sys.platform.startswith("linux")):+ rss_divisor = 1024 * 1024 if sys.platform == "darwin" else 1024+ peak_rss_mib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rss_divisor+ checksum = ledger.checksum(stage)+ if not ledger.stage_valid(stage, checksum):+ raise AssertionError("The scoring pass produced an invalid retention ledger")+ if not operations["scored_candidates"]:+ raise AssertionError("The benchmark did not score any real associations")+ if args.require_native and not operations["native_fit_successes"]:+ raise AssertionError("No successful native ridge fit occurred inside the timed scoring call")+ if args.require_native_acoustic and not operations["native_acoustic_successes"]:+ raise AssertionError("No successful native acoustic solve occurred inside the timed scoring call")+ if args.require_prepared and not operations["prepared_decode_successes"]:+ raise AssertionError("No successful prepared decode occurred inside the timed query")+ with ledger.connect() as database:+ retention = {name: database.execute(f"SELECT * FROM {name} ORDER BY 1,2").fetchall()+ for name in ("records", "bounds", "stages")}+ results = {"ranked": ranked, "counts": counts,+ "all_scored_candidates": all_rows, "retention": retention}+ if query_accounting is not None:+ results["track_query_accounting"] = query_accounting+ results["workload"] = {key: fixture.get(key) for key in (+ "index_input_hash", "point", "events", "query_bounds", "query_time_s",+ "recording_start_window_s", "anchor_window_s", "budget", "scoring")}+ results["index_coverage"] = index.report["coverage"]+ write_new(args.output / "results.json", results)+ summary = {"wall_seconds": elapsed, "cpu_seconds": cpu_seconds,+ "load_seconds_excluded": load_seconds, "operations": operations,+ "ridge_fit_backend": ridge_backend, "acoustic_backend": acoustic_backend,+ "native_extension_available": callable(native_fit),+ "peak_rss_mib": peak_rss_mib,+ "peak_rss_scope": ("Worker lifetime through scoring, including imports and input loading"+ if peak_rss_mib is not None else "Unavailable on this platform"),+ "retained": len(ranked), "independent_groups": len(events), "scope": fixture["scope"]}+ summary["index_prepared_tracks"] = index.report.get("prepared_tracks")+ summary["track_cache_bytes"] = args.track_cache_bytes+ if "anchor_window_s" in fixture:+ summary["anchor_window_s"] = fixture["anchor_window_s"]+ if args.include_track_query:+ summary["track_query_operations"] = query_operations+ summary["execution_stage_seconds"] = execution_stages+ summary["timing_scope"] = (+ "Initial local index traversal and conservative distance sieve, followed by complete "+ "CPA/association scoring and retention. Excludes imports, index metadata, event "+ "preparation, downloads, terrain sampling, audio discovery, and refinement.")+ if args.stage_timing:+ summary["phase_seconds"] = phase_seconds+ summary["phase_timing_scope"] = "Inclusive instrumented call times; overlapping phases must not be summed."+ write_new(args.output / "summary.json", summary)+ print(json.dumps(summary), flush=True)+++def file_digest(path):+ with Path(path).open("rb") as stream:+ return hashlib.file_digest(stream, "sha256").hexdigest()+++def prepare_index(args):+ """Time a new optional representation without changing the original index."""+ sys.path.insert(0, str(args.source.resolve()))+ import dopplergeist+ from dopplergeist.data_access import FlightIndex++ if Path(dopplergeist.__file__).resolve().parent != args.source.resolve() / "dopplergeist":+ raise RuntimeError("Preparation imported a different source tree")+ if args.output.exists() or args.destination.exists():+ raise FileExistsError("Choose new preparation output and index paths")+ before = file_digest(args.index)+ began, cpu = time.perf_counter(), time.process_time()+ original = FlightIndex(args.index)+ prepared = original.prepare(args.destination)+ elapsed, cpu_seconds = time.perf_counter() - began, time.process_time() - cpu+ if before != file_digest(args.index):+ raise AssertionError("Preparation modified the original observed index")+ for key in ("input_hash", "coverage", "sources"):+ if original.report.get(key) != prepared.report.get(key):+ raise AssertionError(f"Preparation changed the original {key}")+ if not getattr(prepared, "_prepared_tracks", False):+ raise AssertionError("The optional representation is unavailable to its builder")+ report = {+ "wall_seconds": elapsed, "cpu_seconds": cpu_seconds,+ "source": str(args.source.resolve()), "index": str(args.index.resolve()),+ "destination": str(args.destination.resolve()),+ "input_hash": original.report["input_hash"],+ "original_unchanged": True,+ "source_bytes": args.index.stat().st_size,+ "prepared_bytes": args.destination.stat().st_size,+ "prepared_tracks": prepared.report["prepared_tracks"],+ "timing_scope": "One-time index copy, canonical decoding, optional representation and commit; excludes imports and verification.",+ "cache_scope": "Source integrity verification reads the original index before timing; operating-system file caches are not flushed.",+ }+ write_new(args.output, report)+ print(json.dumps(report), flush=True)+++def compare(args):+ args.output.mkdir(parents=True, exist_ok=False)+ sources = {"baseline": args.baseline_source.resolve(), "current": args.current_source.resolve()}+ fixture = json.loads(args.fixture.read_text())+ indexes = {"baseline": Path(args.baseline_index or fixture["real_index"]).resolve(),+ "current": Path(args.current_index or fixture["real_index"]).resolve()}+ environment = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")+ for variable in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS"):+ environment[variable] = "1"+ runs, expected = [], None+ for repetition in range(args.repeats):+ order = ("baseline", "current") if repetition % 2 == 0 else ("current", "baseline")+ for name in order:+ output = args.output / f"{repetition + 1}-{name}"+ command = [sys.executable, str(Path(__file__).resolve()), "_worker",+ "--source", str(sources[name]), "--fixture", str(args.fixture.resolve()),+ "--index", str(indexes[name]), "--output", str(output.resolve()),+ "--track-cache-bytes", str(args.track_cache_bytes)]+ for flag in ("stage_timing", "require_native", "require_native_acoustic"):+ if getattr(args, flag):+ command.append("--" + flag.replace("_", "-"))+ if name == "current" and args.require_current_prepared:+ command.append("--require-prepared")+ print(f"Starting pair {repetition + 1}: {name}", flush=True)+ subprocess.run(command, check=True, env=environment, timeout=args.timeout_s)+ payload = canonical(json.loads((output / "results.json").read_text()))+ if expected is None:+ expected = payload+ elif payload != expected:+ raise AssertionError(f"Exact inventory, scientific-output or retention mismatch: {output}")+ summary = json.loads((output / "summary.json").read_text())+ runs.append({"source": name, "repetition": repetition + 1, **summary})+ medians = {name: {} for name in sources}+ for name in sources:+ for metric in ("wall_seconds", "cpu_seconds", "peak_rss_mib"):+ values = [row[metric] for row in runs if row["source"] == name and row[metric] is not None]+ medians[name][metric] = statistics.median(values) if values else None+ report = {+ "identical_results": True, "repeats": args.repeats,+ "fixture": str(args.fixture.resolve()),+ "sources": {name: str(path) for name, path in sources.items()},+ "indexes": {name: str(path) for name, path in indexes.items()},+ "medians": medians,+ "speedup": {metric: medians["baseline"][metric] / medians["current"][metric]+ for metric in ("wall_seconds", "cpu_seconds")},+ "execution_stage_medians": {+ name: {stage: {metric: statistics.median(+ row["execution_stage_seconds"][stage][metric]+ for row in runs if row["source"] == name)+ for metric in ("wall_seconds", "cpu_seconds")}+ for stage in runs[0]["execution_stage_seconds"]}+ for name in sources},+ "scope": runs[0]["scope"], "timing_scope": runs[0]["timing_scope"],+ "cache_scope": "Fresh interpreter and FlightIndex per arm; operating-system file caches are not flushed.",+ "runs": runs,+ }+ if args.preparation_report:+ preparation = json.loads(args.preparation_report.read_text())+ if (Path(preparation["destination"]).resolve() != indexes["current"]+ or preparation["input_hash"] != fixture["index_input_hash"]):+ raise ValueError("Preparation report does not describe the compared current index")+ report["preparation"] = preparation+ report["preparation_amortization_scope"] = (+ "Preparation cost divided by the complete baseline/current query-and-score saving. "+ "This includes other code improvements and measures the combined upgrade, "+ "not the isolated prepared-format contribution.")+ report["queries_to_amortize_preparation"] = {}+ for metric in ("wall_seconds", "cpu_seconds"):+ saved = medians["baseline"][metric] - medians["current"][metric]+ report["queries_to_amortize_preparation"][metric] = (+ max(1, math.ceil(preparation[metric] / saved)) if saved > 0 else None)+ write_new(args.output / "comparison.json", report)+ print(json.dumps({key: report[key] for key in (+ "identical_results", "medians", "speedup", "execution_stage_medians", "scope")}), flush=True)+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ commands = parser.add_subparsers(dest="command", required=True)+ preparation = commands.add_parser("prepare-index", help="Measure one-time creation of an optional index representation")+ preparation.add_argument("--source", type=Path, default=ROOT / "src")+ preparation.add_argument("--index", type=Path, required=True)+ preparation.add_argument("--destination", type=Path, required=True)+ preparation.add_argument("--output", type=Path, required=True)+ comparison = commands.add_parser("compare", help="Compare equal real query/sieve/scoring workloads")+ comparison.add_argument("--fixture", required=True, type=Path)+ comparison.add_argument("--baseline-source", required=True, type=Path)+ comparison.add_argument("--current-source", type=Path, default=ROOT / "src")+ comparison.add_argument("--baseline-index", type=Path)+ comparison.add_argument("--current-index", type=Path)+ comparison.add_argument("--preparation-report", type=Path)+ comparison.add_argument("--repeats", type=int, default=3)+ comparison.add_argument("--timeout-s", type=float, default=900.0)+ comparison.add_argument("--output", required=True, type=Path)+ comparison.add_argument("--require-current-prepared", action="store_true")+ internal = commands.add_parser("_worker", help=argparse.SUPPRESS)+ internal.add_argument("--source", required=True, type=Path)+ internal.add_argument("--fixture", required=True, type=Path)+ internal.add_argument("--index", type=Path)+ internal.add_argument("--output", required=True, type=Path)+ internal.add_argument("--require-prepared", action="store_true")+ for command in (comparison, internal):+ command.add_argument("--track-cache-bytes", type=int, default=32 * 1024 * 1024)+ command.add_argument("--stage-timing", action="store_true",+ help="Collect inclusive call times in a separate diagnostic run")+ command.add_argument("--require-native", action="store_true",+ help="Require successful native ridge fits in every compared source")+ command.add_argument("--require-native-acoustic", action="store_true",+ help="Require successful native acoustic solves in every compared source")+ command.set_defaults(include_track_query=True)+ args = parser.parse_args()+ if hasattr(args, "track_cache_bytes") and args.track_cache_bytes < 0:+ parser.error("--track-cache-bytes must be nonnegative")+ if args.command == "prepare-index":+ prepare_index(args)+ elif args.command == "compare":+ if args.repeats < 1 or not math.isfinite(args.timeout_s) or args.timeout_s <= 0:+ parser.error("--repeats and --timeout-s must be positive")+ compare(args)+ else:+ worker(args)+++if __name__ == "__main__":+ main()diff --git a/scripts/prepare_flight_index.py b/scripts/prepare_flight_index.pynew file mode 100644index 0000000..3222191--- /dev/null+++ b/scripts/prepare_flight_index.py@@ -0,0 +1,31 @@+"""Prepare a separately named real flight index for repeated search queries.++The copy retains observations, coverage and input identity. It adds validated+numeric track inputs and keeps the original JSON for older readers and later+normalization changes. No trajectories are generated.+"""+import argparse+import json+from pathlib import Path+import time++from dopplergeist.data_access import FlightIndex+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--index", required=True, type=Path)+ parser.add_argument("--output", required=True, type=Path)+ args = parser.parse_args()+ began = time.perf_counter()+ result = FlightIndex(args.index).prepare(args.output)+ print(json.dumps({"output": str(args.output.resolve()),+ "elapsed_s": time.perf_counter() - began,+ "source_bytes": args.index.stat().st_size,+ "prepared_bytes": args.output.stat().st_size,+ "input_hash": result.report["input_hash"],+ "prepared_tracks": result.report["prepared_tracks"]}, indent=2))+++if __name__ == "__main__":+ main()diff --git a/src/dopplergeist/_native_acoustic.c b/src/dopplergeist/_native_acoustic.cindex a591139..9a47080 100644--- a/src/dopplergeist/_native_acoustic.c+++ b/src/dopplergeist/_native_acoustic.c@@ -8,6 +8,10 @@#include <fenv.h>#include <string.h>+/* Larger calls keep the original all-lane solver instead of allocating+ * unbounded temporary state or falling back to the Python implementation. */+#define DG_MAX_STATE_BYTES ((size_t)2 * 1024 * 1024)+/* Compile without contraction or reassociation. SciPy PPoly uses* increasing powers, not Horner evaluation. Preserve that arithmetic exactly.*/@@ -23,10 +27,20 @@ static size_t interval_at(const double *x, size_t m, double value) {}static int evaluate(const double *coefficients, const double *x, size_t m,- size_t width, double origin, double t, double *values) {+ size_t width, double origin, double t, double *values,+ uint16_t *interval) {double relative = t - origin;if (!(relative >= x[0] && relative <= x[m])) return 1;- size_t i = interval_at(x, m, relative);+ size_t i = interval == NULL ? m : *interval;+ /* Newton updates usually stay in one native spline interval. Reuse only+ * after checking its exact half-open bounds; the last interval owns x[m].+ * Both spline shapes are bounded to 4096 intervals by the public wrapper.+ */+ if (!(i < m && relative >= x[i] &&+ (relative < x[i+1] || (i+1 == m && relative == x[m])))) {+ i = interval_at(x, m, relative);+ if (interval != NULL) *interval = (uint16_t)i;+ }double s = relative - x[i];double z = 1.0;for (size_t j = 0; j < width; ++j) values[j] = 0.0;@@ -52,9 +66,9 @@ static double norm3(const double *a) {static int kinematics(const double *coefficients, const double *x,size_t m, double origin, double tau, const double *observer,- double *distance, double *radial) {+ double *distance, double *radial, uint16_t *interval) {double values[6], delta[3];- if (evaluate(coefficients, x, m, 6, origin, tau, values)) return 1;+ if (evaluate(coefficients, x, m, 6, origin, tau, values, interval)) return 1;for (size_t j = 0; j < 3; ++j) delta[j] = values[j] - observer[j];*distance = norm3(delta);*radial = (((0.0 + delta[0]*values[3]) + delta[1]*values[4])@@ -67,12 +81,14 @@ static int dg_acoustic(const double *t, size_t n, const double *observer,const double *packed, const double *packed_x, size_t packed_m,double origin, double start, double end, double c,double arrival_start, double arrival_end,- double *factor, double *tau, double *distance) {+ double *factor, double *tau, double *distance, uint16_t *state) {if (n == 0 || m == 0 || m > 4096) return 1;+ uint16_t initial_interval = (uint16_t)m;for (size_t i = 0; i < n; ++i) {double position[3], delta[3];if (!isfinite(t[i])) return 1;- if (evaluate(p, x, m, 3, origin, clip(t[i], start, end), position)) return 1;+ if (evaluate(p, x, m, 3, origin, clip(t[i], start, end), position,+ &initial_interval)) return 1;for (size_t j = 0; j < 3; ++j) delta[j] = position[j] - observer[j];double initial_distance = norm3(delta);if (!isfinite(initial_distance)) return 1;@@ -80,13 +96,22 @@ static int dg_acoustic(const double *t, size_t n, const double *observer,double estimate = t[i] - initial_delay;if (!isfinite(initial_delay) || !isfinite(estimate)) return 1;tau[i] = clip(estimate, start, end);+ if (state != NULL) state[i] = (uint16_t)packed_m;}int unchanged = 0;+ double fixed_maximum = 0.0;for (size_t iteration = 0; iteration < 10; ++iteration) {unchanged = 1;- double maximum = 0.0;+ double maximum = fixed_maximum;for (size_t i = 0; i < n; ++i) {- if (kinematics(packed, packed_x, packed_m, origin, tau[i], observer, distance+i, factor+i)) return 1;+ /* UINT16_MAX marks an exact floating-point fixed point. Repeating+ * its polynomial evaluation and update cannot change its values.+ * Its correction still participates in the global stopping rule:+ * a clamped unsupported endpoint can have a large correction.+ */+ if (state != NULL && state[i] == UINT16_MAX) continue;+ if (kinematics(packed, packed_x, packed_m, origin, tau[i], observer,+ distance+i, factor+i, state == NULL ? NULL : state+i)) return 1;double derivative = 1.0 + factor[i]/c;if (!isfinite(derivative) || !(derivative > 0.0)) return 1;double correction = ((tau[i] - t[i]) + distance[i]/c)/derivative;@@ -95,6 +120,12 @@ static int dg_acoustic(const double *t, size_t n, const double *observer,if (!isfinite(estimate)) return 1;double updated = clip(estimate, start, end);if (updated != tau[i]) unchanged = 0;+ else if (state != NULL && (updated != 0.0 || signbit(updated) == signbit(tau[i]))) {+ /* A signed-zero change compares equal but need not reproduce+ * the same polynomial arithmetic on a subsequent iteration. */+ state[i] = UINT16_MAX;+ if (fabs(correction) > fixed_maximum) fixed_maximum = fabs(correction);+ }tau[i] = updated;double absolute = fabs(correction);if (absolute > maximum) maximum = absolute;@@ -103,7 +134,9 @@ static int dg_acoustic(const double *t, size_t n, const double *observer,}if (!unchanged) {for (size_t i = 0; i < n; ++i) {- if (kinematics(packed, packed_x, packed_m, origin, tau[i], observer, distance+i, factor+i)) return 1;+ if (state != NULL && state[i] == UINT16_MAX) continue;+ if (kinematics(packed, packed_x, packed_m, origin, tau[i], observer,+ distance+i, factor+i, state == NULL ? NULL : state+i)) return 1;}}for (size_t i = 0; i < n; ++i) {@@ -163,6 +196,8 @@ static int increasing_knots(const Py_buffer *view) {static PyObject *solve(PyObject *self, PyObject *args) {PyObject *objects[9];Py_buffer views[9] = {{0}};+ uint16_t small_state[64];+ uint16_t *state = small_state;double origin, start, end, c, arrival_start, arrival_end;int success = 0;if (!PyArg_ParseTuple(args,"OOOOOOddddddOOO",@@ -182,17 +217,23 @@ static PyObject *solve(PyObject *self, PyObject *args) {if (!same_shape(views,views+i)) goto done;const double *observer = views[1].buf;for (int j = 0; j < 3; ++j) if (!isfinite(observer[j])) goto done;+ size_t samples = (size_t)(views[0].len/sizeof(double));+ if (samples > sizeof(small_state)/sizeof(small_state[0])) {+ state = samples <= DG_MAX_STATE_BYTES/sizeof(*state)+ ? PyMem_Malloc(samples*sizeof(*state)) : NULL;+ }/* A failed probe must not leak arithmetic flags into the NumPy fallback. */fenv_t previous_environment;if (fegetenv(&previous_environment) != 0) goto done;Py_BEGIN_ALLOW_THREADS- success = !dg_acoustic(views[0].buf,(size_t)(views[0].len/sizeof(double)),+ success = !dg_acoustic(views[0].buf,samples,observer,views[2].buf,views[3].buf,(size_t)views[2].shape[1],views[4].buf,views[5].buf,(size_t)views[4].shape[1],origin,start,end,c,- arrival_start,arrival_end,views[6].buf,views[7].buf,views[8].buf);+ arrival_start,arrival_end,views[6].buf,views[7].buf,views[8].buf,state);if (fesetenv(&previous_environment) != 0) success = 0;Py_END_ALLOW_THREADSdone:+ if (state != small_state) PyMem_Free(state);for (int i = 0; i < 9; ++i) if (views[i].obj) PyBuffer_Release(views+i);if (PyErr_Occurred()) PyErr_Clear();if (success) ((module_state *)PyModule_GetState(self))->successful_calls++;diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.pyindex a018a6b..7b48954 100644--- a/src/dopplergeist/data_access.py+++ b/src/dopplergeist/data_access.py@@ -18,6 +18,7 @@ import osfrom pathlib import Pathimport pickleimport sqlite3+import sysimport tarfilefrom threading import Lock@@ -138,6 +139,30 @@ def _payload_frame(records):return _IndexColumns(columns, len(records))+_CANONICAL_FRAME_TRACK = _frame_track+_CANONICAL_PAYLOAD_FRAME = _payload_frame+_CANONICAL_SEGMENT_VELOCITY = _segment_velocity+_CANONICAL_TRACK = Track+_PREPARED_ERROR_POLICY = {"divide": "warn", "over": "warn", "under": "ignore", "invalid": "warn"}+++@lru_cache(maxsize=1)+def _prepared_track_identity():+ """Use prepared velocities only with identical normalization and NumPy."""+ from . import flight_payload+ try:+ import numpy._core._multiarray_umath as numpy_core+ except ImportError:+ import numpy.core._multiarray_umath as numpy_core+ files = (Path(__file__), Path(__file__).with_name("tracks.py"),+ Path(__file__).with_name("geometry.py"), Path(flight_payload.__file__),+ Path(numpy_core.__file__))+ return content_hash({"sources": [sha256(path) for path in files],+ "numpy": np.__version__,+ "pandas": pd.__version__, "python": sys.version,+ "cpu": getattr(numpy_core, "__cpu_features__", {})})++@lru_cache(maxsize=1)def _ecef_to_geographic():from pyproj import Transformer@@ -494,15 +519,34 @@ class FlightIndex:try:with self.connect() as db:self.report = json.loads(db.execute("SELECT value FROM metadata").fetchone()[0])- if self.report["index_version"] not in (1, INDEX_VERSION):- raise DataUnavailable("Flight index version mismatch")+ if self.report["index_version"] not in (1, INDEX_VERSION):+ raise DataUnavailable("Flight index version mismatch")+ self._prepared_tracks = self._prepared_available(db, self.report)except (sqlite3.DatabaseError, TypeError, KeyError, json.JSONDecodeError) as error:raise DataUnavailable(f"Invalid flight index: {path}") from errordef connect(self):return sqlite3.connect(f"{self.path.resolve().as_uri()}?mode=ro", uri=True)- def _decode_track(self, name, payload, expected, *, max_samples=None, cache_scan=None):+ @staticmethod+ def _prepared_available(db, report=None):+ # Recheck optional metadata and schema in the actual read snapshot.+ # The unique integer key must prevent duplicate original segments.+ try:+ if report is None:+ report = json.loads(db.execute("SELECT value FROM metadata").fetchone()[0])+ prepared = report.get("prepared_tracks") if isinstance(report, dict) else None+ if (not isinstance(prepared, dict) or prepared.get("version") != 1+ or prepared.get("builder") != _prepared_track_identity()):+ return False+ columns = [(row[1], row[2].upper(), row[5]) for row in+ db.execute("PRAGMA table_info(prepared_tracks)")]+ return columns == [("id", "INTEGER", 1), ("source_hash", "TEXT", 0),+ ("payload", "BLOB", 0), ("payload_hash", "TEXT", 0)]+ except (sqlite3.DatabaseError, TypeError, KeyError, json.JSONDecodeError):+ return False++ def _decode_track(self, name, payload, expected, *, max_samples=None, cache_scan=None, prepared=None):# Verify every fetched payload, including hits. A cached track must not# conceal damage to an index that was changed after the previous query.if hashlib.sha256(payload).hexdigest() != expected:@@ -519,10 +563,26 @@ class FlightIndex:cache_scan["used"].add(key)if entry is not None:return pickle.loads(cached)- frame = _payload_frame(json.loads(gzip.decompress(payload)))- if max_samples is not None and len(frame) > max_samples:- raise DataUnavailable("Flight lookup sample budget exceeded")- track = _frame_track(name, frame)+ track = None+ if (prepared is not None and _frame_track is _CANONICAL_FRAME_TRACK+ and _segment_velocity is _CANONICAL_SEGMENT_VELOCITY+ and _payload_frame is _CANONICAL_PAYLOAD_FRAME+ and Track is _CANONICAL_TRACK and np.geterr() == _PREPARED_ERROR_POLICY):+ source_hash, packed, packed_hash = prepared+ if source_hash == expected and type(packed) is bytes:+ from .flight_payload import decode_track+ # A damaged optional representation cannot replace the verified+ # original observations. Recompute through the legacy reader.+ if hashlib.sha256(packed).hexdigest() == packed_hash:+ try:+ track = decode_track(name, packed, max_samples=max_samples)+ except DataUnavailable:+ track = None+ if track is None:+ frame = _payload_frame(json.loads(gzip.decompress(payload)))+ if max_samples is not None and len(frame) > max_samples:+ raise DataUnavailable("Flight lookup sample budget exceeded")+ track = _frame_track(name, frame)if self._track_cache_limit and (cache_scan is None or cache_scan["admit"]):# These bytes are produced here, never read from a file/request.# Restoring them returns independent arrays and mutable diagnostics,@@ -551,12 +611,78 @@ class FlightIndex:return track_, (_, previous) = self._track_cache.popitem(last=False)self._track_cache_bytes -= len(previous)- self._track_cache[key] = (len(frame), cached)+ self._track_cache[key] = (len(track.time), cached)self._track_cache_bytes += len(cached)if cache_scan is not None:cache_scan["used"].add(key)return track+ @staticmethod+ def _prepare_tracks(db):+ """Store constructor inputs beside the original observations, once."""+ from .flight_payload import encode_track+ db.execute("DROP TABLE IF EXISTS prepared_tracks")+ db.execute("CREATE TABLE prepared_tracks(id INTEGER PRIMARY KEY, source_hash TEXT, payload BLOB, payload_hash TEXT)")+ count = size = 0+ if (_frame_track is not _CANONICAL_FRAME_TRACK+ or _segment_velocity is not _CANONICAL_SEGMENT_VELOCITY+ or _payload_frame is not _CANONICAL_PAYLOAD_FRAME+ or Track is not _CANONICAL_TRACK or np.geterr() != _PREPARED_ERROR_POLICY):+ return {"version": 1, "builder": None, "tracks": count, "bytes": size}+ for identifier, name, payload, expected in db.execute(+ "SELECT id,name,payload,payload_hash FROM segments ORDER BY id"):+ if hashlib.sha256(payload).hexdigest() != expected:+ raise DataUnavailable(f"Corrupt flight payload: {name}")+ # Use the precise values the old reader gets AFTER the index's+ # JSON serialization and dtype inference, not the staging frame.+ track = _frame_track(name, _payload_frame(json.loads(gzip.decompress(payload))))+ try:+ packed = encode_track(track)+ except (DataUnavailable, TypeError, ValueError):+ continue+ db.execute("INSERT INTO prepared_tracks VALUES(?,?,?,?)",+ (identifier, expected, packed, hashlib.sha256(packed).hexdigest()))+ count += 1+ size += len(packed)+ return {"version": 1, "builder": _prepared_track_identity(), "tracks": count, "bytes": size}++ def prepare(self, destination):+ """Create a new index copy with faster verified track decoding.++ The source stays read-only and the destination must not exist. Coverage,+ original payloads, spatial bounds and scientific input identity stay+ unchanged. Old readers ignore the optional numeric representation.+ """+ from uuid import uuid4+ destination = Path(destination)+ destination.parent.mkdir(parents=True, exist_ok=True)+ with cache_lock(destination.with_suffix(".lock")):+ if destination.exists() or destination.is_symlink():+ raise FileExistsError(f"Prepared index destination already exists: {destination}")+ temporary = destination.with_name(destination.name + ".preparing-" + uuid4().hex)+ db = sqlite3.connect(temporary)+ try:+ with self.connect() as source:+ source.backup(db)+ # Copy metadata from the same snapshot as the observations.+ # The caller's older report object is mutable and may be stale.+ try:+ report = json.loads(db.execute("SELECT value FROM metadata").fetchone()[0])+ if report["index_version"] not in (1, INDEX_VERSION):+ raise DataUnavailable("Flight index version mismatch")+ except (sqlite3.DatabaseError, TypeError, KeyError, json.JSONDecodeError) as error:+ raise DataUnavailable("Invalid source flight index metadata") from error+ report["prepared_tracks"] = self._prepare_tracks(db)+ db.execute("UPDATE metadata SET value=?", (json.dumps(report),))+ db.commit()+ db.close()+ temporary.replace(destination)+ except BaseException:+ db.close()+ temporary.unlink(missing_ok=True)+ raise+ return type(self)(destination, track_cache_bytes=self._track_cache_limit)+@classmethoddef build(cls, path, frames, *, sources, coverage, max_gap_s=30.0, max_flight_samples=250000, class_map=None,prepared=False):@@ -624,6 +750,7 @@ class FlightIndex:report = dict(identity, input_hash=content_hash(identity), rows=rows, invalid_rows=invalid,segments=segments, altitude_basis_counts=dict(altitude), jets_excluded=False,unknown_types_retained=True, geometry_clipped=False, continuous_geometry_bounds=True)+ report["prepared_tracks"] = cls._prepare_tracks(db)db.execute("INSERT INTO metadata VALUES(?)", (json.dumps(report),))db.commit()db.execute("VACUUM")@@ -698,10 +825,17 @@ class FlightIndex:join, predicate, params = self._query_scope(bounds, time_range)cache_scan = {"used": set(), "admit": True}with self.connect() as db:- sql = ("SELECT s.name,s.payload,s.payload_hash FROM segments s" + join+ prepared = self._prepared_tracks+ if prepared:+ db.execute("BEGIN")+ prepared = self._prepared_available(db)+ fields = ",p.source_hash,p.payload,p.payload_hash" if prepared else ""+ prepared_join = " LEFT JOIN prepared_tracks p ON p.id=s.id" if prepared else ""+ sql = ("SELECT s.name,s.payload,s.payload_hash" + fields + " FROM segments s" + join + prepared_join+ " WHERE " + predicate + " ORDER BY s.name")- for name, payload, expected in db.execute(sql, params):- yield self._decode_track(name, payload, expected, cache_scan=cache_scan)+ for row in db.execute(sql, params):+ yield self._decode_track(*row[:3], cache_scan=cache_scan,+ prepared=row[3:] if prepared else None)def lookup(self, identifiers, *, max_tracks=2000, max_samples=500000,bounds=None, time_range=None):@@ -730,16 +864,24 @@ class FlightIndex:join, predicate, scope_params = self._query_scope(bounds, time_range)cache_scan = {"used": set(), "admit": True}with self.connect() as db:+ prepared = self._prepared_tracks+ if prepared:+ # Retain one snapshot across the checks and all lookup batches.+ db.execute("BEGIN")+ prepared = self._prepared_available(db)# Stay below SQLite's conservative bound on bound SQL parameters.for first in range(0, len(names), 500):batch = names[first:first+500]placeholders = ",".join("?" for _ in batch)- sql = ("SELECT s.name,s.payload,s.payload_hash FROM segments s" + join+ fields = ",p.source_hash,p.payload,p.payload_hash" if prepared else ""+ prepared_join = " LEFT JOIN prepared_tracks p ON p.id=s.id" if prepared else ""+ sql = ("SELECT s.name,s.payload,s.payload_hash" + fields + " FROM segments s" + join + prepared_join+ " WHERE s.name IN (" + placeholders + ")"+ (" AND " + predicate if predicate else "") + " ORDER BY s.name")- for name, payload, expected in db.execute(sql, (*batch, *scope_params)):- track = self._decode_track(name, payload, expected, max_samples=max_samples - samples,- cache_scan=cache_scan)+ for row in db.execute(sql, (*batch, *scope_params)):+ track = self._decode_track(*row[:3], max_samples=max_samples - samples,+ cache_scan=cache_scan,+ prepared=row[3:] if prepared else None)samples += len(track.time)yield trackdiff --git a/src/dopplergeist/flight_payload.py b/src/dopplergeist/flight_payload.pynew file mode 100644index 0000000..233b796--- /dev/null+++ b/src/dopplergeist/flight_payload.py@@ -0,0 +1,197 @@+"""Bounded numeric snapshots of canonical flight-decoder inputs.++Encode a Track reconstructed from the stored JSON payload. The original payload+and its integrity/implementation checks remain the caller's responsibility. The+snapshot stores float64 inputs and labels; Track reconstructs its spline and+private caches on every decode. No executable object format is read from disk.+"""+import json+import struct++import numpy as np++from .geometry import Track+++FORMAT_VERSION = 1+_MAGIC = b"DGFLT64\0"+_PREFIX = struct.Struct("<8sIIQ")+_MAX_SAMPLES = 1_000_000+_MAX_HEADER_BYTES = 1024 * 1024+_MAX_PAYLOAD_BYTES = _PREFIX.size + _MAX_HEADER_BYTES + 7 * 8 * _MAX_SAMPLES+_MAX_LABEL_LENGTH = 1024+_COMPONENTS = ("east_mps", "north_mps", "up_mps")+_FIELDS = {"id", "typecode", "aircraft_class", "altitude_basis",+ "velocity_inferred_components", "velocity"}+++def _invalid(reason):+ # data_access may import this codec while defining FlightIndex.+ from .data_access import DataUnavailable+ return DataUnavailable(f"Invalid prepared flight payload: {reason}")+++def _label(value):+ return type(value) is str and len(value) <= _MAX_LABEL_LENGTH+++def _metadata(track, count):+ if not all(_label(value) for value in (track.id, track.typecode,+ getattr(track, "aircraft_class", None))):+ raise _invalid("noncanonical flight labels")+ basis = getattr(track, "altitude_basis", None)+ if type(basis) is not list or len(basis) != count:+ raise _invalid("altitude labels do not match the trajectory")+ runs = []+ # JSON escapes use at most six bytes per character. Bound metadata while+ # forming runs, before serializing a potentially large alternating list.+ header_bound = 512 + 6 * sum(len(value) for value in+ (track.id, track.typecode, track.aircraft_class))+ for label in basis:+ if not _label(label):+ raise _invalid("noncanonical altitude label")+ if runs and runs[-1][0] == label:+ runs[-1][1] += 1+ else:+ header_bound += 6 * len(label) + 32+ if header_bound > _MAX_HEADER_BYTES:+ raise _invalid("metadata exceeds the codec limit")+ runs.append([label, 1])+ inferred = getattr(track, "velocity_inferred_components", None)+ if (type(inferred) is not dict or set(inferred) != set(_COMPONENTS)+ or any(type(inferred[key]) is not int or not 0 <= inferred[key] <= count+ for key in _COMPONENTS)):+ raise _invalid("noncanonical velocity-inference counts")+ return {"id": track.id, "typecode": track.typecode,+ "aircraft_class": track.aircraft_class, "altitude_basis": runs,+ "velocity_inferred_components": {key: inferred[key] for key in _COMPONENTS},+ "velocity": track.velocity is not None}+++def encode_track(track):+ """Encode exact inputs from a freshly decoded canonical Track.++ Noncanonical metadata or oversized trajectories are left to the original+ payload path. This function does not infer velocities or round any values.+ """+ if type(track) is not Track:+ raise _invalid("noncanonical Track type")+ if type(track.time) is not np.ndarray or track.time.ndim != 1:+ raise _invalid("noncanonical trajectory array")+ count = len(track.time)+ if not 3 <= count <= _MAX_SAMPLES:+ raise _invalid("trajectory sample count is outside the codec limit")+ arrays = [track.time, track.lat, track.lon, track.altitude]+ for values in arrays:+ if (type(values) is not np.ndarray or values.dtype != np.dtype(float)+ or values.shape != (count,) or not np.all(np.isfinite(values))):+ raise _invalid("noncanonical trajectory array")+ if not np.all(np.diff(track.time) > 0):+ raise _invalid("trajectory times are not strictly increasing")+ if track.velocity is not None:+ velocity = track.velocity+ if (type(velocity) is not np.ndarray or velocity.dtype != np.dtype(float)+ or velocity.shape != (count, 3) or not np.all(np.isfinite(velocity))):+ raise _invalid("noncanonical velocity array")+ arrays.append(velocity)+ try:+ header = json.dumps(_metadata(track, count), ensure_ascii=False,+ separators=(",", ":"), allow_nan=False).encode("utf-8")+ except (ValueError, TypeError, UnicodeError) as error:+ raise _invalid("noncanonical metadata encoding") from error+ if len(header) > _MAX_HEADER_BYTES:+ raise _invalid("metadata exceeds the codec limit")+ return b"".join([_PREFIX.pack(_MAGIC, FORMAT_VERSION, len(header), count), header,+ *(values.astype("<f8", copy=False).tobytes(order="C") for values in arrays)])+++def _object(pairs):+ result = {}+ for key, value in pairs:+ if key in result:+ raise ValueError("duplicate metadata key")+ result[key] = value+ return result+++def _constant(value):+ raise ValueError("nonfinite JSON metadata")+++def _decode_metadata(header, count, name):+ try:+ metadata = json.loads(header.decode("utf-8"), object_pairs_hook=_object,+ parse_constant=_constant)+ except (ValueError, UnicodeDecodeError, RecursionError) as error:+ raise _invalid("malformed JSON metadata") from error+ if type(metadata) is not dict or set(metadata) != _FIELDS:+ raise _invalid("unexpected metadata fields")+ if (not all(_label(metadata[key]) for key in ("id", "typecode", "aircraft_class"))+ or metadata["id"] != name or type(metadata["velocity"]) is not bool):+ raise _invalid("flight identity or labels do not match")+ inferred = metadata["velocity_inferred_components"]+ if (type(inferred) is not dict or set(inferred) != set(_COMPONENTS)+ or any(type(inferred[key]) is not int or not 0 <= inferred[key] <= count+ for key in _COMPONENTS)):+ raise _invalid("invalid velocity-inference counts")+ runs = metadata["altitude_basis"]+ if type(runs) is not list or not 1 <= len(runs) <= count:+ raise _invalid("invalid altitude label runs")+ length = 0+ for run in runs:+ if (type(run) is not list or len(run) != 2 or not _label(run[0])+ or type(run[1]) is not int or not 1 <= run[1] <= count - length):+ raise _invalid("invalid altitude label run")+ length += run[1]+ if length != count:+ raise _invalid("altitude labels do not match the trajectory")+ return metadata+++def decode_track(name, payload, *, max_samples=None):+ """Return independent writable inputs and a reconstructed Track.++ Validate counts and byte lengths before allocating arrays or expanding label+ runs. A malformed or unsupported snapshot raises DataUnavailable so callers+ can use their original JSON payload under the same integrity checks.+ """+ if type(payload) is not bytes or not _PREFIX.size <= len(payload) <= _MAX_PAYLOAD_BYTES:+ raise _invalid("invalid payload size or buffer type")+ if not _label(name):+ raise _invalid("noncanonical flight identifier")+ if max_samples is not None and (type(max_samples) is not int or max_samples < 0):+ raise _invalid("invalid sample budget")+ magic, version, header_size, count = _PREFIX.unpack_from(payload)+ if magic != _MAGIC or version != FORMAT_VERSION:+ raise _invalid("unsupported magic or format version")+ if not 3 <= count <= _MAX_SAMPLES or not 0 < header_size <= _MAX_HEADER_BYTES:+ raise _invalid("sample or metadata count exceeds the codec limit")+ if max_samples is not None and count > max_samples:+ from .data_access import DataUnavailable+ raise DataUnavailable("Flight lookup sample budget exceeded")+ offset = _PREFIX.size + header_size+ if offset > len(payload):+ raise _invalid("truncated metadata")+ metadata = _decode_metadata(payload[_PREFIX.size:offset], count, name)+ columns = 7 if metadata["velocity"] else 4+ if len(payload) != offset + columns * count * 8:+ raise _invalid("numeric payload length does not match the header")+ arrays = []+ for _ in range(4):+ arrays.append(np.frombuffer(payload, dtype="<f8", count=count, offset=offset)+ .astype(float, copy=True))+ offset += count * 8+ velocity = None+ if metadata["velocity"]:+ velocity = np.frombuffer(payload, dtype="<f8", count=3 * count, offset=offset)+ velocity = velocity.reshape(count, 3).astype(float, copy=True)+ try:+ track = Track(name, *arrays, metadata["typecode"], velocity)+ except (ValueError, TypeError, OverflowError, FloatingPointError) as error:+ raise _invalid("invalid trajectory values") from error+ track.altitude_basis = [label for label, length in metadata["altitude_basis"]+ for _ in range(length)]+ track.aircraft_class = metadata["aircraft_class"]+ track.velocity_inferred_components = {+ key: metadata["velocity_inferred_components"][key] for key in _COMPONENTS}+ return trackdiff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.pyindex 54f689f..2170b72 100644--- a/src/dopplergeist/search.py+++ b/src/dopplergeist/search.py@@ -586,7 +586,7 @@ def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,outcomes = [None] * len(starts)active = list(range(len(starts)))rejected_count = floor_count = 0- prefixes = [0.0] * len(starts)+ prefixes = np.zeros(len(starts), dtype=np.float64)limits = [3 * _frequency_scale(event) for event in events] if score_floor is not None else []bound_enabled = bool(limits) and all(np.isfinite(value) and value > 0 for value in limits)batches = iter(acoustic_batches)@@ -597,16 +597,17 @@ def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,if use_dual and batch is not None:rejected, upper = ridge_trial_rejections(measurement, event, *batch,score_floor=score_floor, group_count=len(events))- eligible = []- for trial in active:- if rejected[trial]:- rejected_count += 1- outcomes[trial] = (None, -np.inf, None)- elif np.isfinite(upper[trial]):- floor_count += 1- outcomes[trial] = (None, float(upper[trial]), None)- else:- eligible.append(trial)+ indices = np.asarray(active, dtype=np.intp)+ rejected = rejected[indices]+ upper = upper[indices]+ finite_upper = ~rejected & np.isfinite(upper)+ rejected_count += int(np.count_nonzero(rejected))+ floor_count += int(np.count_nonzero(finite_upper))+ for trial in indices[rejected].tolist():+ outcomes[trial] = (None, -np.inf, None)+ for trial, bound in zip(indices[finite_upper].tolist(), upper[finite_upper].tolist()):+ outcomes[trial] = (None, float(bound), None)+ eligible = indices[~(rejected | finite_upper)].tolist()else:eligible = activefitted = fitted_ceilings = None@@ -619,6 +620,38 @@ def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,*arguments, limits[index], legacy_arithmetic=legacy)else:fitted = first_event_fits(*arguments, legacy_arithmetic=legacy)+ if fitted_ceilings is not None:+ # Only the unchanged core validator supplies this complete vector.+ # Custom hooks and numerical policies keep the scalar path below.+ for trial, fit in zip(eligible, fitted):+ diagnostics[trial].append(fit)+ indices = np.asarray(eligible, dtype=np.intp)+ contributions = np.asarray(fitted_ceilings, dtype=np.float64)+ rejected = contributions == -np.inf+ for trial in indices[rejected].tolist():+ outcomes[trial] = (None, -np.inf, None)+ indices = indices[~rejected]+ prefixes[indices] = np.nextafter(prefixes[indices] + contributions[~rejected], np.inf)+ bounds = prefixes[indices]+ # From a nonnegative prefix, each remaining +1/upward-rounding+ # step leaves a bound at least as large as its step count. The+ # final cap is also >= remaining, so none can fall below this+ # floor. Check prefixes explicitly to preserve earlier custom fits.+ if score_floor <= remaining and np.all(bounds >= 0.0):+ active = indices.tolist()+ continue+ # Keep each addition and upward rounding in the scalar order. In+ # particular, summing losses first or rounding only once is unsafe.+ for _ in range(remaining):+ bounds = np.nextafter(bounds + 1.0, np.inf)+ # Python min(maximum, NaN) retains maximum, including a NaN prefix+ # supplied by an earlier custom fitter before core validation resumes.+ np.fmin(float(len(events)), bounds, out=bounds)+ below_floor = bounds < score_floor+ for trial, bound in zip(indices[below_floor].tolist(), bounds[below_floor].tolist()):+ outcomes[trial] = (None, bound, None)+ active = indices[~below_floor].tolist()+ continuenext_active = []for fit_index, trial in enumerate(eligible):try:@@ -644,8 +677,8 @@ def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,# Reuse the already validated prefix. Add unperformed# groups one at a time with the identical outward rounding;# capping a prefix or summing losses first changes the bound.- prefixes[trial] = math.nextafter(prefixes[trial] + contribution, math.inf)- bound = prefixes[trial]+ prefixes[trial] = math.nextafter(float(prefixes[trial]) + contribution, math.inf)+ bound = float(prefixes[trial])for _ in range(remaining):bound = math.nextafter(bound + 1.0, math.inf)bound = min(float(len(events)), bound)diff --git a/tests/test_real_flight_payload.py b/tests/test_real_flight_payload.pynew file mode 100644index 0000000..6b24e3f--- /dev/null+++ b/tests/test_real_flight_payload.py@@ -0,0 +1,126 @@+"""Prepared payload parity and corruption checks on archived observations."""+import json+from pathlib import Path++import numpy as np+import pandas as pd+import pytest++from dopplergeist import flight_payload+from dopplergeist.data_access import DataUnavailable, _frame_track++[email protected](scope="module")+def observed_tracks():+ root = Path(__file__).resolve().parents[1]+ tracks = []+ for source in sorted((root / "research/trajectory-motion/frozen").glob("fresh-*.json")):+ frame = pd.DataFrame(json.loads(source.read_text())).sort_values("time_s")+ for number, (_, rows) in enumerate(frame.groupby((frame.time_s.diff() > 30.).cumsum())):+ if len(rows) >= 3:+ tracks.append(_frame_track(f"{rows.flight_id.iloc[0]}:{number}", rows))+ assert tracks+ return tracks+++def assert_exact_track(actual, expected):+ assert actual.id == expected.id and actual.typecode == expected.typecode+ for name in ("time", "lat", "lon", "altitude", "velocity", "xyz", "relative_time",+ "_xyz_min", "_xyz_max", "_speed_upper", "_speed_peak"):+ left, right = getattr(actual, name), getattr(expected, name)+ if right is None:+ assert left is None+ else:+ assert left.dtype == right.dtype and left.shape == right.shape, name+ assert left.tobytes() == right.tobytes(), name+ for name in ("spline", "derivative"):+ left, right = getattr(actual, name), getattr(expected, name)+ assert left.axis == right.axis and left.extrapolate == right.extrapolate+ for field in ("x", "c"):+ assert getattr(left, field).tobytes() == getattr(right, field).tobytes()+ for name in ("origin", "_max_sample_gap", "altitude_basis", "aircraft_class",+ "velocity_inferred_components"):+ assert getattr(actual, name) == getattr(expected, name), name+++def test_observed_tracks_round_trip_exactly(observed_tracks):+ for expected in observed_tracks:+ payload = flight_payload.encode_track(expected)+ actual = flight_payload.decode_track(expected.id, payload)+ assert_exact_track(actual, expected)+ for name in ("time", "lat", "lon", "altitude", "velocity"):+ array = getattr(actual, name)+ if array is not None:+ assert array.flags.writeable and array.flags.c_contiguous+++def test_missing_velocity_uses_same_recorded_positions(observed_tracks):+ original = observed_tracks[0]+ frame = pd.DataFrame({"time_s": original.time, "lat": original.lat,+ "lon": original.lon, "alt_m": original.altitude})+ expected = _frame_track(original.id, frame)+ assert expected.velocity is None+ actual = flight_payload.decode_track(expected.id, flight_payload.encode_track(expected))+ assert_exact_track(actual, expected)+++def test_decoded_tracks_are_independently_mutable(observed_tracks):+ expected = observed_tracks[0]+ payload = flight_payload.encode_track(expected)+ changed = flight_payload.decode_track(expected.id, payload)+ untouched = flight_payload.decode_track(expected.id, payload)+ changed.time[0] = np.nextafter(changed.time[0], np.inf)+ changed.altitude[0] += 1.+ if changed.velocity is not None:+ changed.velocity[0, 0] += 1.+ changed.altitude_basis[0] = "changed"+ changed.velocity_inferred_components["east_mps"] += 1+ assert_exact_track(untouched, expected)+ assert_exact_track(flight_payload.decode_track(expected.id, payload), expected)+++def _header_variant(payload, mutate):+ magic, version, size, count = flight_payload._PREFIX.unpack_from(payload)+ first = flight_payload._PREFIX.size+ metadata = json.loads(payload[first:first+size])+ mutate(metadata)+ header = json.dumps(metadata, separators=(",", ":")).encode()+ return (flight_payload._PREFIX.pack(magic, version, len(header), count)+ + header + payload[first+size:])+++def test_invalid_sizes_types_and_metadata_fail(observed_tracks):+ track = observed_tracks[0]+ payload = flight_payload.encode_track(track)+ magic, version, size, count = flight_payload._PREFIX.unpack_from(payload)+ invalid = [b"", payload[:-1], payload+b"x", b"x"+payload[1:], bytearray(payload),+ flight_payload._PREFIX.pack(magic, version+1, size, count)+payload[flight_payload._PREFIX.size:],+ flight_payload._PREFIX.pack(magic, version, flight_payload._MAX_HEADER_BYTES+1, count),+ flight_payload._PREFIX.pack(magic, version, size, flight_payload._MAX_SAMPLES+1)]+ for change in (+ lambda meta: meta.update(extra=True),+ lambda meta: meta.update(velocity=1),+ lambda meta: meta.update(aircraft_class=None),+ lambda meta: meta["velocity_inferred_components"].update(east_mps=True),+ lambda meta: meta.update(altitude_basis=[["ellipsoidal", count+1]]),+ lambda meta: meta.update(altitude_basis=[["ellipsoidal", -1]]),+ ):+ invalid.append(_header_variant(payload, change))+ for damaged in invalid:+ with pytest.raises(DataUnavailable):+ flight_payload.decode_track(track.id, damaged)+ with pytest.raises(DataUnavailable):+ flight_payload.decode_track(track.id+"-other", payload)+ with pytest.raises(DataUnavailable, match="sample budget exceeded"):+ flight_payload.decode_track(track.id, payload, max_samples=count-1)+ assert_exact_track(flight_payload.decode_track(track.id, payload, max_samples=count), track)+++def test_nonfinite_numeric_payload_is_rejected(observed_tracks):+ track = observed_tracks[0]+ payload = flight_payload.encode_track(track)+ _, _, size, _ = flight_payload._PREFIX.unpack_from(payload)+ first = flight_payload._PREFIX.size+size+ damaged = payload[:first] + np.array([np.nan], dtype="<f8").tobytes() + payload[first+8:]+ with pytest.raises(DataUnavailable, match="invalid trajectory values"):+ flight_payload.decode_track(track.id, damaged)diff --git a/tests/test_real_native_convergence.py b/tests/test_real_native_convergence.pynew file mode 100644index 0000000..bf48420--- /dev/null+++ b/tests/test_real_native_convergence.py@@ -0,0 +1,67 @@+"""Native convergence state stays exact across bounded allocation paths."""+import json+from pathlib import Path+from unittest.mock import patch++import numpy as np+import pandas as pd+import pytest++from dopplergeist import geometry, native_acoustic+from dopplergeist.tracks import _segment_velocity+++pytestmark = pytest.mark.skipif(native_acoustic._native is None,+ reason="optional acoustic extension is not built")++[email protected](scope="module")+def observed_query():+ root = Path(__file__).resolve().parents[1]+ for source in sorted((root / "research/trajectory-motion/frozen").glob("fresh-*.json")):+ frame = pd.DataFrame(json.loads(source.read_text())).sort_values("time_s")+ for _, rows in frame.groupby((frame.time_s.diff() > 30.).cumsum()):+ if len(rows) < 3:+ continue+ velocity, _ = _segment_velocity(rows)+ track = geometry.Track(str(rows.flight_id.iloc[0]), rows.time_s.to_numpy(),+ rows.lat.to_numpy(), rows.lon.to_numpy(), rows.alt_m.to_numpy(),+ str(rows.typecode.iloc[0]), velocity)+ try:+ track.validate_motion_support()+ except geometry.AcousticSupportError:+ continue+ observer = geometry.ecef(track.lat[0], track.lon[0], 0.)+ receive = track.time + np.linalg.norm(track.xyz-observer, axis=1)/343.+ return track, observer, receive+ pytest.fail("No supported archived aircraft observations")++[email protected]("count", (1, 63, 64, 65, 257))+def test_observed_requests_preserve_bytes_at_state_allocation_boundary(observed_query, count):+ track, observer, receive = observed_query+ query = np.resize(receive, count)+ with patch.object(native_acoustic, "_native", None):+ expected = track.acoustic(query, observer)+ before = native_acoustic._native.successful_calls()+ actual = track.acoustic(query, observer)+ assert native_acoustic._native.successful_calls() == before + 1+ for left, right in zip(actual, expected):+ assert left.dtype == right.dtype and left.shape == right.shape+ assert left.tobytes() == right.tobytes()+++def test_repeated_observed_requests_above_state_budget_stay_native(observed_query):+ track, observer, receive = observed_query+ with patch.object(native_acoustic, "_native", None):+ expected = track.acoustic(receive, observer)+ # Repeat identical recorded requests to cross the 2 MiB state budget.+ # Their global maximum correction and iteration count are unchanged.+ repetitions = (2 * 1024 * 1024 // np.dtype(np.uint16).itemsize) // len(receive) + 1+ query = np.tile(receive, (repetitions, 1))+ before = native_acoustic._native.successful_calls()+ actual = track.acoustic(query, observer)+ assert native_acoustic._native.successful_calls() == before + 1+ for left, right in zip(actual, expected):+ expected_bits = np.broadcast_to(right.view(np.uint64), query.shape)+ np.testing.assert_array_equal(left.view(np.uint64), expected_bits)diff --git a/tests/test_real_prepared_index.py b/tests/test_real_prepared_index.pynew file mode 100644index 0000000..420fcbb--- /dev/null+++ b/tests/test_real_prepared_index.py@@ -0,0 +1,215 @@+"""Prepared index checks using archived, unmodified ADS-B observations."""+import gzip+import hashlib+import json+from contextlib import contextmanager+from pathlib import Path+import sqlite3++import pandas as pd+import pytest++from dopplergeist import data_access+from dopplergeist.data_access import DataUnavailable, FlightIndex+++def observed_index(tmp_path):+ source = Path(__file__).resolve().parents[1] / "research/trajectory-motion/frozen/canonical-observations.json"+ frame = pd.DataFrame(json.loads(source.read_text()))+ coverage = {"bounds": [-180., -90., 180., 90.],+ "time_range_s": [float(frame.time_s.min()), float(frame.time_s.max())],+ "all_aircraft": False, "source_note": "Archived real observations; incomplete excerpt"}+ index = FlightIndex.build(tmp_path / "observed.sqlite", [frame], sources=[], coverage=coverage)+ assert index._prepared_tracks+ assert index.report["prepared_tracks"]["tracks"] > 0+ return index, coverage+++def signature(track):+ arrays = []+ for name in ("time", "lat", "lon", "altitude", "velocity", "xyz", "relative_time",+ "_xyz_min", "_xyz_max", "_speed_upper", "_speed_peak"):+ value = getattr(track, name)+ arrays.append(None if value is None else (value.dtype.str, value.shape, value.tobytes()))+ for name in ("spline", "derivative"):+ value = getattr(track, name)+ arrays.extend((part.dtype.str, part.shape, part.tobytes()) for part in (value.c, value.x))+ return (track.id, arrays, track.typecode, track.aircraft_class, track.altitude_basis,+ track.velocity_inferred_components, track.origin, track._max_sample_gap)+++def queried(index, coverage):+ return list(index.query(coverage["bounds"], coverage["time_range_s"]))+++def test_prepared_query_and_lookup_match_original_bytes(observed_index, monkeypatch):+ index, coverage = observed_index+ legacy = FlightIndex(index.path, track_cache_bytes=0)+ legacy._prepared_tracks = False+ expected = queried(legacy, coverage)+ fast = FlightIndex(index.path, track_cache_bytes=0)+ calls = []+ from dopplergeist import flight_payload+ original = flight_payload.decode_track++ def counted(*args, **kwargs):+ calls.append(args[0])+ return original(*args, **kwargs)++ monkeypatch.setattr(flight_payload, "decode_track", counted)+ actual = queried(fast, coverage)+ assert [signature(track) for track in actual] == [signature(track) for track in expected]+ assert len(calls) == len(expected)+ looked_up = list(fast.lookup([track.id for track in expected]))+ assert [signature(track) for track in looked_up] == [signature(track) for track in expected]+ assert len(calls) == 2 * len(expected)+ actual[0].time[0] += 1+ actual[0].spline.c[0, 0, 0] += 1+ actual[0].altitude_basis[0] = "caller modification"+ assert signature(next(fast.lookup([expected[0].id]))) == signature(expected[0])+++def test_prepared_index_copy_preserves_source_and_input_identity(observed_index, tmp_path):+ index, coverage = observed_index+ before = data_access.sha256(index.path)+ destination = tmp_path / "prepared.sqlite"+ prepared = index.prepare(destination)+ assert data_access.sha256(index.path) == before+ assert prepared.report["input_hash"] == index.report["input_hash"]+ assert prepared.coverage_for(coverage["bounds"], coverage["time_range_s"]) == index.coverage_for(+ coverage["bounds"], coverage["time_range_s"])+ assert [signature(track) for track in queried(prepared, coverage)] == [+ signature(track) for track in queried(index, coverage)]+ with pytest.raises(FileExistsError):+ index.prepare(destination)+++def test_prepare_and_build_coordinate_the_destination(observed_index, tmp_path, monkeypatch):+ index, coverage = observed_index+ original_lock, locks = data_access.cache_lock, []++ @contextmanager+ def recorded_lock(path):+ locks.append(Path(path))+ with original_lock(path):+ yield++ monkeypatch.setattr(data_access, "cache_lock", recorded_lock)+ before = data_access.sha256(index.path)+ destination = tmp_path / "shared-destination.sqlite"+ prepared = index.prepare(destination)+ reused = FlightIndex.build(destination, [], sources=[], coverage=coverage)+ assert len(locks) == 2 and locks[0] == locks[1]+ assert prepared.report["input_hash"] == reused.report["input_hash"] == index.report["input_hash"]+ assert data_access.sha256(index.path) == before+++def test_prepare_uses_metadata_from_the_copied_snapshot(observed_index, tmp_path):+ index, _ = observed_index+ with sqlite3.connect(index.path) as db:+ original = json.loads(db.execute("SELECT value FROM metadata").fetchone()[0])+ original["snapshot_note"] = "Same archived observations; updated report note"+ db.execute("UPDATE metadata SET value=?", (json.dumps(original),))+ before = data_access.sha256(index.path)+ index.report["coverage"]["all_aircraft"] = True+ index.report["input_hash"] = "caller-modified-report"+ prepared = index.prepare(tmp_path / "snapshot.sqlite")+ assert {key: value for key, value in prepared.report.items() if key != "prepared_tracks"} == {+ key: value for key, value in original.items() if key != "prepared_tracks"}+ assert data_access.sha256(index.path) == before+++def test_prepared_reader_falls_back_after_implementation_change(observed_index, monkeypatch):+ index, coverage = observed_index+ expected = [signature(track) for track in queried(index, coverage)]+ monkeypatch.setattr(data_access, "_prepared_track_identity", lambda: "other-normalization-build")+ reader = FlightIndex(index.path, track_cache_bytes=0)+ assert reader._prepared_tracks is False+ assert [signature(track) for track in queried(reader, coverage)] == expected+++def test_reader_rechecks_changed_preparation_metadata(observed_index, monkeypatch):+ index, coverage = observed_index+ expected = queried(index, coverage)+ reader = FlightIndex(index.path, track_cache_bytes=0)+ with sqlite3.connect(index.path) as db:+ report = json.loads(db.execute("SELECT value FROM metadata").fetchone()[0])+ report["prepared_tracks"]["builder"] = "other-normalization-build"+ db.execute("UPDATE metadata SET value=?", (json.dumps(report),))+ from dopplergeist import flight_payload++ def unavailable(*args, **kwargs):+ pytest.fail("Stale preparation metadata reached the numeric decoder")++ monkeypatch.setattr(flight_payload, "decode_track", unavailable)+ assert [signature(track) for track in queried(reader, coverage)] == [+ signature(track) for track in expected]+ assert [signature(track) for track in reader.lookup([track.id for track in expected])] == [+ signature(track) for track in expected]++[email protected]("damaged,repair_digest", [+ (b"invalid prepared representation", False),+ (b"invalid prepared representation", True),+ ("wrong SQLite storage type", False),+ (17, False),+])+def test_damaged_optional_payload_uses_verified_original(observed_index, damaged, repair_digest):+ index, coverage = observed_index+ expected = [signature(track) for track in queried(index, coverage)]+ with sqlite3.connect(index.path) as db:+ if repair_digest:+ db.execute("UPDATE prepared_tracks SET payload=?,payload_hash=?",+ (damaged, hashlib.sha256(damaged).hexdigest()))+ else:+ db.execute("UPDATE prepared_tracks SET payload=?", (damaged,))+ reader = FlightIndex(index.path, track_cache_bytes=0)+ assert [signature(track) for track in queried(reader, coverage)] == expected++[email protected]("replacement", [+ None,+ "CREATE TABLE prepared_tracks(id INTEGER PRIMARY KEY, payload BLOB)",+ "CREATE TABLE prepared_tracks(id INTEGER, source_hash TEXT, payload BLOB, payload_hash TEXT)",+])[email protected]("opened_before_damage", [False, True])+def test_missing_or_malformed_optional_table_uses_original(observed_index, replacement, opened_before_damage):+ index, coverage = observed_index+ expected = queried(index, coverage)+ reader = FlightIndex(index.path, track_cache_bytes=0) if opened_before_damage else None+ with sqlite3.connect(index.path) as db:+ db.execute("DROP TABLE prepared_tracks")+ if replacement is not None:+ db.execute(replacement)+ if reader is None:+ reader = FlightIndex(index.path, track_cache_bytes=0)+ assert reader._prepared_tracks is False+ assert [signature(track) for track in queried(reader, coverage)] == [+ signature(track) for track in expected]+ assert [signature(track) for track in reader.lookup([track.id for track in expected])] == [+ signature(track) for track in expected]+++def test_prepared_reader_does_not_hide_changed_original_payload(observed_index):+ index, coverage = observed_index+ first = queried(index, coverage)[0]+ with sqlite3.connect(index.path) as db:+ old = db.execute("SELECT payload FROM segments WHERE name=?", (first.id,)).fetchone()[0]+ changed = gzip.compress(gzip.decompress(old), mtime=1)+ db.execute("UPDATE segments SET payload=?,payload_hash=? WHERE name=?",+ (changed, hashlib.sha256(changed).hexdigest(), first.id))+ reader = FlightIndex(index.path, track_cache_bytes=0)+ assert signature(next(reader.lookup([first.id]))) == signature(first)+ with sqlite3.connect(index.path) as db:+ db.execute("UPDATE segments SET payload=? WHERE name=?", (b"damaged original", first.id))+ with pytest.raises(DataUnavailable, match="Corrupt flight payload"):+ list(reader.lookup([first.id]))+++def test_prepared_lookup_preserves_sample_budget(observed_index):+ index, coverage = observed_index+ first = queried(index, coverage)[0]+ reader = FlightIndex(index.path, track_cache_bytes=0)+ with pytest.raises(DataUnavailable, match="Flight lookup sample budget exceeded"):+ list(reader.lookup([first.id], max_samples=len(first.time) - 1))diff --git a/tests/test_real_track_cache.py b/tests/test_real_track_cache.pyindex 5060349..2743f68 100644--- a/tests/test_real_track_cache.py+++ b/tests/test_real_track_cache.py@@ -20,6 +20,25 @@ def snapshot_size(track):return len(pickle.dumps(track, protocol=pickle.HIGHEST_PROTOCOL))+def record_track_setup(monkeypatch, prepared):+ """Count the selected decoder without switching paths after sizing it."""+ if prepared:+ from dopplergeist import flight_payload+ module, name = flight_payload, "decode_track"+ else:+ module, name = data_access, "_frame_track"+ original, calls = getattr(module, name), []++ def decode(identifier, *args, **kwargs):+ calls.append(identifier)+ return original(identifier, *args, **kwargs)++ # A custom _frame_track hook intentionally selects the original reader.+ # Prepared tracks can have different pickle sizes from shared label strings.+ monkeypatch.setattr(module, name, decode)+ return calls++@pytest.fixturedef real_index(tmp_path):source = Path(__file__).resolve().parents[1] / "research/trajectory-motion/frozen/canonical-observations.json"@@ -118,55 +137,58 @@ def test_real_cache_observes_replaced_payload(real_index, monkeypatch):assert calls == [expected.id]-def test_real_cache_evicts_and_skips_oversized_tracks(real_index, monkeypatch):[email protected]("prepared", (False, True), ids=("original-json", "prepared-numeric"))+def test_real_cache_evicts_and_skips_oversized_tracks(real_index, monkeypatch, prepared):path, coverage = real_index+ calls = record_track_setup(monkeypatch, prepared)tracks = list(FlightIndex(path, track_cache_bytes=0).query(coverage["bounds"], coverage["time_range_s"]))+ assert calls == [track.id for track in tracks]+ calls.clear()first, second = tracks[:2]- sizes = [snapshot_size(track) for track in (first, second)]- cached = FlightIndex(path, track_cache_bytes=max(sizes))- original, calls = data_access._frame_track, []-- def decode(name, frame):- calls.append(name)- return original(name, frame)-- monkeypatch.setattr(data_access, "_frame_track", decode)+ sizes = {track.id: snapshot_size(track) for track in (first, second)}+ cached = FlightIndex(path, track_cache_bytes=max(sizes.values()))for track in (first, second, first):assert_tracks_equal(list(cached.lookup([track.id])), [track])assert len(cached._track_cache) == 1- assert cached._track_cache_bytes <= max(sizes)+ assert next(iter(cached._track_cache))[0] == track.id+ assert cached._track_cache_bytes == sizes[track.id]+ assert cached._track_cache_bytes == sum(len(value[1]) for value in cached._track_cache.values())+ assert cached._track_cache_bytes <= cached._track_cache_limitassert calls == [first.id, second.id, first.id]- too_small = FlightIndex(path, track_cache_bytes=min(sizes) - 1)- assert_tracks_equal(list(too_small.lookup([first.id])), [first])+ too_small = FlightIndex(path, track_cache_bytes=min(sizes.values()) - 1)+ for _ in range(2):+ assert_tracks_equal(list(too_small.lookup([first.id])), [first])+ assert calls == [first.id, second.id, first.id, first.id, first.id]assert not too_small._track_cacheassert too_small._track_cache_bytes == 0-def test_real_query_larger_than_cache_keeps_a_reusable_subset(real_index, monkeypatch):[email protected]("prepared", (False, True), ids=("original-json", "prepared-numeric"))+def test_real_query_larger_than_cache_keeps_a_reusable_subset(real_index, monkeypatch, prepared):path, coverage = real_index+ calls = record_track_setup(monkeypatch, prepared)expected = list(FlightIndex(path, track_cache_bytes=0).query(coverage["bounds"], coverage["time_range_s"]))+ assert calls == [track.id for track in expected]+ calls.clear()size = snapshot_size(expected[0])cached = FlightIndex(path, track_cache_bytes=size)- original, calls = data_access._frame_track, []serialize, snapshots = data_access.pickle.dumps, []- def decode(name, frame):- calls.append(name)- return original(name, frame)-def snapshot(track, **kwargs):snapshots.append(track.id)return serialize(track, **kwargs)- monkeypatch.setattr(data_access, "_frame_track", decode)monkeypatch.setattr(data_access.pickle, "dumps", snapshot)assert_tracks_equal(list(cached.query(coverage["bounds"], coverage["time_range_s"])), expected)assert len(calls) == len(expected)assert len(cached._track_cache) == 1assert next(iter(cached._track_cache))[0] == expected[0].id+ assert cached._track_cache_bytes == size+ assert cached._track_cache_bytes == sum(len(value[1]) for value in cached._track_cache.values())assert_tracks_equal(list(cached.query(coverage["bounds"], coverage["time_range_s"])), expected)assert len(calls) == 2 * len(expected) - 1assert cached._track_cache_bytes == size+ assert cached._track_cache_bytes == sum(len(value[1]) for value in cached._track_cache.values())assert len(snapshots) < len(calls) # Admission failure avoids work on later misses.diff --git a/tests/test_real_trial_vectors.py b/tests/test_real_trial_vectors.pynew file mode 100644index 0000000..d832508--- /dev/null+++ b/tests/test_real_trial_vectors.py@@ -0,0 +1,162 @@+"""Clock-vector screens preserve scalar decisions on recorded associations."""+import json+import os+from pathlib import Path++import numpy as np+import pandas as pd+import pytest++from dopplergeist import acoustic_screen, batch_ridge, matching, search+from dopplergeist.data_access import FlightIndex+from dopplergeist.geometry import AcousticSupportError, ecef++[email protected](scope="module")+def recorded_associations():+ source = os.environ.get("DOPPLERGEIST_REAL_CPA_FIXTURE")+ if not source:+ pytest.skip("set DOPPLERGEIST_REAL_CPA_FIXTURE to an actual recorded-data diagnostic")+ fixture = json.loads(Path(source).read_text())+ events, measurements = matching.prepare_events(fixture["events"])+ candidates, _ = matching.association_candidates(+ pd.DataFrame(fixture["cpa_rows"]), events,+ max_candidates=fixture.get("budget", {}).get("max_candidates_per_tile", 20000),+ anchor_window=fixture.get("recording_start_window_s"))+ grouped = {}+ for candidate in candidates:+ count = len(candidate["event_indices"])+ if count >= 2:+ grouped.setdefault(count, []).append(candidate)+ assert grouped, "Recorded fixture needs a multiple-group association"+ sizes = sorted(grouped)+ selected = [grouped[size][position]+ for size in sorted({sizes[0], sizes[len(sizes) // 2], sizes[-1]})+ for position in (0, -1)]+ identifiers = sorted({entry["flight_id"] for candidate in selected for entry in candidate["passes"]})+ tracks = {track.id: track for track in FlightIndex(fixture["real_index"]).lookup(identifiers)}+ point = fixture["point"]+ observer = ecef(point["lat"], point["lon"], point["elev_m"])+ result = []+ for candidate in selected:+ indices = candidate["event_indices"]+ chosen_tracks = [tracks[entry["flight_id"]] for entry in candidate["passes"]]+ chosen_events = [events[index] for index in indices]+ chosen_measurements = [measurements[index] for index in indices]+ lower, upper = candidate["recording_start_bounds_s"]+ starts = np.linspace(lower - 5.0, upper + 5.0, 21)+ batches = matching.batch_acoustic_trials(chosen_tracks, chosen_measurements, observer, starts)+ result.append((chosen_tracks, chosen_events, chosen_measurements, observer, starts, batches))+ return result+++def scalar_trials(case, floor, use_dual):+ tracks, events, measurements, observer, starts, batches = case+ flags = [None if batch is None or not use_dual else+ acoustic_screen.ridge_trial_rejections(measurement, event, *batch,+ score_floor=floor, group_count=len(events))+ for measurement, event, batch in zip(measurements, events, batches)]+ rejected_count = floor_count = 0+ outcomes = []+ for trial, start in enumerate(starts):+ def acoustic_at(index):+ batch = batches[index]+ return None if batch is None else (batch[0][trial], batch[1][trial])++ def dual_at(index):+ nonlocal rejected_count, floor_count+ if flags[index] is None:+ return None+ rejected, upper = flags[index]+ if rejected[trial]:+ rejected_count += 1+ return -np.inf+ if np.isfinite(upper[trial]):+ floor_count += 1+ return float(upper[trial])+ return None++ try:+ result, upper = search._screen_association_trial(+ tracks, events, measurements, observer, start, 343.0,+ acoustic_at, floor, dual_at if use_dual else None)+ outcomes.append((result, upper, None))+ except AcousticSupportError as error:+ outcomes.append((None, None, error))+ return outcomes, rejected_count, floor_count+++def encoded(value):+ def default(item):+ if isinstance(item, AcousticSupportError):+ return {"error": str(item), "evidence": vars(item)}+ if isinstance(item, np.ndarray):+ return item.tolist()+ if isinstance(item, np.generic):+ return item.item()+ raise TypeError(type(item).__name__)+ return json.dumps(value, sort_keys=True, default=default)++[email protected]("use_dual", [False, True])+def test_recorded_vectors_keep_scalar_decisions_at_adjacent_float_floors(+ recorded_associations, monkeypatch, use_dual):+ original = batch_ridge._first_event_fits_with_ceilings+ vector_batches = []+ boundary_cases = []++ def counted(*args, **kwargs):+ result = original(*args, **kwargs)+ if result[1] is not None:+ vector_batches.append(1)+ return result++ monkeypatch.setattr(batch_ridge, "_first_event_fits_with_ceilings", counted)++ @matching._cache_ridge_fits+ def run():+ for case in recorded_associations:+ tracks, events, measurements, observer, starts, batches = case+ first_bounds = scalar_trials(case, float(len(events)), False)[0]+ finite = sorted(upper for _, upper, _ in first_bounds+ if upper is not None and np.isfinite(upper))+ # Exercise the shortcut's remaining-group threshold on both sides+ # as well as actual fitted ceilings below. Observations stay fixed.+ remaining = float(len(events) - 1)+ floors = {0.0, float(len(events)), remaining,+ float(np.nextafter(remaining, -np.inf)),+ float(np.nextafter(remaining, np.inf))}+ if finite:+ boundary = finite[len(finite) // 2]+ boundary_cases.append(boundary)+ floors.update((float(np.nextafter(boundary, -np.inf)), boundary,+ float(np.nextafter(boundary, np.inf))))+ for floor in sorted(floors):+ expected = scalar_trials(case, floor, use_dual)+ actual = search._batched_ridge_trials(+ tracks, events, measurements, observer, starts, 343.0,+ iter(batches), floor, use_dual)+ assert encoded(actual) == encoded(expected), (len(events), floor, use_dual)+ run()+ assert vector_batches, "Recorded cases must exercise the canonical ceiling vector"+ assert boundary_cases, "Recorded cases must exercise adjacent-float retention boundaries"+++def test_recorded_vectors_keep_noncanonical_validation_fallback(recorded_associations, monkeypatch):+ original = search._ridge_trial_classification++ def custom(*args):+ return original(*args)++ monkeypatch.setattr(search, "_ridge_trial_classification", custom)++ @matching._cache_ridge_fits+ def run():+ for case in recorded_associations:+ tracks, events, measurements, observer, starts, batches = case+ expected = scalar_trials(case, 0.0, False)+ actual = search._batched_ridge_trials(+ tracks, events, measurements, observer, starts, 343.0,+ iter(batches), 0.0, False)+ assert encoded(actual) == encoded(expected)+ run()