Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Prepare verified trajectory inputs and reduce exact search work

Cinna Davisecb1726fb987
diff --git a/.github/workflows/real-performance.yml b/.github/workflows/real-performance.yml
index d727775..19d2d43 100644
--- a/.github/workflows/real-performance.yml
+++ b/.github/workflows/real-performance.yml
@@ -4,18 +4,20 @@ on:
inputs:
baseline:
description: Exact baseline commit for both real-input comparisons
- default: 27da08a878ac2ec90cd7e2c07f43765ceeea609b
+ default: 994d1ee48e25df6d30b23a356bba8f6502d8b7f6
required: true
regional:
description: Also compare the complete regional HTTP workflow
type: boolean
default: true
push:
- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103]
+ branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103, perf/prepared-flight-search-20261103]
paths:
- .github/workflows/real-performance.yml
- src/dopplergeist/**
- scripts/benchmark_prepared_measurements.py
+ - scripts/benchmark_prepared_flight_search.py
+ - scripts/prepare_flight_index.py
- scripts/benchmark_real_cpa_rows.py
- scripts/export_real_acoustic_controls.py
- scripts/benchmark_real_regional_upload.py
@@ -38,7 +40,7 @@ env:
NUMEXPR_NUM_THREADS: '1'
CC: gcc
PYTHONPATH: src
- BASELINE_REF: ${{ inputs.baseline || '27da08a878ac2ec90cd7e2c07f43765ceeea609b' }}
+ BASELINE_REF: ${{ inputs.baseline || '994d1ee48e25df6d30b23a356bba8f6502d8b7f6' }}
jobs:
compare:
if: github.event_name != 'push' || github.ref != 'refs/heads/perf/real-pipeline-throughput-20261103'
@@ -92,6 +94,12 @@ jobs:
--full-day --max-tracks 2000 --max-samples 500000 --max-candidates 20000 \
--anchor-window-s 1788864888.7151732 1788865236.7214916 \
--include-track-query --output benchmark-inputs/dense-query.json
+ - name: Prepare reusable numeric trajectory inputs
+ run: |
+ python scripts/benchmark_prepared_flight_search.py prepare-index \
+ --source src --index benchmark-inputs/reference-region.sqlite \
+ --destination benchmark-inputs/reference-region-prepared.sqlite \
+ --output results/flight-preparation.json
- name: Export real pipeline objective controls
env:
DOPPLERGEIST_RIDGE_BACKEND: python
@@ -102,7 +110,7 @@ jobs:
DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json
DOPPLERGEIST_REAL_REFERENCE_ROOT: benchmark-inputs/reference-checkout
DOPPLERGEIST_REAL_REFERENCE_DISCOVERY: evidence/audio/w1-sensitivity-20261102/reference/discovery.json
- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_native_ridge.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py
+ run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_native_ridge.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py tests/test_real_native_convergence.py tests/test_real_flight_payload.py tests/test_real_prepared_index.py tests/test_real_trial_vectors.py
- name: Verify the explicit Python fallback on recorded arithmetic
env:
DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json
@@ -129,6 +137,14 @@ jobs:
--baseline-source "$RUNNER_TEMP/baseline/src" --current-source src \
--repeats 3 --timeout-s 600 --output results/performance-dense-query \
"${native_option[@]}"
+ taskset -c "$benchmark_cpu" python scripts/benchmark_prepared_flight_search.py compare \
+ --fixture benchmark-inputs/dense-query.json \
+ --baseline-source "$RUNNER_TEMP/baseline/src" --current-source src \
+ --baseline-index benchmark-inputs/reference-region-prepared.sqlite \
+ --current-index benchmark-inputs/reference-region-prepared.sqlite \
+ --preparation-report results/flight-preparation.json \
+ --repeats 3 --timeout-s 600 --output results/performance-prepared-flight-search \
+ --require-native --require-native-acoustic --require-current-prepared
- name: Profile remaining point work outside the timing comparison
run: |
python -m cProfile -o results/current-point.prof scripts/benchmark_prepared_measurements.py _worker --source src --fixture benchmark-inputs/point-120.json --output results/current-point-profile --require-native --require-native-acoustic
diff --git a/scripts/benchmark_prepared_flight_search.py b/scripts/benchmark_prepared_flight_search.py
new file mode 100644
index 0000000..b70e45e
--- /dev/null
+++ b/scripts/benchmark_prepared_flight_search.py
@@ -0,0 +1,484 @@
+"""Measure real flight-query, overload-sieve and scoring work with exact parity.
+
+Use a frozen recorded-data fixture and two pinned src/ directories. Each arm runs
+in a fresh interpreter. An optional prepared index changes representation only;
+its one-time construction is measured separately by the prepare-index command.
+The source-informed regional receiver and anchor partition are performance
+workloads, not independent localization or complete national-search evidence.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import statistics
+import subprocess
+import sys
+import time
+
+try:
+ import resource
+except ImportError:
+ resource = None
+
+ROOT = Path(__file__).resolve().parents[1]
+
+# Query and scientific-output capture adapted from the project's PR #258
+# benchmark_prepared_measurements.py; keep the original scoring harness intact.
+
+
+def json_scalar(value):
+ if hasattr(value, "tolist"):
+ return value.tolist()
+ raise TypeError(f"Cannot serialize {type(value).__name__}")
+
+def write_new(path, value):
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with path.open("x") as output:
+ json.dump(value, output, indent=2, allow_nan=True, default=json_scalar)
+ output.write("\n")
+
+def canonical(value):
+ return json.dumps(value, sort_keys=True, separators=(",", ":"),
+ allow_nan=True, default=json_scalar)
+
+def query_tracks(index, points, events, budget, scoring, bounds, time_range):
+ """Count actual index traversal while exercising the production overload path."""
+ from dopplergeist.tiled_search import _tracks_for_points
+
+ if "elev_m" not in points:
+ raise ValueError("The recorded receiver hypothesis must include its elevation")
+ operations = {"query_calls": 0, "decoded_track_yields": 0, "decoded_sample_yields": 0}
+ inventory = {}
+ ordered_inventory = []
+
+ class CountedIndex:
+ def query(self, query_bounds, query_time):
+ operations["query_calls"] += 1
+ for track in index.query(query_bounds, query_time):
+ operations["decoded_track_yields"] += 1
+ operations["decoded_sample_yields"] += len(track.time)
+ inventory[track.id] = len(track.time)
+ ordered_inventory.append([track.id, len(track.time)])
+ yield track
+
+ radius = max(event["slant_range_m"][1] for event in events)
+ tracks, points = _tracks_for_points(
+ [CountedIndex()], bounds, time_range, budget, points, None, radius,
+ scoring.get("c", 343.0))
+ accounting = {
+ "queried_tracks_in_order": ordered_inventory,
+ "queried_track_ids": sorted(inventory),
+ "queried_trajectory_samples": sum(inventory.values()),
+ "selected_track_ids": [track.id for track in tracks],
+ "selected_trajectory_samples": sum(len(track.time) for track in tracks),
+ }
+ return tracks, points, operations, accounting
+
+def worker(args):
+ source = args.source.resolve()
+ sys.path.insert(0, str(source))
+ import dopplergeist
+ import pandas as pd
+ from dopplergeist import geometry, matching, tiled_search
+ from dopplergeist.data_access import FlightIndex
+ from dopplergeist.finite_search import RetentionLedger
+ from dopplergeist.geometry import AcousticSupportError, Track
+
+ if Path(dopplergeist.__file__).resolve().parent != source / "dopplergeist":
+ raise RuntimeError("Comparison imported a different source tree")
+ native_fit = getattr(matching, "_native_fit_irls", None)
+ ridge_backend = getattr(matching, "RIDGE_FIT_BACKEND", "python-irls-v1")
+ acoustic_backend = getattr(geometry, "ACOUSTIC_BACKEND", "python-acoustic-v1")
+ try:
+ from dopplergeist.native_acoustic import _native as native_acoustic
+ except ImportError:
+ native_acoustic = None
+ if args.require_native and not callable(native_fit):
+ raise RuntimeError("The selected source's native ridge fitter is unavailable")
+ if args.require_native_acoustic and native_acoustic is None:
+ raise RuntimeError("The selected source's native acoustic solver is unavailable")
+ fixture = json.loads(args.fixture.read_text())
+ index = FlightIndex(args.index or fixture["real_index"],
+ track_cache_bytes=args.track_cache_bytes)
+ if index.report["input_hash"] != fixture["index_input_hash"]:
+ raise ValueError("The real trajectory index no longer matches the fixture")
+ if args.require_prepared and not getattr(index, "_prepared_tracks", False):
+ raise RuntimeError("The selected source cannot use this prepared index")
+ budget = tiled_search.SearchBudget(**fixture["budget"])
+ loaded_at = time.perf_counter()
+ tracks = None
+ if not args.include_track_query:
+ tracks = list(index.lookup(fixture["track_ids"], max_tracks=budget.max_tracks_per_tile,
+ max_samples=budget.max_track_samples_per_tile))
+ if sorted(track.id for track in tracks) != sorted(fixture["track_ids"]):
+ raise ValueError("The index is missing a recorded trajectory")
+ events, measurements = matching.prepare_events(fixture["events"])
+ points = pd.DataFrame([fixture["point"]])
+ load_seconds = time.perf_counter() - loaded_at
+ args.output.mkdir(parents=True, exist_ok=False)
+ ledger = RetentionLedger(args.output / "retention.sqlite", "recorded-point-benchmark",
+ "matched_group_support_minus_normalized_residual")
+ stage = "coarse:recorded-point"
+ operations = {"scalar_fit_calls": 0, "acoustic_calls": 0, "scored_candidates": 0,
+ "native_fit_calls": 0, "native_fit_successes": 0,
+ "native_acoustic_successes": 0, "prepared_decode_calls": 0,
+ "prepared_decode_successes": 0}
+ all_rows = []
+ original_fit, original_acoustic = matching._fit_ridge_uncached, Track.acoustic
+ original_score = tiled_search.score_candidate
+ phase_seconds, phase_patches = {}, []
+ original_decoder, payload_module = None, None
+ if getattr(index, "_prepared_tracks", False):
+ from dopplergeist import flight_payload as payload_module
+ original_decoder = payload_module.decode_track
+
+ def counted_decode(*values, **options):
+ operations["prepared_decode_calls"] += 1
+ track = original_decoder(*values, **options)
+ operations["prepared_decode_successes"] += 1
+ return track
+
+ payload_module.decode_track = counted_decode
+
+ def timed_phase(name, function):
+ def run(*values, **options):
+ began, cpu = time.perf_counter(), time.process_time()
+ try:
+ return function(*values, **options)
+ finally:
+ phase = phase_seconds.setdefault(name, {"wall_seconds": 0.0, "cpu_seconds": 0.0, "calls": 0})
+ phase["wall_seconds"] += time.perf_counter() - began
+ phase["cpu_seconds"] += time.process_time() - cpu
+ phase["calls"] += 1
+ return run
+
+ if args.stage_timing:
+ phase_targets = [
+ (Track, "outside_observer_radius", "distance_sieve"),
+ (matching, "association_candidates", "association_enumeration"),
+ (tiled_search, "merge_candidates", "retention"),
+ ]
+ if callable(getattr(tiled_search, "cpa_table", None)):
+ phase_targets.append((tiled_search, "cpa_table", "closest_approach"))
+ else:
+ prepared_cpa = getattr(tiled_search, "_PreparedCPATrack", None)
+ if prepared_cpa is not None:
+ phase_targets.extend([
+ (prepared_cpa, "__init__", "closest_approach_preparation"),
+ (prepared_cpa, "raw_rows", "closest_approach"),
+ ])
+ for owner, attribute, name in phase_targets:
+ function = getattr(owner, attribute, None)
+ if function is not None:
+ phase_patches.append((owner, attribute, function))
+ setattr(owner, attribute, timed_phase(name, function))
+ score_function = timed_phase("candidate_scoring", original_score) if args.stage_timing else original_score
+
+ def counted_fit(*values, **options):
+ operations["scalar_fit_calls"] += 1
+ return original_fit(*values, **options)
+
+ def counted_acoustic(*values, **options):
+ operations["acoustic_calls"] += 1
+ return original_acoustic(*values, **options)
+
+ def counted_native(*values, **options):
+ operations["native_fit_calls"] += 1
+ result = native_fit(*values, **options)
+ if result is not None and all(math.isfinite(value) for value in result):
+ operations["native_fit_successes"] += 1
+ return result
+
+ def captured_score(*values, **options):
+ operations["scored_candidates"] += 1
+ try:
+ result = score_function(*values, **options)
+ except AcousticSupportError as error:
+ all_rows.append({"support_failure": tiled_search._support_error(error)})
+ raise
+ all_rows.append(result)
+ return result
+
+ matching._fit_ridge_uncached, Track.acoustic = counted_fit, counted_acoustic
+ tiled_search.score_candidate = captured_score
+ if callable(native_fit):
+ matching._native_fit_irls = counted_native
+ native_acoustic_before = native_acoustic.successful_calls() if native_acoustic is not None else 0
+ query_accounting, query_operations, execution_stages = None, None, {}
+ began, cpu = time.perf_counter(), time.process_time()
+ try:
+ if args.include_track_query:
+ query_began, query_cpu = time.perf_counter(), time.process_time()
+ tracks, points, query_operations, query_accounting = query_tracks(
+ index, points, events, budget, fixture["scoring"],
+ fixture["query_bounds"], fixture["query_time_s"])
+ execution_stages["track_query_and_distance_sieve"] = {
+ "wall_seconds": time.perf_counter() - query_began,
+ "cpu_seconds": time.process_time() - query_cpu,
+ }
+ if query_accounting["selected_track_ids"] != fixture["track_ids"]:
+ raise AssertionError("The real query changed the ordered selected-track inventory")
+ if query_accounting["selected_trajectory_samples"] != fixture["counts"]["trajectory_samples"]:
+ raise AssertionError("The real query changed the selected trajectory sample count")
+ prepared_query = fixture.get("track_query_preparation")
+ if prepared_query is not None:
+ if (len(query_accounting["queried_track_ids"]) != prepared_query["queried_trajectories"]
+ or query_accounting["queried_trajectory_samples"] != prepared_query["queried_trajectory_samples"]):
+ raise AssertionError("The real query changed the complete observed-track inventory")
+ scoring_began, scoring_cpu = time.perf_counter(), time.process_time()
+ partition = ({"anchor_window": fixture["anchor_window_s"]}
+ if "anchor_window_s" in fixture else {})
+ ranked, counts = tiled_search._score(
+ points, tracks, events, measurements, budget, fixture["scoring"], {},
+ start_window=fixture["recording_start_window_s"], ledger=ledger, stage=stage, **partition)
+ if args.include_track_query:
+ execution_stages["cpa_association_scoring_and_retention"] = {
+ "wall_seconds": time.perf_counter() - scoring_began,
+ "cpu_seconds": time.process_time() - scoring_cpu,
+ }
+ finally:
+ elapsed, cpu_seconds = time.perf_counter() - began, time.process_time() - cpu
+ matching._fit_ridge_uncached, Track.acoustic = original_fit, original_acoustic
+ tiled_search.score_candidate = original_score
+ if callable(native_fit):
+ matching._native_fit_irls = native_fit
+ if native_acoustic is not None:
+ operations["native_acoustic_successes"] = native_acoustic.successful_calls() - native_acoustic_before
+ for owner, attribute, function in reversed(phase_patches):
+ setattr(owner, attribute, function)
+ if payload_module is not None:
+ payload_module.decode_track = original_decoder
+ expected_counts = fixture.get("expected_counts")
+ if expected_counts is not None:
+ actual_counts = {"cpa_rows": counts["cpa_rows"],
+ "associations": counts["candidates"], "retained": len(ranked)}
+ if actual_counts != expected_counts:
+ raise AssertionError(f"Recorded workload counts changed: {actual_counts}")
+ # macOS reports bytes and Linux reports KiB. Keep scoring usable where
+ # resource or its unit convention is unavailable, without guessing a value.
+ peak_rss_mib = None
+ if resource is not None and (sys.platform == "darwin" or sys.platform.startswith("linux")):
+ rss_divisor = 1024 * 1024 if sys.platform == "darwin" else 1024
+ peak_rss_mib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rss_divisor
+ checksum = ledger.checksum(stage)
+ if not ledger.stage_valid(stage, checksum):
+ raise AssertionError("The scoring pass produced an invalid retention ledger")
+ if not operations["scored_candidates"]:
+ raise AssertionError("The benchmark did not score any real associations")
+ if args.require_native and not operations["native_fit_successes"]:
+ raise AssertionError("No successful native ridge fit occurred inside the timed scoring call")
+ if args.require_native_acoustic and not operations["native_acoustic_successes"]:
+ raise AssertionError("No successful native acoustic solve occurred inside the timed scoring call")
+ if args.require_prepared and not operations["prepared_decode_successes"]:
+ raise AssertionError("No successful prepared decode occurred inside the timed query")
+ with ledger.connect() as database:
+ retention = {name: database.execute(f"SELECT * FROM {name} ORDER BY 1,2").fetchall()
+ for name in ("records", "bounds", "stages")}
+ results = {"ranked": ranked, "counts": counts,
+ "all_scored_candidates": all_rows, "retention": retention}
+ if query_accounting is not None:
+ results["track_query_accounting"] = query_accounting
+ results["workload"] = {key: fixture.get(key) for key in (
+ "index_input_hash", "point", "events", "query_bounds", "query_time_s",
+ "recording_start_window_s", "anchor_window_s", "budget", "scoring")}
+ results["index_coverage"] = index.report["coverage"]
+ write_new(args.output / "results.json", results)
+ summary = {"wall_seconds": elapsed, "cpu_seconds": cpu_seconds,
+ "load_seconds_excluded": load_seconds, "operations": operations,
+ "ridge_fit_backend": ridge_backend, "acoustic_backend": acoustic_backend,
+ "native_extension_available": callable(native_fit),
+ "peak_rss_mib": peak_rss_mib,
+ "peak_rss_scope": ("Worker lifetime through scoring, including imports and input loading"
+ if peak_rss_mib is not None else "Unavailable on this platform"),
+ "retained": len(ranked), "independent_groups": len(events), "scope": fixture["scope"]}
+ summary["index_prepared_tracks"] = index.report.get("prepared_tracks")
+ summary["track_cache_bytes"] = args.track_cache_bytes
+ if "anchor_window_s" in fixture:
+ summary["anchor_window_s"] = fixture["anchor_window_s"]
+ if args.include_track_query:
+ summary["track_query_operations"] = query_operations
+ summary["execution_stage_seconds"] = execution_stages
+ summary["timing_scope"] = (
+ "Initial local index traversal and conservative distance sieve, followed by complete "
+ "CPA/association scoring and retention. Excludes imports, index metadata, event "
+ "preparation, downloads, terrain sampling, audio discovery, and refinement.")
+ if args.stage_timing:
+ summary["phase_seconds"] = phase_seconds
+ summary["phase_timing_scope"] = "Inclusive instrumented call times; overlapping phases must not be summed."
+ write_new(args.output / "summary.json", summary)
+ print(json.dumps(summary), flush=True)
+
+
+def file_digest(path):
+ with Path(path).open("rb") as stream:
+ return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def prepare_index(args):
+ """Time a new optional representation without changing the original index."""
+ sys.path.insert(0, str(args.source.resolve()))
+ import dopplergeist
+ from dopplergeist.data_access import FlightIndex
+
+ if Path(dopplergeist.__file__).resolve().parent != args.source.resolve() / "dopplergeist":
+ raise RuntimeError("Preparation imported a different source tree")
+ if args.output.exists() or args.destination.exists():
+ raise FileExistsError("Choose new preparation output and index paths")
+ before = file_digest(args.index)
+ began, cpu = time.perf_counter(), time.process_time()
+ original = FlightIndex(args.index)
+ prepared = original.prepare(args.destination)
+ elapsed, cpu_seconds = time.perf_counter() - began, time.process_time() - cpu
+ if before != file_digest(args.index):
+ raise AssertionError("Preparation modified the original observed index")
+ for key in ("input_hash", "coverage", "sources"):
+ if original.report.get(key) != prepared.report.get(key):
+ raise AssertionError(f"Preparation changed the original {key}")
+ if not getattr(prepared, "_prepared_tracks", False):
+ raise AssertionError("The optional representation is unavailable to its builder")
+ report = {
+ "wall_seconds": elapsed, "cpu_seconds": cpu_seconds,
+ "source": str(args.source.resolve()), "index": str(args.index.resolve()),
+ "destination": str(args.destination.resolve()),
+ "input_hash": original.report["input_hash"],
+ "original_unchanged": True,
+ "source_bytes": args.index.stat().st_size,
+ "prepared_bytes": args.destination.stat().st_size,
+ "prepared_tracks": prepared.report["prepared_tracks"],
+ "timing_scope": "One-time index copy, canonical decoding, optional representation and commit; excludes imports and verification.",
+ "cache_scope": "Source integrity verification reads the original index before timing; operating-system file caches are not flushed.",
+ }
+ write_new(args.output, report)
+ print(json.dumps(report), flush=True)
+
+
+def compare(args):
+ args.output.mkdir(parents=True, exist_ok=False)
+ sources = {"baseline": args.baseline_source.resolve(), "current": args.current_source.resolve()}
+ fixture = json.loads(args.fixture.read_text())
+ indexes = {"baseline": Path(args.baseline_index or fixture["real_index"]).resolve(),
+ "current": Path(args.current_index or fixture["real_index"]).resolve()}
+ environment = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")
+ for variable in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS"):
+ environment[variable] = "1"
+ runs, expected = [], None
+ for repetition in range(args.repeats):
+ order = ("baseline", "current") if repetition % 2 == 0 else ("current", "baseline")
+ for name in order:
+ output = args.output / f"{repetition + 1}-{name}"
+ command = [sys.executable, str(Path(__file__).resolve()), "_worker",
+ "--source", str(sources[name]), "--fixture", str(args.fixture.resolve()),
+ "--index", str(indexes[name]), "--output", str(output.resolve()),
+ "--track-cache-bytes", str(args.track_cache_bytes)]
+ for flag in ("stage_timing", "require_native", "require_native_acoustic"):
+ if getattr(args, flag):
+ command.append("--" + flag.replace("_", "-"))
+ if name == "current" and args.require_current_prepared:
+ command.append("--require-prepared")
+ print(f"Starting pair {repetition + 1}: {name}", flush=True)
+ subprocess.run(command, check=True, env=environment, timeout=args.timeout_s)
+ payload = canonical(json.loads((output / "results.json").read_text()))
+ if expected is None:
+ expected = payload
+ elif payload != expected:
+ raise AssertionError(f"Exact inventory, scientific-output or retention mismatch: {output}")
+ summary = json.loads((output / "summary.json").read_text())
+ runs.append({"source": name, "repetition": repetition + 1, **summary})
+ medians = {name: {} for name in sources}
+ for name in sources:
+ for metric in ("wall_seconds", "cpu_seconds", "peak_rss_mib"):
+ values = [row[metric] for row in runs if row["source"] == name and row[metric] is not None]
+ medians[name][metric] = statistics.median(values) if values else None
+ report = {
+ "identical_results": True, "repeats": args.repeats,
+ "fixture": str(args.fixture.resolve()),
+ "sources": {name: str(path) for name, path in sources.items()},
+ "indexes": {name: str(path) for name, path in indexes.items()},
+ "medians": medians,
+ "speedup": {metric: medians["baseline"][metric] / medians["current"][metric]
+ for metric in ("wall_seconds", "cpu_seconds")},
+ "execution_stage_medians": {
+ name: {stage: {metric: statistics.median(
+ row["execution_stage_seconds"][stage][metric]
+ for row in runs if row["source"] == name)
+ for metric in ("wall_seconds", "cpu_seconds")}
+ for stage in runs[0]["execution_stage_seconds"]}
+ for name in sources},
+ "scope": runs[0]["scope"], "timing_scope": runs[0]["timing_scope"],
+ "cache_scope": "Fresh interpreter and FlightIndex per arm; operating-system file caches are not flushed.",
+ "runs": runs,
+ }
+ if args.preparation_report:
+ preparation = json.loads(args.preparation_report.read_text())
+ if (Path(preparation["destination"]).resolve() != indexes["current"]
+ or preparation["input_hash"] != fixture["index_input_hash"]):
+ raise ValueError("Preparation report does not describe the compared current index")
+ report["preparation"] = preparation
+ report["preparation_amortization_scope"] = (
+ "Preparation cost divided by the complete baseline/current query-and-score saving. "
+ "This includes other code improvements and measures the combined upgrade, "
+ "not the isolated prepared-format contribution.")
+ report["queries_to_amortize_preparation"] = {}
+ for metric in ("wall_seconds", "cpu_seconds"):
+ saved = medians["baseline"][metric] - medians["current"][metric]
+ report["queries_to_amortize_preparation"][metric] = (
+ max(1, math.ceil(preparation[metric] / saved)) if saved > 0 else None)
+ write_new(args.output / "comparison.json", report)
+ print(json.dumps({key: report[key] for key in (
+ "identical_results", "medians", "speedup", "execution_stage_medians", "scope")}), flush=True)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ commands = parser.add_subparsers(dest="command", required=True)
+ preparation = commands.add_parser("prepare-index", help="Measure one-time creation of an optional index representation")
+ preparation.add_argument("--source", type=Path, default=ROOT / "src")
+ preparation.add_argument("--index", type=Path, required=True)
+ preparation.add_argument("--destination", type=Path, required=True)
+ preparation.add_argument("--output", type=Path, required=True)
+ comparison = commands.add_parser("compare", help="Compare equal real query/sieve/scoring workloads")
+ comparison.add_argument("--fixture", required=True, type=Path)
+ comparison.add_argument("--baseline-source", required=True, type=Path)
+ comparison.add_argument("--current-source", type=Path, default=ROOT / "src")
+ comparison.add_argument("--baseline-index", type=Path)
+ comparison.add_argument("--current-index", type=Path)
+ comparison.add_argument("--preparation-report", type=Path)
+ comparison.add_argument("--repeats", type=int, default=3)
+ comparison.add_argument("--timeout-s", type=float, default=900.0)
+ comparison.add_argument("--output", required=True, type=Path)
+ comparison.add_argument("--require-current-prepared", action="store_true")
+ internal = commands.add_parser("_worker", help=argparse.SUPPRESS)
+ internal.add_argument("--source", required=True, type=Path)
+ internal.add_argument("--fixture", required=True, type=Path)
+ internal.add_argument("--index", type=Path)
+ internal.add_argument("--output", required=True, type=Path)
+ internal.add_argument("--require-prepared", action="store_true")
+ for command in (comparison, internal):
+ command.add_argument("--track-cache-bytes", type=int, default=32 * 1024 * 1024)
+ command.add_argument("--stage-timing", action="store_true",
+ help="Collect inclusive call times in a separate diagnostic run")
+ command.add_argument("--require-native", action="store_true",
+ help="Require successful native ridge fits in every compared source")
+ command.add_argument("--require-native-acoustic", action="store_true",
+ help="Require successful native acoustic solves in every compared source")
+ command.set_defaults(include_track_query=True)
+ args = parser.parse_args()
+ if hasattr(args, "track_cache_bytes") and args.track_cache_bytes < 0:
+ parser.error("--track-cache-bytes must be nonnegative")
+ if args.command == "prepare-index":
+ prepare_index(args)
+ elif args.command == "compare":
+ if args.repeats < 1 or not math.isfinite(args.timeout_s) or args.timeout_s <= 0:
+ parser.error("--repeats and --timeout-s must be positive")
+ compare(args)
+ else:
+ worker(args)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/prepare_flight_index.py b/scripts/prepare_flight_index.py
new file mode 100644
index 0000000..3222191
--- /dev/null
+++ b/scripts/prepare_flight_index.py
@@ -0,0 +1,31 @@
+"""Prepare a separately named real flight index for repeated search queries.
+
+The copy retains observations, coverage and input identity. It adds validated
+numeric track inputs and keeps the original JSON for older readers and later
+normalization changes. No trajectories are generated.
+"""
+import argparse
+import json
+from pathlib import Path
+import time
+
+from dopplergeist.data_access import FlightIndex
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--index", required=True, type=Path)
+ parser.add_argument("--output", required=True, type=Path)
+ args = parser.parse_args()
+ began = time.perf_counter()
+ result = FlightIndex(args.index).prepare(args.output)
+ print(json.dumps({"output": str(args.output.resolve()),
+ "elapsed_s": time.perf_counter() - began,
+ "source_bytes": args.index.stat().st_size,
+ "prepared_bytes": args.output.stat().st_size,
+ "input_hash": result.report["input_hash"],
+ "prepared_tracks": result.report["prepared_tracks"]}, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/dopplergeist/_native_acoustic.c b/src/dopplergeist/_native_acoustic.c
index a591139..9a47080 100644
--- a/src/dopplergeist/_native_acoustic.c
+++ b/src/dopplergeist/_native_acoustic.c
@@ -8,6 +8,10 @@
#include <fenv.h>
#include <string.h>
+/* Larger calls keep the original all-lane solver instead of allocating
+ * unbounded temporary state or falling back to the Python implementation. */
+#define DG_MAX_STATE_BYTES ((size_t)2 * 1024 * 1024)
+
/* Compile without contraction or reassociation. SciPy PPoly uses
* increasing powers, not Horner evaluation. Preserve that arithmetic exactly.
*/
@@ -23,10 +27,20 @@ static size_t interval_at(const double *x, size_t m, double value) {
}
static int evaluate(const double *coefficients, const double *x, size_t m,
- size_t width, double origin, double t, double *values) {
+ size_t width, double origin, double t, double *values,
+ uint16_t *interval) {
double relative = t - origin;
if (!(relative >= x[0] && relative <= x[m])) return 1;
- size_t i = interval_at(x, m, relative);
+ size_t i = interval == NULL ? m : *interval;
+ /* Newton updates usually stay in one native spline interval. Reuse only
+ * after checking its exact half-open bounds; the last interval owns x[m].
+ * Both spline shapes are bounded to 4096 intervals by the public wrapper.
+ */
+ if (!(i < m && relative >= x[i] &&
+ (relative < x[i+1] || (i+1 == m && relative == x[m])))) {
+ i = interval_at(x, m, relative);
+ if (interval != NULL) *interval = (uint16_t)i;
+ }
double s = relative - x[i];
double z = 1.0;
for (size_t j = 0; j < width; ++j) values[j] = 0.0;
@@ -52,9 +66,9 @@ static double norm3(const double *a) {
static int kinematics(const double *coefficients, const double *x,
size_t m, double origin, double tau, const double *observer,
- double *distance, double *radial) {
+ double *distance, double *radial, uint16_t *interval) {
double values[6], delta[3];
- if (evaluate(coefficients, x, m, 6, origin, tau, values)) return 1;
+ if (evaluate(coefficients, x, m, 6, origin, tau, values, interval)) return 1;
for (size_t j = 0; j < 3; ++j) delta[j] = values[j] - observer[j];
*distance = norm3(delta);
*radial = (((0.0 + delta[0]*values[3]) + delta[1]*values[4])
@@ -67,12 +81,14 @@ static int dg_acoustic(const double *t, size_t n, const double *observer,
const double *packed, const double *packed_x, size_t packed_m,
double origin, double start, double end, double c,
double arrival_start, double arrival_end,
- double *factor, double *tau, double *distance) {
+ double *factor, double *tau, double *distance, uint16_t *state) {
if (n == 0 || m == 0 || m > 4096) return 1;
+ uint16_t initial_interval = (uint16_t)m;
for (size_t i = 0; i < n; ++i) {
double position[3], delta[3];
if (!isfinite(t[i])) return 1;
- if (evaluate(p, x, m, 3, origin, clip(t[i], start, end), position)) return 1;
+ if (evaluate(p, x, m, 3, origin, clip(t[i], start, end), position,
+ &initial_interval)) return 1;
for (size_t j = 0; j < 3; ++j) delta[j] = position[j] - observer[j];
double initial_distance = norm3(delta);
if (!isfinite(initial_distance)) return 1;
@@ -80,13 +96,22 @@ static int dg_acoustic(const double *t, size_t n, const double *observer,
double estimate = t[i] - initial_delay;
if (!isfinite(initial_delay) || !isfinite(estimate)) return 1;
tau[i] = clip(estimate, start, end);
+ if (state != NULL) state[i] = (uint16_t)packed_m;
}
int unchanged = 0;
+ double fixed_maximum = 0.0;
for (size_t iteration = 0; iteration < 10; ++iteration) {
unchanged = 1;
- double maximum = 0.0;
+ double maximum = fixed_maximum;
for (size_t i = 0; i < n; ++i) {
- if (kinematics(packed, packed_x, packed_m, origin, tau[i], observer, distance+i, factor+i)) return 1;
+ /* UINT16_MAX marks an exact floating-point fixed point. Repeating
+ * its polynomial evaluation and update cannot change its values.
+ * Its correction still participates in the global stopping rule:
+ * a clamped unsupported endpoint can have a large correction.
+ */
+ if (state != NULL && state[i] == UINT16_MAX) continue;
+ if (kinematics(packed, packed_x, packed_m, origin, tau[i], observer,
+ distance+i, factor+i, state == NULL ? NULL : state+i)) return 1;
double derivative = 1.0 + factor[i]/c;
if (!isfinite(derivative) || !(derivative > 0.0)) return 1;
double correction = ((tau[i] - t[i]) + distance[i]/c)/derivative;
@@ -95,6 +120,12 @@ static int dg_acoustic(const double *t, size_t n, const double *observer,
if (!isfinite(estimate)) return 1;
double updated = clip(estimate, start, end);
if (updated != tau[i]) unchanged = 0;
+ else if (state != NULL && (updated != 0.0 || signbit(updated) == signbit(tau[i]))) {
+ /* A signed-zero change compares equal but need not reproduce
+ * the same polynomial arithmetic on a subsequent iteration. */
+ state[i] = UINT16_MAX;
+ if (fabs(correction) > fixed_maximum) fixed_maximum = fabs(correction);
+ }
tau[i] = updated;
double absolute = fabs(correction);
if (absolute > maximum) maximum = absolute;
@@ -103,7 +134,9 @@ static int dg_acoustic(const double *t, size_t n, const double *observer,
}
if (!unchanged) {
for (size_t i = 0; i < n; ++i) {
- if (kinematics(packed, packed_x, packed_m, origin, tau[i], observer, distance+i, factor+i)) return 1;
+ if (state != NULL && state[i] == UINT16_MAX) continue;
+ if (kinematics(packed, packed_x, packed_m, origin, tau[i], observer,
+ distance+i, factor+i, state == NULL ? NULL : state+i)) return 1;
}
}
for (size_t i = 0; i < n; ++i) {
@@ -163,6 +196,8 @@ static int increasing_knots(const Py_buffer *view) {
static PyObject *solve(PyObject *self, PyObject *args) {
PyObject *objects[9];
Py_buffer views[9] = {{0}};
+ uint16_t small_state[64];
+ uint16_t *state = small_state;
double origin, start, end, c, arrival_start, arrival_end;
int success = 0;
if (!PyArg_ParseTuple(args,"OOOOOOddddddOOO",
@@ -182,17 +217,23 @@ static PyObject *solve(PyObject *self, PyObject *args) {
if (!same_shape(views,views+i)) goto done;
const double *observer = views[1].buf;
for (int j = 0; j < 3; ++j) if (!isfinite(observer[j])) goto done;
+ size_t samples = (size_t)(views[0].len/sizeof(double));
+ if (samples > sizeof(small_state)/sizeof(small_state[0])) {
+ state = samples <= DG_MAX_STATE_BYTES/sizeof(*state)
+ ? PyMem_Malloc(samples*sizeof(*state)) : NULL;
+ }
/* A failed probe must not leak arithmetic flags into the NumPy fallback. */
fenv_t previous_environment;
if (fegetenv(&previous_environment) != 0) goto done;
Py_BEGIN_ALLOW_THREADS
- success = !dg_acoustic(views[0].buf,(size_t)(views[0].len/sizeof(double)),
+ success = !dg_acoustic(views[0].buf,samples,
observer,views[2].buf,views[3].buf,(size_t)views[2].shape[1],
views[4].buf,views[5].buf,(size_t)views[4].shape[1],origin,start,end,c,
- arrival_start,arrival_end,views[6].buf,views[7].buf,views[8].buf);
+ arrival_start,arrival_end,views[6].buf,views[7].buf,views[8].buf,state);
if (fesetenv(&previous_environment) != 0) success = 0;
Py_END_ALLOW_THREADS
done:
+ if (state != small_state) PyMem_Free(state);
for (int i = 0; i < 9; ++i) if (views[i].obj) PyBuffer_Release(views+i);
if (PyErr_Occurred()) PyErr_Clear();
if (success) ((module_state *)PyModule_GetState(self))->successful_calls++;
diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.py
index a018a6b..7b48954 100644
--- a/src/dopplergeist/data_access.py
+++ b/src/dopplergeist/data_access.py
@@ -18,6 +18,7 @@ import os
from pathlib import Path
import pickle
import sqlite3
+import sys
import tarfile
from threading import Lock
@@ -138,6 +139,30 @@ def _payload_frame(records):
return _IndexColumns(columns, len(records))
+_CANONICAL_FRAME_TRACK = _frame_track
+_CANONICAL_PAYLOAD_FRAME = _payload_frame
+_CANONICAL_SEGMENT_VELOCITY = _segment_velocity
+_CANONICAL_TRACK = Track
+_PREPARED_ERROR_POLICY = {"divide": "warn", "over": "warn", "under": "ignore", "invalid": "warn"}
+
+
+@lru_cache(maxsize=1)
+def _prepared_track_identity():
+ """Use prepared velocities only with identical normalization and NumPy."""
+ from . import flight_payload
+ try:
+ import numpy._core._multiarray_umath as numpy_core
+ except ImportError:
+ import numpy.core._multiarray_umath as numpy_core
+ files = (Path(__file__), Path(__file__).with_name("tracks.py"),
+ Path(__file__).with_name("geometry.py"), Path(flight_payload.__file__),
+ Path(numpy_core.__file__))
+ return content_hash({"sources": [sha256(path) for path in files],
+ "numpy": np.__version__,
+ "pandas": pd.__version__, "python": sys.version,
+ "cpu": getattr(numpy_core, "__cpu_features__", {})})
+
+
@lru_cache(maxsize=1)
def _ecef_to_geographic():
from pyproj import Transformer
@@ -494,15 +519,34 @@ class FlightIndex:
try:
with self.connect() as db:
self.report = json.loads(db.execute("SELECT value FROM metadata").fetchone()[0])
- if self.report["index_version"] not in (1, INDEX_VERSION):
- raise DataUnavailable("Flight index version mismatch")
+ if self.report["index_version"] not in (1, INDEX_VERSION):
+ raise DataUnavailable("Flight index version mismatch")
+ self._prepared_tracks = self._prepared_available(db, self.report)
except (sqlite3.DatabaseError, TypeError, KeyError, json.JSONDecodeError) as error:
raise DataUnavailable(f"Invalid flight index: {path}") from error
def connect(self):
return sqlite3.connect(f"{self.path.resolve().as_uri()}?mode=ro", uri=True)
- def _decode_track(self, name, payload, expected, *, max_samples=None, cache_scan=None):
+ @staticmethod
+ def _prepared_available(db, report=None):
+ # Recheck optional metadata and schema in the actual read snapshot.
+ # The unique integer key must prevent duplicate original segments.
+ try:
+ if report is None:
+ report = json.loads(db.execute("SELECT value FROM metadata").fetchone()[0])
+ prepared = report.get("prepared_tracks") if isinstance(report, dict) else None
+ if (not isinstance(prepared, dict) or prepared.get("version") != 1
+ or prepared.get("builder") != _prepared_track_identity()):
+ return False
+ columns = [(row[1], row[2].upper(), row[5]) for row in
+ db.execute("PRAGMA table_info(prepared_tracks)")]
+ return columns == [("id", "INTEGER", 1), ("source_hash", "TEXT", 0),
+ ("payload", "BLOB", 0), ("payload_hash", "TEXT", 0)]
+ except (sqlite3.DatabaseError, TypeError, KeyError, json.JSONDecodeError):
+ return False
+
+ def _decode_track(self, name, payload, expected, *, max_samples=None, cache_scan=None, prepared=None):
# Verify every fetched payload, including hits. A cached track must not
# conceal damage to an index that was changed after the previous query.
if hashlib.sha256(payload).hexdigest() != expected:
@@ -519,10 +563,26 @@ class FlightIndex:
cache_scan["used"].add(key)
if entry is not None:
return pickle.loads(cached)
- frame = _payload_frame(json.loads(gzip.decompress(payload)))
- if max_samples is not None and len(frame) > max_samples:
- raise DataUnavailable("Flight lookup sample budget exceeded")
- track = _frame_track(name, frame)
+ track = None
+ if (prepared is not None and _frame_track is _CANONICAL_FRAME_TRACK
+ and _segment_velocity is _CANONICAL_SEGMENT_VELOCITY
+ and _payload_frame is _CANONICAL_PAYLOAD_FRAME
+ and Track is _CANONICAL_TRACK and np.geterr() == _PREPARED_ERROR_POLICY):
+ source_hash, packed, packed_hash = prepared
+ if source_hash == expected and type(packed) is bytes:
+ from .flight_payload import decode_track
+ # A damaged optional representation cannot replace the verified
+ # original observations. Recompute through the legacy reader.
+ if hashlib.sha256(packed).hexdigest() == packed_hash:
+ try:
+ track = decode_track(name, packed, max_samples=max_samples)
+ except DataUnavailable:
+ track = None
+ if track is None:
+ frame = _payload_frame(json.loads(gzip.decompress(payload)))
+ if max_samples is not None and len(frame) > max_samples:
+ raise DataUnavailable("Flight lookup sample budget exceeded")
+ track = _frame_track(name, frame)
if self._track_cache_limit and (cache_scan is None or cache_scan["admit"]):
# These bytes are produced here, never read from a file/request.
# Restoring them returns independent arrays and mutable diagnostics,
@@ -551,12 +611,78 @@ class FlightIndex:
return track
_, (_, previous) = self._track_cache.popitem(last=False)
self._track_cache_bytes -= len(previous)
- self._track_cache[key] = (len(frame), cached)
+ self._track_cache[key] = (len(track.time), cached)
self._track_cache_bytes += len(cached)
if cache_scan is not None:
cache_scan["used"].add(key)
return track
+ @staticmethod
+ def _prepare_tracks(db):
+ """Store constructor inputs beside the original observations, once."""
+ from .flight_payload import encode_track
+ db.execute("DROP TABLE IF EXISTS prepared_tracks")
+ db.execute("CREATE TABLE prepared_tracks(id INTEGER PRIMARY KEY, source_hash TEXT, payload BLOB, payload_hash TEXT)")
+ count = size = 0
+ if (_frame_track is not _CANONICAL_FRAME_TRACK
+ or _segment_velocity is not _CANONICAL_SEGMENT_VELOCITY
+ or _payload_frame is not _CANONICAL_PAYLOAD_FRAME
+ or Track is not _CANONICAL_TRACK or np.geterr() != _PREPARED_ERROR_POLICY):
+ return {"version": 1, "builder": None, "tracks": count, "bytes": size}
+ for identifier, name, payload, expected in db.execute(
+ "SELECT id,name,payload,payload_hash FROM segments ORDER BY id"):
+ if hashlib.sha256(payload).hexdigest() != expected:
+ raise DataUnavailable(f"Corrupt flight payload: {name}")
+ # Use the precise values the old reader gets AFTER the index's
+ # JSON serialization and dtype inference, not the staging frame.
+ track = _frame_track(name, _payload_frame(json.loads(gzip.decompress(payload))))
+ try:
+ packed = encode_track(track)
+ except (DataUnavailable, TypeError, ValueError):
+ continue
+ db.execute("INSERT INTO prepared_tracks VALUES(?,?,?,?)",
+ (identifier, expected, packed, hashlib.sha256(packed).hexdigest()))
+ count += 1
+ size += len(packed)
+ return {"version": 1, "builder": _prepared_track_identity(), "tracks": count, "bytes": size}
+
+ def prepare(self, destination):
+ """Create a new index copy with faster verified track decoding.
+
+ The source stays read-only and the destination must not exist. Coverage,
+ original payloads, spatial bounds and scientific input identity stay
+ unchanged. Old readers ignore the optional numeric representation.
+ """
+ from uuid import uuid4
+ destination = Path(destination)
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ with cache_lock(destination.with_suffix(".lock")):
+ if destination.exists() or destination.is_symlink():
+ raise FileExistsError(f"Prepared index destination already exists: {destination}")
+ temporary = destination.with_name(destination.name + ".preparing-" + uuid4().hex)
+ db = sqlite3.connect(temporary)
+ try:
+ with self.connect() as source:
+ source.backup(db)
+ # Copy metadata from the same snapshot as the observations.
+ # The caller's older report object is mutable and may be stale.
+ try:
+ report = json.loads(db.execute("SELECT value FROM metadata").fetchone()[0])
+ if report["index_version"] not in (1, INDEX_VERSION):
+ raise DataUnavailable("Flight index version mismatch")
+ except (sqlite3.DatabaseError, TypeError, KeyError, json.JSONDecodeError) as error:
+ raise DataUnavailable("Invalid source flight index metadata") from error
+ report["prepared_tracks"] = self._prepare_tracks(db)
+ db.execute("UPDATE metadata SET value=?", (json.dumps(report),))
+ db.commit()
+ db.close()
+ temporary.replace(destination)
+ except BaseException:
+ db.close()
+ temporary.unlink(missing_ok=True)
+ raise
+ return type(self)(destination, track_cache_bytes=self._track_cache_limit)
+
@classmethod
def build(cls, path, frames, *, sources, coverage, max_gap_s=30.0, max_flight_samples=250000, class_map=None,
prepared=False):
@@ -624,6 +750,7 @@ class FlightIndex:
report = dict(identity, input_hash=content_hash(identity), rows=rows, invalid_rows=invalid,
segments=segments, altitude_basis_counts=dict(altitude), jets_excluded=False,
unknown_types_retained=True, geometry_clipped=False, continuous_geometry_bounds=True)
+ report["prepared_tracks"] = cls._prepare_tracks(db)
db.execute("INSERT INTO metadata VALUES(?)", (json.dumps(report),))
db.commit()
db.execute("VACUUM")
@@ -698,10 +825,17 @@ class FlightIndex:
join, predicate, params = self._query_scope(bounds, time_range)
cache_scan = {"used": set(), "admit": True}
with self.connect() as db:
- sql = ("SELECT s.name,s.payload,s.payload_hash FROM segments s" + join
+ prepared = self._prepared_tracks
+ if prepared:
+ db.execute("BEGIN")
+ prepared = self._prepared_available(db)
+ fields = ",p.source_hash,p.payload,p.payload_hash" if prepared else ""
+ prepared_join = " LEFT JOIN prepared_tracks p ON p.id=s.id" if prepared else ""
+ sql = ("SELECT s.name,s.payload,s.payload_hash" + fields + " FROM segments s" + join + prepared_join
+ " WHERE " + predicate + " ORDER BY s.name")
- for name, payload, expected in db.execute(sql, params):
- yield self._decode_track(name, payload, expected, cache_scan=cache_scan)
+ for row in db.execute(sql, params):
+ yield self._decode_track(*row[:3], cache_scan=cache_scan,
+ prepared=row[3:] if prepared else None)
def lookup(self, identifiers, *, max_tracks=2000, max_samples=500000,
bounds=None, time_range=None):
@@ -730,16 +864,24 @@ class FlightIndex:
join, predicate, scope_params = self._query_scope(bounds, time_range)
cache_scan = {"used": set(), "admit": True}
with self.connect() as db:
+ prepared = self._prepared_tracks
+ if prepared:
+ # Retain one snapshot across the checks and all lookup batches.
+ db.execute("BEGIN")
+ prepared = self._prepared_available(db)
# Stay below SQLite's conservative bound on bound SQL parameters.
for first in range(0, len(names), 500):
batch = names[first:first+500]
placeholders = ",".join("?" for _ in batch)
- sql = ("SELECT s.name,s.payload,s.payload_hash FROM segments s" + join
+ fields = ",p.source_hash,p.payload,p.payload_hash" if prepared else ""
+ prepared_join = " LEFT JOIN prepared_tracks p ON p.id=s.id" if prepared else ""
+ sql = ("SELECT s.name,s.payload,s.payload_hash" + fields + " FROM segments s" + join + prepared_join
+ " WHERE s.name IN (" + placeholders + ")"
+ (" AND " + predicate if predicate else "") + " ORDER BY s.name")
- for name, payload, expected in db.execute(sql, (*batch, *scope_params)):
- track = self._decode_track(name, payload, expected, max_samples=max_samples - samples,
- cache_scan=cache_scan)
+ for row in db.execute(sql, (*batch, *scope_params)):
+ track = self._decode_track(*row[:3], max_samples=max_samples - samples,
+ cache_scan=cache_scan,
+ prepared=row[3:] if prepared else None)
samples += len(track.time)
yield track
diff --git a/src/dopplergeist/flight_payload.py b/src/dopplergeist/flight_payload.py
new file mode 100644
index 0000000..233b796
--- /dev/null
+++ b/src/dopplergeist/flight_payload.py
@@ -0,0 +1,197 @@
+"""Bounded numeric snapshots of canonical flight-decoder inputs.
+
+Encode a Track reconstructed from the stored JSON payload. The original payload
+and its integrity/implementation checks remain the caller's responsibility. The
+snapshot stores float64 inputs and labels; Track reconstructs its spline and
+private caches on every decode. No executable object format is read from disk.
+"""
+import json
+import struct
+
+import numpy as np
+
+from .geometry import Track
+
+
+FORMAT_VERSION = 1
+_MAGIC = b"DGFLT64\0"
+_PREFIX = struct.Struct("<8sIIQ")
+_MAX_SAMPLES = 1_000_000
+_MAX_HEADER_BYTES = 1024 * 1024
+_MAX_PAYLOAD_BYTES = _PREFIX.size + _MAX_HEADER_BYTES + 7 * 8 * _MAX_SAMPLES
+_MAX_LABEL_LENGTH = 1024
+_COMPONENTS = ("east_mps", "north_mps", "up_mps")
+_FIELDS = {"id", "typecode", "aircraft_class", "altitude_basis",
+ "velocity_inferred_components", "velocity"}
+
+
+def _invalid(reason):
+ # data_access may import this codec while defining FlightIndex.
+ from .data_access import DataUnavailable
+ return DataUnavailable(f"Invalid prepared flight payload: {reason}")
+
+
+def _label(value):
+ return type(value) is str and len(value) <= _MAX_LABEL_LENGTH
+
+
+def _metadata(track, count):
+ if not all(_label(value) for value in (track.id, track.typecode,
+ getattr(track, "aircraft_class", None))):
+ raise _invalid("noncanonical flight labels")
+ basis = getattr(track, "altitude_basis", None)
+ if type(basis) is not list or len(basis) != count:
+ raise _invalid("altitude labels do not match the trajectory")
+ runs = []
+ # JSON escapes use at most six bytes per character. Bound metadata while
+ # forming runs, before serializing a potentially large alternating list.
+ header_bound = 512 + 6 * sum(len(value) for value in
+ (track.id, track.typecode, track.aircraft_class))
+ for label in basis:
+ if not _label(label):
+ raise _invalid("noncanonical altitude label")
+ if runs and runs[-1][0] == label:
+ runs[-1][1] += 1
+ else:
+ header_bound += 6 * len(label) + 32
+ if header_bound > _MAX_HEADER_BYTES:
+ raise _invalid("metadata exceeds the codec limit")
+ runs.append([label, 1])
+ inferred = getattr(track, "velocity_inferred_components", None)
+ if (type(inferred) is not dict or set(inferred) != set(_COMPONENTS)
+ or any(type(inferred[key]) is not int or not 0 <= inferred[key] <= count
+ for key in _COMPONENTS)):
+ raise _invalid("noncanonical velocity-inference counts")
+ return {"id": track.id, "typecode": track.typecode,
+ "aircraft_class": track.aircraft_class, "altitude_basis": runs,
+ "velocity_inferred_components": {key: inferred[key] for key in _COMPONENTS},
+ "velocity": track.velocity is not None}
+
+
+def encode_track(track):
+ """Encode exact inputs from a freshly decoded canonical Track.
+
+ Noncanonical metadata or oversized trajectories are left to the original
+ payload path. This function does not infer velocities or round any values.
+ """
+ if type(track) is not Track:
+ raise _invalid("noncanonical Track type")
+ if type(track.time) is not np.ndarray or track.time.ndim != 1:
+ raise _invalid("noncanonical trajectory array")
+ count = len(track.time)
+ if not 3 <= count <= _MAX_SAMPLES:
+ raise _invalid("trajectory sample count is outside the codec limit")
+ arrays = [track.time, track.lat, track.lon, track.altitude]
+ for values in arrays:
+ if (type(values) is not np.ndarray or values.dtype != np.dtype(float)
+ or values.shape != (count,) or not np.all(np.isfinite(values))):
+ raise _invalid("noncanonical trajectory array")
+ if not np.all(np.diff(track.time) > 0):
+ raise _invalid("trajectory times are not strictly increasing")
+ if track.velocity is not None:
+ velocity = track.velocity
+ if (type(velocity) is not np.ndarray or velocity.dtype != np.dtype(float)
+ or velocity.shape != (count, 3) or not np.all(np.isfinite(velocity))):
+ raise _invalid("noncanonical velocity array")
+ arrays.append(velocity)
+ try:
+ header = json.dumps(_metadata(track, count), ensure_ascii=False,
+ separators=(",", ":"), allow_nan=False).encode("utf-8")
+ except (ValueError, TypeError, UnicodeError) as error:
+ raise _invalid("noncanonical metadata encoding") from error
+ if len(header) > _MAX_HEADER_BYTES:
+ raise _invalid("metadata exceeds the codec limit")
+ return b"".join([_PREFIX.pack(_MAGIC, FORMAT_VERSION, len(header), count), header,
+ *(values.astype("<f8", copy=False).tobytes(order="C") for values in arrays)])
+
+
+def _object(pairs):
+ result = {}
+ for key, value in pairs:
+ if key in result:
+ raise ValueError("duplicate metadata key")
+ result[key] = value
+ return result
+
+
+def _constant(value):
+ raise ValueError("nonfinite JSON metadata")
+
+
+def _decode_metadata(header, count, name):
+ try:
+ metadata = json.loads(header.decode("utf-8"), object_pairs_hook=_object,
+ parse_constant=_constant)
+ except (ValueError, UnicodeDecodeError, RecursionError) as error:
+ raise _invalid("malformed JSON metadata") from error
+ if type(metadata) is not dict or set(metadata) != _FIELDS:
+ raise _invalid("unexpected metadata fields")
+ if (not all(_label(metadata[key]) for key in ("id", "typecode", "aircraft_class"))
+ or metadata["id"] != name or type(metadata["velocity"]) is not bool):
+ raise _invalid("flight identity or labels do not match")
+ inferred = metadata["velocity_inferred_components"]
+ if (type(inferred) is not dict or set(inferred) != set(_COMPONENTS)
+ or any(type(inferred[key]) is not int or not 0 <= inferred[key] <= count
+ for key in _COMPONENTS)):
+ raise _invalid("invalid velocity-inference counts")
+ runs = metadata["altitude_basis"]
+ if type(runs) is not list or not 1 <= len(runs) <= count:
+ raise _invalid("invalid altitude label runs")
+ length = 0
+ for run in runs:
+ if (type(run) is not list or len(run) != 2 or not _label(run[0])
+ or type(run[1]) is not int or not 1 <= run[1] <= count - length):
+ raise _invalid("invalid altitude label run")
+ length += run[1]
+ if length != count:
+ raise _invalid("altitude labels do not match the trajectory")
+ return metadata
+
+
+def decode_track(name, payload, *, max_samples=None):
+ """Return independent writable inputs and a reconstructed Track.
+
+ Validate counts and byte lengths before allocating arrays or expanding label
+ runs. A malformed or unsupported snapshot raises DataUnavailable so callers
+ can use their original JSON payload under the same integrity checks.
+ """
+ if type(payload) is not bytes or not _PREFIX.size <= len(payload) <= _MAX_PAYLOAD_BYTES:
+ raise _invalid("invalid payload size or buffer type")
+ if not _label(name):
+ raise _invalid("noncanonical flight identifier")
+ if max_samples is not None and (type(max_samples) is not int or max_samples < 0):
+ raise _invalid("invalid sample budget")
+ magic, version, header_size, count = _PREFIX.unpack_from(payload)
+ if magic != _MAGIC or version != FORMAT_VERSION:
+ raise _invalid("unsupported magic or format version")
+ if not 3 <= count <= _MAX_SAMPLES or not 0 < header_size <= _MAX_HEADER_BYTES:
+ raise _invalid("sample or metadata count exceeds the codec limit")
+ if max_samples is not None and count > max_samples:
+ from .data_access import DataUnavailable
+ raise DataUnavailable("Flight lookup sample budget exceeded")
+ offset = _PREFIX.size + header_size
+ if offset > len(payload):
+ raise _invalid("truncated metadata")
+ metadata = _decode_metadata(payload[_PREFIX.size:offset], count, name)
+ columns = 7 if metadata["velocity"] else 4
+ if len(payload) != offset + columns * count * 8:
+ raise _invalid("numeric payload length does not match the header")
+ arrays = []
+ for _ in range(4):
+ arrays.append(np.frombuffer(payload, dtype="<f8", count=count, offset=offset)
+ .astype(float, copy=True))
+ offset += count * 8
+ velocity = None
+ if metadata["velocity"]:
+ velocity = np.frombuffer(payload, dtype="<f8", count=3 * count, offset=offset)
+ velocity = velocity.reshape(count, 3).astype(float, copy=True)
+ try:
+ track = Track(name, *arrays, metadata["typecode"], velocity)
+ except (ValueError, TypeError, OverflowError, FloatingPointError) as error:
+ raise _invalid("invalid trajectory values") from error
+ track.altitude_basis = [label for label, length in metadata["altitude_basis"]
+ for _ in range(length)]
+ track.aircraft_class = metadata["aircraft_class"]
+ track.velocity_inferred_components = {
+ key: metadata["velocity_inferred_components"][key] for key in _COMPONENTS}
+ return track
diff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.py
index 54f689f..2170b72 100644
--- a/src/dopplergeist/search.py
+++ b/src/dopplergeist/search.py
@@ -586,7 +586,7 @@ def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,
outcomes = [None] * len(starts)
active = list(range(len(starts)))
rejected_count = floor_count = 0
- prefixes = [0.0] * len(starts)
+ prefixes = np.zeros(len(starts), dtype=np.float64)
limits = [3 * _frequency_scale(event) for event in events] if score_floor is not None else []
bound_enabled = bool(limits) and all(np.isfinite(value) and value > 0 for value in limits)
batches = iter(acoustic_batches)
@@ -597,16 +597,17 @@ def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,
if use_dual and batch is not None:
rejected, upper = ridge_trial_rejections(measurement, event, *batch,
score_floor=score_floor, group_count=len(events))
- eligible = []
- for trial in active:
- if rejected[trial]:
- rejected_count += 1
- outcomes[trial] = (None, -np.inf, None)
- elif np.isfinite(upper[trial]):
- floor_count += 1
- outcomes[trial] = (None, float(upper[trial]), None)
- else:
- eligible.append(trial)
+ indices = np.asarray(active, dtype=np.intp)
+ rejected = rejected[indices]
+ upper = upper[indices]
+ finite_upper = ~rejected & np.isfinite(upper)
+ rejected_count += int(np.count_nonzero(rejected))
+ floor_count += int(np.count_nonzero(finite_upper))
+ for trial in indices[rejected].tolist():
+ outcomes[trial] = (None, -np.inf, None)
+ for trial, bound in zip(indices[finite_upper].tolist(), upper[finite_upper].tolist()):
+ outcomes[trial] = (None, float(bound), None)
+ eligible = indices[~(rejected | finite_upper)].tolist()
else:
eligible = active
fitted = fitted_ceilings = None
@@ -619,6 +620,38 @@ def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,
*arguments, limits[index], legacy_arithmetic=legacy)
else:
fitted = first_event_fits(*arguments, legacy_arithmetic=legacy)
+ if fitted_ceilings is not None:
+ # Only the unchanged core validator supplies this complete vector.
+ # Custom hooks and numerical policies keep the scalar path below.
+ for trial, fit in zip(eligible, fitted):
+ diagnostics[trial].append(fit)
+ indices = np.asarray(eligible, dtype=np.intp)
+ contributions = np.asarray(fitted_ceilings, dtype=np.float64)
+ rejected = contributions == -np.inf
+ for trial in indices[rejected].tolist():
+ outcomes[trial] = (None, -np.inf, None)
+ indices = indices[~rejected]
+ prefixes[indices] = np.nextafter(prefixes[indices] + contributions[~rejected], np.inf)
+ bounds = prefixes[indices]
+ # From a nonnegative prefix, each remaining +1/upward-rounding
+ # step leaves a bound at least as large as its step count. The
+ # final cap is also >= remaining, so none can fall below this
+ # floor. Check prefixes explicitly to preserve earlier custom fits.
+ if score_floor <= remaining and np.all(bounds >= 0.0):
+ active = indices.tolist()
+ continue
+ # Keep each addition and upward rounding in the scalar order. In
+ # particular, summing losses first or rounding only once is unsafe.
+ for _ in range(remaining):
+ bounds = np.nextafter(bounds + 1.0, np.inf)
+ # Python min(maximum, NaN) retains maximum, including a NaN prefix
+ # supplied by an earlier custom fitter before core validation resumes.
+ np.fmin(float(len(events)), bounds, out=bounds)
+ below_floor = bounds < score_floor
+ for trial, bound in zip(indices[below_floor].tolist(), bounds[below_floor].tolist()):
+ outcomes[trial] = (None, bound, None)
+ active = indices[~below_floor].tolist()
+ continue
next_active = []
for fit_index, trial in enumerate(eligible):
try:
@@ -644,8 +677,8 @@ def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,
# Reuse the already validated prefix. Add unperformed
# groups one at a time with the identical outward rounding;
# capping a prefix or summing losses first changes the bound.
- prefixes[trial] = math.nextafter(prefixes[trial] + contribution, math.inf)
- bound = prefixes[trial]
+ prefixes[trial] = math.nextafter(float(prefixes[trial]) + contribution, math.inf)
+ bound = float(prefixes[trial])
for _ in range(remaining):
bound = math.nextafter(bound + 1.0, math.inf)
bound = min(float(len(events)), bound)
diff --git a/tests/test_real_flight_payload.py b/tests/test_real_flight_payload.py
new file mode 100644
index 0000000..6b24e3f
--- /dev/null
+++ b/tests/test_real_flight_payload.py
@@ -0,0 +1,126 @@
+"""Prepared payload parity and corruption checks on archived observations."""
+import json
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from dopplergeist import flight_payload
+from dopplergeist.data_access import DataUnavailable, _frame_track
+
+
[email protected](scope="module")
+def observed_tracks():
+ root = Path(__file__).resolve().parents[1]
+ tracks = []
+ for source in sorted((root / "research/trajectory-motion/frozen").glob("fresh-*.json")):
+ frame = pd.DataFrame(json.loads(source.read_text())).sort_values("time_s")
+ for number, (_, rows) in enumerate(frame.groupby((frame.time_s.diff() > 30.).cumsum())):
+ if len(rows) >= 3:
+ tracks.append(_frame_track(f"{rows.flight_id.iloc[0]}:{number}", rows))
+ assert tracks
+ return tracks
+
+
+def assert_exact_track(actual, expected):
+ assert actual.id == expected.id and actual.typecode == expected.typecode
+ for name in ("time", "lat", "lon", "altitude", "velocity", "xyz", "relative_time",
+ "_xyz_min", "_xyz_max", "_speed_upper", "_speed_peak"):
+ left, right = getattr(actual, name), getattr(expected, name)
+ if right is None:
+ assert left is None
+ else:
+ assert left.dtype == right.dtype and left.shape == right.shape, name
+ assert left.tobytes() == right.tobytes(), name
+ for name in ("spline", "derivative"):
+ left, right = getattr(actual, name), getattr(expected, name)
+ assert left.axis == right.axis and left.extrapolate == right.extrapolate
+ for field in ("x", "c"):
+ assert getattr(left, field).tobytes() == getattr(right, field).tobytes()
+ for name in ("origin", "_max_sample_gap", "altitude_basis", "aircraft_class",
+ "velocity_inferred_components"):
+ assert getattr(actual, name) == getattr(expected, name), name
+
+
+def test_observed_tracks_round_trip_exactly(observed_tracks):
+ for expected in observed_tracks:
+ payload = flight_payload.encode_track(expected)
+ actual = flight_payload.decode_track(expected.id, payload)
+ assert_exact_track(actual, expected)
+ for name in ("time", "lat", "lon", "altitude", "velocity"):
+ array = getattr(actual, name)
+ if array is not None:
+ assert array.flags.writeable and array.flags.c_contiguous
+
+
+def test_missing_velocity_uses_same_recorded_positions(observed_tracks):
+ original = observed_tracks[0]
+ frame = pd.DataFrame({"time_s": original.time, "lat": original.lat,
+ "lon": original.lon, "alt_m": original.altitude})
+ expected = _frame_track(original.id, frame)
+ assert expected.velocity is None
+ actual = flight_payload.decode_track(expected.id, flight_payload.encode_track(expected))
+ assert_exact_track(actual, expected)
+
+
+def test_decoded_tracks_are_independently_mutable(observed_tracks):
+ expected = observed_tracks[0]
+ payload = flight_payload.encode_track(expected)
+ changed = flight_payload.decode_track(expected.id, payload)
+ untouched = flight_payload.decode_track(expected.id, payload)
+ changed.time[0] = np.nextafter(changed.time[0], np.inf)
+ changed.altitude[0] += 1.
+ if changed.velocity is not None:
+ changed.velocity[0, 0] += 1.
+ changed.altitude_basis[0] = "changed"
+ changed.velocity_inferred_components["east_mps"] += 1
+ assert_exact_track(untouched, expected)
+ assert_exact_track(flight_payload.decode_track(expected.id, payload), expected)
+
+
+def _header_variant(payload, mutate):
+ magic, version, size, count = flight_payload._PREFIX.unpack_from(payload)
+ first = flight_payload._PREFIX.size
+ metadata = json.loads(payload[first:first+size])
+ mutate(metadata)
+ header = json.dumps(metadata, separators=(",", ":")).encode()
+ return (flight_payload._PREFIX.pack(magic, version, len(header), count)
+ + header + payload[first+size:])
+
+
+def test_invalid_sizes_types_and_metadata_fail(observed_tracks):
+ track = observed_tracks[0]
+ payload = flight_payload.encode_track(track)
+ magic, version, size, count = flight_payload._PREFIX.unpack_from(payload)
+ invalid = [b"", payload[:-1], payload+b"x", b"x"+payload[1:], bytearray(payload),
+ flight_payload._PREFIX.pack(magic, version+1, size, count)+payload[flight_payload._PREFIX.size:],
+ flight_payload._PREFIX.pack(magic, version, flight_payload._MAX_HEADER_BYTES+1, count),
+ flight_payload._PREFIX.pack(magic, version, size, flight_payload._MAX_SAMPLES+1)]
+ for change in (
+ lambda meta: meta.update(extra=True),
+ lambda meta: meta.update(velocity=1),
+ lambda meta: meta.update(aircraft_class=None),
+ lambda meta: meta["velocity_inferred_components"].update(east_mps=True),
+ lambda meta: meta.update(altitude_basis=[["ellipsoidal", count+1]]),
+ lambda meta: meta.update(altitude_basis=[["ellipsoidal", -1]]),
+ ):
+ invalid.append(_header_variant(payload, change))
+ for damaged in invalid:
+ with pytest.raises(DataUnavailable):
+ flight_payload.decode_track(track.id, damaged)
+ with pytest.raises(DataUnavailable):
+ flight_payload.decode_track(track.id+"-other", payload)
+ with pytest.raises(DataUnavailable, match="sample budget exceeded"):
+ flight_payload.decode_track(track.id, payload, max_samples=count-1)
+ assert_exact_track(flight_payload.decode_track(track.id, payload, max_samples=count), track)
+
+
+def test_nonfinite_numeric_payload_is_rejected(observed_tracks):
+ track = observed_tracks[0]
+ payload = flight_payload.encode_track(track)
+ _, _, size, _ = flight_payload._PREFIX.unpack_from(payload)
+ first = flight_payload._PREFIX.size+size
+ damaged = payload[:first] + np.array([np.nan], dtype="<f8").tobytes() + payload[first+8:]
+ with pytest.raises(DataUnavailable, match="invalid trajectory values"):
+ flight_payload.decode_track(track.id, damaged)
diff --git a/tests/test_real_native_convergence.py b/tests/test_real_native_convergence.py
new file mode 100644
index 0000000..bf48420
--- /dev/null
+++ b/tests/test_real_native_convergence.py
@@ -0,0 +1,67 @@
+"""Native convergence state stays exact across bounded allocation paths."""
+import json
+from pathlib import Path
+from unittest.mock import patch
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from dopplergeist import geometry, native_acoustic
+from dopplergeist.tracks import _segment_velocity
+
+
+pytestmark = pytest.mark.skipif(native_acoustic._native is None,
+ reason="optional acoustic extension is not built")
+
+
[email protected](scope="module")
+def observed_query():
+ root = Path(__file__).resolve().parents[1]
+ for source in sorted((root / "research/trajectory-motion/frozen").glob("fresh-*.json")):
+ frame = pd.DataFrame(json.loads(source.read_text())).sort_values("time_s")
+ for _, rows in frame.groupby((frame.time_s.diff() > 30.).cumsum()):
+ if len(rows) < 3:
+ continue
+ velocity, _ = _segment_velocity(rows)
+ track = geometry.Track(str(rows.flight_id.iloc[0]), rows.time_s.to_numpy(),
+ rows.lat.to_numpy(), rows.lon.to_numpy(), rows.alt_m.to_numpy(),
+ str(rows.typecode.iloc[0]), velocity)
+ try:
+ track.validate_motion_support()
+ except geometry.AcousticSupportError:
+ continue
+ observer = geometry.ecef(track.lat[0], track.lon[0], 0.)
+ receive = track.time + np.linalg.norm(track.xyz-observer, axis=1)/343.
+ return track, observer, receive
+ pytest.fail("No supported archived aircraft observations")
+
+
[email protected]("count", (1, 63, 64, 65, 257))
+def test_observed_requests_preserve_bytes_at_state_allocation_boundary(observed_query, count):
+ track, observer, receive = observed_query
+ query = np.resize(receive, count)
+ with patch.object(native_acoustic, "_native", None):
+ expected = track.acoustic(query, observer)
+ before = native_acoustic._native.successful_calls()
+ actual = track.acoustic(query, observer)
+ assert native_acoustic._native.successful_calls() == before + 1
+ for left, right in zip(actual, expected):
+ assert left.dtype == right.dtype and left.shape == right.shape
+ assert left.tobytes() == right.tobytes()
+
+
+def test_repeated_observed_requests_above_state_budget_stay_native(observed_query):
+ track, observer, receive = observed_query
+ with patch.object(native_acoustic, "_native", None):
+ expected = track.acoustic(receive, observer)
+ # Repeat identical recorded requests to cross the 2 MiB state budget.
+ # Their global maximum correction and iteration count are unchanged.
+ repetitions = (2 * 1024 * 1024 // np.dtype(np.uint16).itemsize) // len(receive) + 1
+ query = np.tile(receive, (repetitions, 1))
+ before = native_acoustic._native.successful_calls()
+ actual = track.acoustic(query, observer)
+ assert native_acoustic._native.successful_calls() == before + 1
+ for left, right in zip(actual, expected):
+ expected_bits = np.broadcast_to(right.view(np.uint64), query.shape)
+ np.testing.assert_array_equal(left.view(np.uint64), expected_bits)
diff --git a/tests/test_real_prepared_index.py b/tests/test_real_prepared_index.py
new file mode 100644
index 0000000..420fcbb
--- /dev/null
+++ b/tests/test_real_prepared_index.py
@@ -0,0 +1,215 @@
+"""Prepared index checks using archived, unmodified ADS-B observations."""
+import gzip
+import hashlib
+import json
+from contextlib import contextmanager
+from pathlib import Path
+import sqlite3
+
+import pandas as pd
+import pytest
+
+from dopplergeist import data_access
+from dopplergeist.data_access import DataUnavailable, FlightIndex
+
+
+def observed_index(tmp_path):
+ source = Path(__file__).resolve().parents[1] / "research/trajectory-motion/frozen/canonical-observations.json"
+ frame = pd.DataFrame(json.loads(source.read_text()))
+ coverage = {"bounds": [-180., -90., 180., 90.],
+ "time_range_s": [float(frame.time_s.min()), float(frame.time_s.max())],
+ "all_aircraft": False, "source_note": "Archived real observations; incomplete excerpt"}
+ index = FlightIndex.build(tmp_path / "observed.sqlite", [frame], sources=[], coverage=coverage)
+ assert index._prepared_tracks
+ assert index.report["prepared_tracks"]["tracks"] > 0
+ return index, coverage
+
+
+def signature(track):
+ arrays = []
+ for name in ("time", "lat", "lon", "altitude", "velocity", "xyz", "relative_time",
+ "_xyz_min", "_xyz_max", "_speed_upper", "_speed_peak"):
+ value = getattr(track, name)
+ arrays.append(None if value is None else (value.dtype.str, value.shape, value.tobytes()))
+ for name in ("spline", "derivative"):
+ value = getattr(track, name)
+ arrays.extend((part.dtype.str, part.shape, part.tobytes()) for part in (value.c, value.x))
+ return (track.id, arrays, track.typecode, track.aircraft_class, track.altitude_basis,
+ track.velocity_inferred_components, track.origin, track._max_sample_gap)
+
+
+def queried(index, coverage):
+ return list(index.query(coverage["bounds"], coverage["time_range_s"]))
+
+
+def test_prepared_query_and_lookup_match_original_bytes(observed_index, monkeypatch):
+ index, coverage = observed_index
+ legacy = FlightIndex(index.path, track_cache_bytes=0)
+ legacy._prepared_tracks = False
+ expected = queried(legacy, coverage)
+ fast = FlightIndex(index.path, track_cache_bytes=0)
+ calls = []
+ from dopplergeist import flight_payload
+ original = flight_payload.decode_track
+
+ def counted(*args, **kwargs):
+ calls.append(args[0])
+ return original(*args, **kwargs)
+
+ monkeypatch.setattr(flight_payload, "decode_track", counted)
+ actual = queried(fast, coverage)
+ assert [signature(track) for track in actual] == [signature(track) for track in expected]
+ assert len(calls) == len(expected)
+ looked_up = list(fast.lookup([track.id for track in expected]))
+ assert [signature(track) for track in looked_up] == [signature(track) for track in expected]
+ assert len(calls) == 2 * len(expected)
+ actual[0].time[0] += 1
+ actual[0].spline.c[0, 0, 0] += 1
+ actual[0].altitude_basis[0] = "caller modification"
+ assert signature(next(fast.lookup([expected[0].id]))) == signature(expected[0])
+
+
+def test_prepared_index_copy_preserves_source_and_input_identity(observed_index, tmp_path):
+ index, coverage = observed_index
+ before = data_access.sha256(index.path)
+ destination = tmp_path / "prepared.sqlite"
+ prepared = index.prepare(destination)
+ assert data_access.sha256(index.path) == before
+ assert prepared.report["input_hash"] == index.report["input_hash"]
+ assert prepared.coverage_for(coverage["bounds"], coverage["time_range_s"]) == index.coverage_for(
+ coverage["bounds"], coverage["time_range_s"])
+ assert [signature(track) for track in queried(prepared, coverage)] == [
+ signature(track) for track in queried(index, coverage)]
+ with pytest.raises(FileExistsError):
+ index.prepare(destination)
+
+
+def test_prepare_and_build_coordinate_the_destination(observed_index, tmp_path, monkeypatch):
+ index, coverage = observed_index
+ original_lock, locks = data_access.cache_lock, []
+
+ @contextmanager
+ def recorded_lock(path):
+ locks.append(Path(path))
+ with original_lock(path):
+ yield
+
+ monkeypatch.setattr(data_access, "cache_lock", recorded_lock)
+ before = data_access.sha256(index.path)
+ destination = tmp_path / "shared-destination.sqlite"
+ prepared = index.prepare(destination)
+ reused = FlightIndex.build(destination, [], sources=[], coverage=coverage)
+ assert len(locks) == 2 and locks[0] == locks[1]
+ assert prepared.report["input_hash"] == reused.report["input_hash"] == index.report["input_hash"]
+ assert data_access.sha256(index.path) == before
+
+
+def test_prepare_uses_metadata_from_the_copied_snapshot(observed_index, tmp_path):
+ index, _ = observed_index
+ with sqlite3.connect(index.path) as db:
+ original = json.loads(db.execute("SELECT value FROM metadata").fetchone()[0])
+ original["snapshot_note"] = "Same archived observations; updated report note"
+ db.execute("UPDATE metadata SET value=?", (json.dumps(original),))
+ before = data_access.sha256(index.path)
+ index.report["coverage"]["all_aircraft"] = True
+ index.report["input_hash"] = "caller-modified-report"
+ prepared = index.prepare(tmp_path / "snapshot.sqlite")
+ assert {key: value for key, value in prepared.report.items() if key != "prepared_tracks"} == {
+ key: value for key, value in original.items() if key != "prepared_tracks"}
+ assert data_access.sha256(index.path) == before
+
+
+def test_prepared_reader_falls_back_after_implementation_change(observed_index, monkeypatch):
+ index, coverage = observed_index
+ expected = [signature(track) for track in queried(index, coverage)]
+ monkeypatch.setattr(data_access, "_prepared_track_identity", lambda: "other-normalization-build")
+ reader = FlightIndex(index.path, track_cache_bytes=0)
+ assert reader._prepared_tracks is False
+ assert [signature(track) for track in queried(reader, coverage)] == expected
+
+
+def test_reader_rechecks_changed_preparation_metadata(observed_index, monkeypatch):
+ index, coverage = observed_index
+ expected = queried(index, coverage)
+ reader = FlightIndex(index.path, track_cache_bytes=0)
+ with sqlite3.connect(index.path) as db:
+ report = json.loads(db.execute("SELECT value FROM metadata").fetchone()[0])
+ report["prepared_tracks"]["builder"] = "other-normalization-build"
+ db.execute("UPDATE metadata SET value=?", (json.dumps(report),))
+ from dopplergeist import flight_payload
+
+ def unavailable(*args, **kwargs):
+ pytest.fail("Stale preparation metadata reached the numeric decoder")
+
+ monkeypatch.setattr(flight_payload, "decode_track", unavailable)
+ assert [signature(track) for track in queried(reader, coverage)] == [
+ signature(track) for track in expected]
+ assert [signature(track) for track in reader.lookup([track.id for track in expected])] == [
+ signature(track) for track in expected]
+
+
[email protected]("damaged,repair_digest", [
+ (b"invalid prepared representation", False),
+ (b"invalid prepared representation", True),
+ ("wrong SQLite storage type", False),
+ (17, False),
+])
+def test_damaged_optional_payload_uses_verified_original(observed_index, damaged, repair_digest):
+ index, coverage = observed_index
+ expected = [signature(track) for track in queried(index, coverage)]
+ with sqlite3.connect(index.path) as db:
+ if repair_digest:
+ db.execute("UPDATE prepared_tracks SET payload=?,payload_hash=?",
+ (damaged, hashlib.sha256(damaged).hexdigest()))
+ else:
+ db.execute("UPDATE prepared_tracks SET payload=?", (damaged,))
+ reader = FlightIndex(index.path, track_cache_bytes=0)
+ assert [signature(track) for track in queried(reader, coverage)] == expected
+
+
[email protected]("replacement", [
+ None,
+ "CREATE TABLE prepared_tracks(id INTEGER PRIMARY KEY, payload BLOB)",
+ "CREATE TABLE prepared_tracks(id INTEGER, source_hash TEXT, payload BLOB, payload_hash TEXT)",
+])
[email protected]("opened_before_damage", [False, True])
+def test_missing_or_malformed_optional_table_uses_original(observed_index, replacement, opened_before_damage):
+ index, coverage = observed_index
+ expected = queried(index, coverage)
+ reader = FlightIndex(index.path, track_cache_bytes=0) if opened_before_damage else None
+ with sqlite3.connect(index.path) as db:
+ db.execute("DROP TABLE prepared_tracks")
+ if replacement is not None:
+ db.execute(replacement)
+ if reader is None:
+ reader = FlightIndex(index.path, track_cache_bytes=0)
+ assert reader._prepared_tracks is False
+ assert [signature(track) for track in queried(reader, coverage)] == [
+ signature(track) for track in expected]
+ assert [signature(track) for track in reader.lookup([track.id for track in expected])] == [
+ signature(track) for track in expected]
+
+
+def test_prepared_reader_does_not_hide_changed_original_payload(observed_index):
+ index, coverage = observed_index
+ first = queried(index, coverage)[0]
+ with sqlite3.connect(index.path) as db:
+ old = db.execute("SELECT payload FROM segments WHERE name=?", (first.id,)).fetchone()[0]
+ changed = gzip.compress(gzip.decompress(old), mtime=1)
+ db.execute("UPDATE segments SET payload=?,payload_hash=? WHERE name=?",
+ (changed, hashlib.sha256(changed).hexdigest(), first.id))
+ reader = FlightIndex(index.path, track_cache_bytes=0)
+ assert signature(next(reader.lookup([first.id]))) == signature(first)
+ with sqlite3.connect(index.path) as db:
+ db.execute("UPDATE segments SET payload=? WHERE name=?", (b"damaged original", first.id))
+ with pytest.raises(DataUnavailable, match="Corrupt flight payload"):
+ list(reader.lookup([first.id]))
+
+
+def test_prepared_lookup_preserves_sample_budget(observed_index):
+ index, coverage = observed_index
+ first = queried(index, coverage)[0]
+ reader = FlightIndex(index.path, track_cache_bytes=0)
+ with pytest.raises(DataUnavailable, match="Flight lookup sample budget exceeded"):
+ list(reader.lookup([first.id], max_samples=len(first.time) - 1))
diff --git a/tests/test_real_track_cache.py b/tests/test_real_track_cache.py
index 5060349..2743f68 100644
--- a/tests/test_real_track_cache.py
+++ b/tests/test_real_track_cache.py
@@ -20,6 +20,25 @@ def snapshot_size(track):
return len(pickle.dumps(track, protocol=pickle.HIGHEST_PROTOCOL))
+def record_track_setup(monkeypatch, prepared):
+ """Count the selected decoder without switching paths after sizing it."""
+ if prepared:
+ from dopplergeist import flight_payload
+ module, name = flight_payload, "decode_track"
+ else:
+ module, name = data_access, "_frame_track"
+ original, calls = getattr(module, name), []
+
+ def decode(identifier, *args, **kwargs):
+ calls.append(identifier)
+ return original(identifier, *args, **kwargs)
+
+ # A custom _frame_track hook intentionally selects the original reader.
+ # Prepared tracks can have different pickle sizes from shared label strings.
+ monkeypatch.setattr(module, name, decode)
+ return calls
+
+
@pytest.fixture
def real_index(tmp_path):
source = Path(__file__).resolve().parents[1] / "research/trajectory-motion/frozen/canonical-observations.json"
@@ -118,55 +137,58 @@ def test_real_cache_observes_replaced_payload(real_index, monkeypatch):
assert calls == [expected.id]
-def test_real_cache_evicts_and_skips_oversized_tracks(real_index, monkeypatch):
[email protected]("prepared", (False, True), ids=("original-json", "prepared-numeric"))
+def test_real_cache_evicts_and_skips_oversized_tracks(real_index, monkeypatch, prepared):
path, coverage = real_index
+ calls = record_track_setup(monkeypatch, prepared)
tracks = list(FlightIndex(path, track_cache_bytes=0).query(coverage["bounds"], coverage["time_range_s"]))
+ assert calls == [track.id for track in tracks]
+ calls.clear()
first, second = tracks[:2]
- sizes = [snapshot_size(track) for track in (first, second)]
- cached = FlightIndex(path, track_cache_bytes=max(sizes))
- original, calls = data_access._frame_track, []
-
- def decode(name, frame):
- calls.append(name)
- return original(name, frame)
-
- monkeypatch.setattr(data_access, "_frame_track", decode)
+ sizes = {track.id: snapshot_size(track) for track in (first, second)}
+ cached = FlightIndex(path, track_cache_bytes=max(sizes.values()))
for track in (first, second, first):
assert_tracks_equal(list(cached.lookup([track.id])), [track])
assert len(cached._track_cache) == 1
- assert cached._track_cache_bytes <= max(sizes)
+ assert next(iter(cached._track_cache))[0] == track.id
+ assert cached._track_cache_bytes == sizes[track.id]
+ assert cached._track_cache_bytes == sum(len(value[1]) for value in cached._track_cache.values())
+ assert cached._track_cache_bytes <= cached._track_cache_limit
assert calls == [first.id, second.id, first.id]
- too_small = FlightIndex(path, track_cache_bytes=min(sizes) - 1)
- assert_tracks_equal(list(too_small.lookup([first.id])), [first])
+ too_small = FlightIndex(path, track_cache_bytes=min(sizes.values()) - 1)
+ for _ in range(2):
+ assert_tracks_equal(list(too_small.lookup([first.id])), [first])
+ assert calls == [first.id, second.id, first.id, first.id, first.id]
assert not too_small._track_cache
assert too_small._track_cache_bytes == 0
-def test_real_query_larger_than_cache_keeps_a_reusable_subset(real_index, monkeypatch):
[email protected]("prepared", (False, True), ids=("original-json", "prepared-numeric"))
+def test_real_query_larger_than_cache_keeps_a_reusable_subset(real_index, monkeypatch, prepared):
path, coverage = real_index
+ calls = record_track_setup(monkeypatch, prepared)
expected = list(FlightIndex(path, track_cache_bytes=0).query(coverage["bounds"], coverage["time_range_s"]))
+ assert calls == [track.id for track in expected]
+ calls.clear()
size = snapshot_size(expected[0])
cached = FlightIndex(path, track_cache_bytes=size)
- original, calls = data_access._frame_track, []
serialize, snapshots = data_access.pickle.dumps, []
- def decode(name, frame):
- calls.append(name)
- return original(name, frame)
-
def snapshot(track, **kwargs):
snapshots.append(track.id)
return serialize(track, **kwargs)
- monkeypatch.setattr(data_access, "_frame_track", decode)
monkeypatch.setattr(data_access.pickle, "dumps", snapshot)
assert_tracks_equal(list(cached.query(coverage["bounds"], coverage["time_range_s"])), expected)
assert len(calls) == len(expected)
assert len(cached._track_cache) == 1
assert next(iter(cached._track_cache))[0] == expected[0].id
+ assert cached._track_cache_bytes == size
+ assert cached._track_cache_bytes == sum(len(value[1]) for value in cached._track_cache.values())
assert_tracks_equal(list(cached.query(coverage["bounds"], coverage["time_range_s"])), expected)
assert len(calls) == 2 * len(expected) - 1
assert cached._track_cache_bytes == size
+ assert cached._track_cache_bytes == sum(len(value[1]) for value in cached._track_cache.values())
assert len(snapshots) < len(calls) # Admission failure avoids work on later misses.
diff --git a/tests/test_real_trial_vectors.py b/tests/test_real_trial_vectors.py
new file mode 100644
index 0000000..d832508
--- /dev/null
+++ b/tests/test_real_trial_vectors.py
@@ -0,0 +1,162 @@
+"""Clock-vector screens preserve scalar decisions on recorded associations."""
+import json
+import os
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from dopplergeist import acoustic_screen, batch_ridge, matching, search
+from dopplergeist.data_access import FlightIndex
+from dopplergeist.geometry import AcousticSupportError, ecef
+
+
[email protected](scope="module")
+def recorded_associations():
+ source = os.environ.get("DOPPLERGEIST_REAL_CPA_FIXTURE")
+ if not source:
+ pytest.skip("set DOPPLERGEIST_REAL_CPA_FIXTURE to an actual recorded-data diagnostic")
+ fixture = json.loads(Path(source).read_text())
+ events, measurements = matching.prepare_events(fixture["events"])
+ candidates, _ = matching.association_candidates(
+ pd.DataFrame(fixture["cpa_rows"]), events,
+ max_candidates=fixture.get("budget", {}).get("max_candidates_per_tile", 20000),
+ anchor_window=fixture.get("recording_start_window_s"))
+ grouped = {}
+ for candidate in candidates:
+ count = len(candidate["event_indices"])
+ if count >= 2:
+ grouped.setdefault(count, []).append(candidate)
+ assert grouped, "Recorded fixture needs a multiple-group association"
+ sizes = sorted(grouped)
+ selected = [grouped[size][position]
+ for size in sorted({sizes[0], sizes[len(sizes) // 2], sizes[-1]})
+ for position in (0, -1)]
+ identifiers = sorted({entry["flight_id"] for candidate in selected for entry in candidate["passes"]})
+ tracks = {track.id: track for track in FlightIndex(fixture["real_index"]).lookup(identifiers)}
+ point = fixture["point"]
+ observer = ecef(point["lat"], point["lon"], point["elev_m"])
+ result = []
+ for candidate in selected:
+ indices = candidate["event_indices"]
+ chosen_tracks = [tracks[entry["flight_id"]] for entry in candidate["passes"]]
+ chosen_events = [events[index] for index in indices]
+ chosen_measurements = [measurements[index] for index in indices]
+ lower, upper = candidate["recording_start_bounds_s"]
+ starts = np.linspace(lower - 5.0, upper + 5.0, 21)
+ batches = matching.batch_acoustic_trials(chosen_tracks, chosen_measurements, observer, starts)
+ result.append((chosen_tracks, chosen_events, chosen_measurements, observer, starts, batches))
+ return result
+
+
+def scalar_trials(case, floor, use_dual):
+ tracks, events, measurements, observer, starts, batches = case
+ flags = [None if batch is None or not use_dual else
+ acoustic_screen.ridge_trial_rejections(measurement, event, *batch,
+ score_floor=floor, group_count=len(events))
+ for measurement, event, batch in zip(measurements, events, batches)]
+ rejected_count = floor_count = 0
+ outcomes = []
+ for trial, start in enumerate(starts):
+ def acoustic_at(index):
+ batch = batches[index]
+ return None if batch is None else (batch[0][trial], batch[1][trial])
+
+ def dual_at(index):
+ nonlocal rejected_count, floor_count
+ if flags[index] is None:
+ return None
+ rejected, upper = flags[index]
+ if rejected[trial]:
+ rejected_count += 1
+ return -np.inf
+ if np.isfinite(upper[trial]):
+ floor_count += 1
+ return float(upper[trial])
+ return None
+
+ try:
+ result, upper = search._screen_association_trial(
+ tracks, events, measurements, observer, start, 343.0,
+ acoustic_at, floor, dual_at if use_dual else None)
+ outcomes.append((result, upper, None))
+ except AcousticSupportError as error:
+ outcomes.append((None, None, error))
+ return outcomes, rejected_count, floor_count
+
+
+def encoded(value):
+ def default(item):
+ if isinstance(item, AcousticSupportError):
+ return {"error": str(item), "evidence": vars(item)}
+ if isinstance(item, np.ndarray):
+ return item.tolist()
+ if isinstance(item, np.generic):
+ return item.item()
+ raise TypeError(type(item).__name__)
+ return json.dumps(value, sort_keys=True, default=default)
+
+
[email protected]("use_dual", [False, True])
+def test_recorded_vectors_keep_scalar_decisions_at_adjacent_float_floors(
+ recorded_associations, monkeypatch, use_dual):
+ original = batch_ridge._first_event_fits_with_ceilings
+ vector_batches = []
+ boundary_cases = []
+
+ def counted(*args, **kwargs):
+ result = original(*args, **kwargs)
+ if result[1] is not None:
+ vector_batches.append(1)
+ return result
+
+ monkeypatch.setattr(batch_ridge, "_first_event_fits_with_ceilings", counted)
+
+ @matching._cache_ridge_fits
+ def run():
+ for case in recorded_associations:
+ tracks, events, measurements, observer, starts, batches = case
+ first_bounds = scalar_trials(case, float(len(events)), False)[0]
+ finite = sorted(upper for _, upper, _ in first_bounds
+ if upper is not None and np.isfinite(upper))
+ # Exercise the shortcut's remaining-group threshold on both sides
+ # as well as actual fitted ceilings below. Observations stay fixed.
+ remaining = float(len(events) - 1)
+ floors = {0.0, float(len(events)), remaining,
+ float(np.nextafter(remaining, -np.inf)),
+ float(np.nextafter(remaining, np.inf))}
+ if finite:
+ boundary = finite[len(finite) // 2]
+ boundary_cases.append(boundary)
+ floors.update((float(np.nextafter(boundary, -np.inf)), boundary,
+ float(np.nextafter(boundary, np.inf))))
+ for floor in sorted(floors):
+ expected = scalar_trials(case, floor, use_dual)
+ actual = search._batched_ridge_trials(
+ tracks, events, measurements, observer, starts, 343.0,
+ iter(batches), floor, use_dual)
+ assert encoded(actual) == encoded(expected), (len(events), floor, use_dual)
+ run()
+ assert vector_batches, "Recorded cases must exercise the canonical ceiling vector"
+ assert boundary_cases, "Recorded cases must exercise adjacent-float retention boundaries"
+
+
+def test_recorded_vectors_keep_noncanonical_validation_fallback(recorded_associations, monkeypatch):
+ original = search._ridge_trial_classification
+
+ def custom(*args):
+ return original(*args)
+
+ monkeypatch.setattr(search, "_ridge_trial_classification", custom)
+
+ @matching._cache_ridge_fits
+ def run():
+ for case in recorded_associations:
+ tracks, events, measurements, observer, starts, batches = case
+ expected = scalar_trials(case, 0.0, False)
+ actual = search._batched_ridge_trials(
+ tracks, events, measurements, observer, starts, 343.0,
+ iter(batches), 0.0, False)
+ assert encoded(actual) == encoded(expected)
+ run()