Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

perf: accelerate indexed track decoding and ridge evidence checks

Cinna Davisa82f4b02a5a4
diff --git a/.github/workflows/real-performance.yml b/.github/workflows/real-performance.yml
index 9893f56..11e9477 100644
--- a/.github/workflows/real-performance.yml
+++ b/.github/workflows/real-performance.yml
@@ -4,14 +4,14 @@ on:
inputs:
baseline:
description: Exact baseline commit for both real-input comparisons
- default: c37128bb08f12b8a68cf1f1cbf86adc2436f60fd
+ default: 27da08a878ac2ec90cd7e2c07f43765ceeea609b
required: true
regional:
description: Also compare the complete regional HTTP workflow
type: boolean
default: true
push:
- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103]
+ branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103]
paths:
- .github/workflows/real-performance.yml
- src/dopplergeist/**
@@ -38,7 +38,7 @@ env:
NUMEXPR_NUM_THREADS: '1'
CC: gcc
PYTHONPATH: src
- BASELINE_REF: ${{ inputs.baseline || 'c37128bb08f12b8a68cf1f1cbf86adc2436f60fd' }}
+ BASELINE_REF: ${{ inputs.baseline || '27da08a878ac2ec90cd7e2c07f43765ceeea609b' }}
jobs:
compare:
if: github.event_name != 'push' || github.ref != 'refs/heads/perf/real-pipeline-throughput-20261103'
@@ -57,6 +57,7 @@ jobs:
python - <<'PY'
from dopplergeist import matching, native_acoustic
assert callable(matching._native_fit_irls), "Native ridge extension did not build"
+ assert callable(matching._native_validate_ridge_events), "Native evidence validator did not build"
assert matching.RIDGE_FIT_BACKEND.startswith("native-c-")
assert native_acoustic._native is not None, "Native acoustic extension did not build"
assert native_acoustic.ACOUSTIC_BACKEND.startswith("native-c-")
@@ -83,6 +84,13 @@ jobs:
for width in 120 10; do
python scripts/benchmark_prepared_measurements.py prepare --discovery evidence/audio/w1-sensitivity-20261102/reference/discovery.json --index benchmark-inputs/reference-region.sqlite --clock-half-width-s "$width" --output "benchmark-inputs/point-$width.json"
done
+ python scripts/benchmark_prepared_measurements.py prepare \
+ --discovery evidence/audio/w1-sensitivity-20261102/reference/discovery.json \
+ --index benchmark-inputs/reference-region.sqlite \
+ --reference-report evidence/results/reference-local/report.json \
+ --full-day --max-tracks 2000 --max-samples 500000 --max-candidates 20000 \
+ --anchor-window-s 1788864888.7151732 1788865236.7214916 \
+ --include-track-query --output benchmark-inputs/dense-query.json
- name: Export real pipeline objective controls
env:
DOPPLERGEIST_RIDGE_BACKEND: python
@@ -93,7 +101,7 @@ jobs:
DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json
DOPPLERGEIST_REAL_REFERENCE_ROOT: benchmark-inputs/reference-checkout
DOPPLERGEIST_REAL_REFERENCE_DISCOVERY: evidence/audio/w1-sensitivity-20261102/reference/discovery.json
- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_native_ridge.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py
+ run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_native_ridge.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py
- name: Verify the explicit Python fallback on recorded arithmetic
env:
DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json
@@ -104,6 +112,7 @@ jobs:
run: python scripts/benchmark_real_cpa_rows.py --baseline-source "$RUNNER_TEMP/baseline/src" --output results/cpa-rows.json --repeats 5 --iterations 20
- name: Compare complete payloads and timings
run: |
+ benchmark_cpu=$(python -c 'import os; print(min(os.sched_getaffinity(0)))')
native_option=()
if [ -f src/dopplergeist/_native_ridge.c ]; then
native_option=(--require-current-native)
@@ -112,8 +121,13 @@ jobs:
native_option+=(--require-current-native-acoustic)
fi
for width in 10 120; do
- python scripts/benchmark_prepared_measurements.py compare --fixture "benchmark-inputs/point-$width.json" --baseline-source "$RUNNER_TEMP/baseline/src" --current-source src --repeats 2 --timeout-s 600 --output "results/performance-$width" "${native_option[@]}"
+ taskset -c "$benchmark_cpu" python scripts/benchmark_prepared_measurements.py compare --fixture "benchmark-inputs/point-$width.json" --baseline-source "$RUNNER_TEMP/baseline/src" --current-source src --repeats 3 --timeout-s 600 --output "results/performance-$width" "${native_option[@]}"
done
+ taskset -c "$benchmark_cpu" python scripts/benchmark_prepared_measurements.py compare \
+ --fixture benchmark-inputs/dense-query.json --include-track-query \
+ --baseline-source "$RUNNER_TEMP/baseline/src" --current-source src \
+ --repeats 3 --timeout-s 600 --output results/performance-dense-query \
+ "${native_option[@]}"
- name: Profile remaining point work outside the timing comparison
run: |
python -m cProfile -o results/current-point.prof scripts/benchmark_prepared_measurements.py _worker --source src --fixture benchmark-inputs/point-120.json --output results/current-point-profile --require-native --require-native-acoustic
diff --git a/scripts/benchmark_prepared_measurements.py b/scripts/benchmark_prepared_measurements.py
index cd7b44e..038c87a 100644
--- a/scripts/benchmark_prepared_measurements.py
+++ b/scripts/benchmark_prepared_measurements.py
@@ -44,6 +44,37 @@ def canonical(value):
allow_nan=True, default=json_scalar)
+def query_tracks(index, points, events, budget, scoring, bounds, time_range):
+ """Count actual index traversal while exercising the production overload path."""
+ from dopplergeist.tiled_search import _tracks_for_points
+
+ if "elev_m" not in points:
+ raise ValueError("The recorded receiver hypothesis must include its elevation")
+ operations = {"query_calls": 0, "decoded_track_yields": 0, "decoded_sample_yields": 0}
+ inventory = {}
+
+ class CountedIndex:
+ def query(self, query_bounds, query_time):
+ operations["query_calls"] += 1
+ for track in index.query(query_bounds, query_time):
+ operations["decoded_track_yields"] += 1
+ operations["decoded_sample_yields"] += len(track.time)
+ inventory[track.id] = len(track.time)
+ yield track
+
+ radius = max(event["slant_range_m"][1] for event in events)
+ tracks, points = _tracks_for_points(
+ [CountedIndex()], bounds, time_range, budget, points, None, radius,
+ scoring.get("c", 343.0))
+ accounting = {
+ "queried_track_ids": sorted(inventory),
+ "queried_trajectory_samples": sum(inventory.values()),
+ "selected_track_ids": [track.id for track in tracks],
+ "selected_trajectory_samples": sum(len(track.time) for track in tracks),
+ }
+ return tracks, points, operations, accounting
+
+
def prepare(args):
import pandas as pd
from dopplergeist.data_access import FlightIndex
@@ -69,15 +100,26 @@ def prepare(args):
budget = SearchBudget(max_tracks_per_tile=args.max_tracks,
max_track_samples_per_tile=args.max_samples,
max_candidates_per_tile=args.max_candidates, top_k=args.top_k)
- tracks, samples = [], 0
- for track in index.query(query_bounds, query_time):
- tracks.append(track)
- samples += len(track.time)
- if len(tracks) > args.max_tracks or samples > args.max_samples:
- raise ValueError("Declared trajectory budget exceeded; no input was silently truncated")
+ points = pd.DataFrame([point])
+ query_preparation = None
+ if args.include_track_query:
+ tracks, points, query_operations, query_accounting = query_tracks(
+ index, points, events, budget, {"mode": "robust_ridge"}, query_bounds, query_time)
+ samples = query_accounting["selected_trajectory_samples"]
+ query_preparation = {
+ "queried_trajectories": len(query_accounting["queried_track_ids"]),
+ "queried_trajectory_samples": query_accounting["queried_trajectory_samples"],
+ "operations": query_operations,
+ }
+ else:
+ tracks, samples = [], 0
+ for track in index.query(query_bounds, query_time):
+ tracks.append(track)
+ samples += len(track.time)
+ if len(tracks) > args.max_tracks or samples > args.max_samples:
+ raise ValueError("Declared trajectory budget exceeded; no input was silently truncated")
if not tracks:
raise ValueError("The declared real-data query returned no trajectories")
- points = pd.DataFrame([point])
observers = ecef(points.lat.to_numpy(), points.lon.to_numpy(), points.elev_m.to_numpy())
radius = max(e["slant_range_m"][1] for e in events)
tables, unsupported, row_count = [], [], 0
@@ -97,7 +139,9 @@ def prepare(args):
if not tables:
raise ValueError("The real-data point has no supported CPA rows")
fixture = {
- "scope": "Recorded-input computation comparison; source-informed receiver/clock, no accuracy claim",
+ "scope": ("Recorded-input computation comparison; source-informed receiver, no recording-clock prior or accuracy claim"
+ if args.full_day else
+ "Recorded-input computation comparison; source-informed receiver/clock, no accuracy claim"),
"discovery_source": str(args.discovery.resolve()),
"reference_report": str(args.reference_report.resolve()),
"real_index": str(args.index.resolve()), "index_input_hash": index.report["input_hash"],
@@ -113,9 +157,16 @@ def prepare(args):
"ridge_samples": sum(len(measurement[1]) for measurement in measurements),
"cpa_rows": row_count, "unsupported_cpa_tracks": len(unsupported)},
}
+ if args.anchor_window_s is not None:
+ fixture["anchor_window_s"] = args.anchor_window_s
+ fixture["scope"] += ". One association-anchor partition. Physical fit intervals remain unchanged."
+ if query_preparation is not None:
+ fixture["track_query_preparation"] = query_preparation
write_new(args.output, fixture)
print(json.dumps({"fixture": str(args.output), **fixture["counts"],
- "scope": fixture["scope"]}), flush=True)
+ "scope": fixture["scope"],
+ **({"track_query_preparation": query_preparation}
+ if query_preparation is not None else {})}), flush=True)
def worker(args):
@@ -131,26 +182,28 @@ def worker(args):
if Path(dopplergeist.__file__).resolve().parent != source / "dopplergeist":
raise RuntimeError("Comparison imported a different source tree")
native_fit = getattr(matching, "_native_fit_irls", None)
- backend = getattr(matching, "RIDGE_FIT_BACKEND", "python-irls-v1")
+ ridge_backend = getattr(matching, "RIDGE_FIT_BACKEND", "python-irls-v1")
acoustic_backend = getattr(geometry, "ACOUSTIC_BACKEND", "python-acoustic-v1")
try:
from dopplergeist.native_acoustic import _native as native_acoustic
except ImportError:
native_acoustic = None
if args.require_native and not callable(native_fit):
- raise RuntimeError("The source-tree native fitter is unavailable; build it in place before timing")
+ raise RuntimeError("The selected source's native ridge fitter is unavailable")
if args.require_native_acoustic and native_acoustic is None:
- raise RuntimeError("The source-tree native acoustic solver is unavailable; build it in place before timing")
+ raise RuntimeError("The selected source's native acoustic solver is unavailable")
fixture = json.loads(args.fixture.read_text())
index = FlightIndex(args.index or fixture["real_index"])
if index.report["input_hash"] != fixture["index_input_hash"]:
raise ValueError("The real trajectory index no longer matches the fixture")
budget = tiled_search.SearchBudget(**fixture["budget"])
loaded_at = time.perf_counter()
- tracks = list(index.lookup(fixture["track_ids"], max_tracks=budget.max_tracks_per_tile,
- max_samples=budget.max_track_samples_per_tile))
- if sorted(track.id for track in tracks) != sorted(fixture["track_ids"]):
- raise ValueError("The index is missing a recorded trajectory")
+ tracks = None
+ if not args.include_track_query:
+ tracks = list(index.lookup(fixture["track_ids"], max_tracks=budget.max_tracks_per_tile,
+ max_samples=budget.max_track_samples_per_tile))
+ if sorted(track.id for track in tracks) != sorted(fixture["track_ids"]):
+ raise ValueError("The index is missing a recorded trajectory")
events, measurements = matching.prepare_events(fixture["events"])
points = pd.DataFrame([fixture["point"]])
load_seconds = time.perf_counter() - loaded_at
@@ -164,6 +217,41 @@ def worker(args):
all_rows = []
original_fit, original_acoustic = matching._fit_ridge_uncached, Track.acoustic
original_score = tiled_search.score_candidate
+ phase_seconds, phase_patches = {}, []
+
+ def timed_phase(name, function):
+ def run(*values, **options):
+ began, cpu = time.perf_counter(), time.process_time()
+ try:
+ return function(*values, **options)
+ finally:
+ phase = phase_seconds.setdefault(name, {"wall_seconds": 0.0, "cpu_seconds": 0.0, "calls": 0})
+ phase["wall_seconds"] += time.perf_counter() - began
+ phase["cpu_seconds"] += time.process_time() - cpu
+ phase["calls"] += 1
+ return run
+
+ if args.stage_timing:
+ phase_targets = [
+ (Track, "outside_observer_radius", "distance_sieve"),
+ (matching, "association_candidates", "association_enumeration"),
+ (tiled_search, "merge_candidates", "retention"),
+ ]
+ if callable(getattr(tiled_search, "cpa_table", None)):
+ phase_targets.append((tiled_search, "cpa_table", "closest_approach"))
+ else:
+ prepared_cpa = getattr(tiled_search, "_PreparedCPATrack", None)
+ if prepared_cpa is not None:
+ phase_targets.extend([
+ (prepared_cpa, "__init__", "closest_approach_preparation"),
+ (prepared_cpa, "raw_rows", "closest_approach"),
+ ])
+ for owner, attribute, name in phase_targets:
+ function = getattr(owner, attribute, None)
+ if function is not None:
+ phase_patches.append((owner, attribute, function))
+ setattr(owner, attribute, timed_phase(name, function))
+ score_function = timed_phase("candidate_scoring", original_score) if args.stage_timing else original_score
def counted_fit(*values, **options):
operations["scalar_fit_calls"] += 1
@@ -183,7 +271,7 @@ def worker(args):
def captured_score(*values, **options):
operations["scored_candidates"] += 1
try:
- result = original_score(*values, **options)
+ result = score_function(*values, **options)
except AcousticSupportError as error:
all_rows.append({"support_failure": tiled_search._support_error(error)})
raise
@@ -191,23 +279,52 @@ def worker(args):
return result
matching._fit_ridge_uncached, Track.acoustic = counted_fit, counted_acoustic
+ tiled_search.score_candidate = captured_score
if callable(native_fit):
matching._native_fit_irls = counted_native
- tiled_search.score_candidate = captured_score
native_acoustic_before = native_acoustic.successful_calls() if native_acoustic is not None else 0
+ query_accounting, query_operations, execution_stages = None, None, {}
began, cpu = time.perf_counter(), time.process_time()
try:
+ if args.include_track_query:
+ query_began, query_cpu = time.perf_counter(), time.process_time()
+ tracks, points, query_operations, query_accounting = query_tracks(
+ index, points, events, budget, fixture["scoring"],
+ fixture["query_bounds"], fixture["query_time_s"])
+ execution_stages["track_query_and_distance_sieve"] = {
+ "wall_seconds": time.perf_counter() - query_began,
+ "cpu_seconds": time.process_time() - query_cpu,
+ }
+ if query_accounting["selected_track_ids"] != fixture["track_ids"]:
+ raise AssertionError("The real query changed the ordered selected-track inventory")
+ if query_accounting["selected_trajectory_samples"] != fixture["counts"]["trajectory_samples"]:
+ raise AssertionError("The real query changed the selected trajectory sample count")
+ prepared_query = fixture.get("track_query_preparation")
+ if prepared_query is not None:
+ if (len(query_accounting["queried_track_ids"]) != prepared_query["queried_trajectories"]
+ or query_accounting["queried_trajectory_samples"] != prepared_query["queried_trajectory_samples"]):
+ raise AssertionError("The real query changed the complete observed-track inventory")
+ scoring_began, scoring_cpu = time.perf_counter(), time.process_time()
+ partition = ({"anchor_window": fixture["anchor_window_s"]}
+ if "anchor_window_s" in fixture else {})
ranked, counts = tiled_search._score(
points, tracks, events, measurements, budget, fixture["scoring"], {},
- start_window=fixture["recording_start_window_s"], ledger=ledger, stage=stage)
+ start_window=fixture["recording_start_window_s"], ledger=ledger, stage=stage, **partition)
+ if args.include_track_query:
+ execution_stages["cpa_association_scoring_and_retention"] = {
+ "wall_seconds": time.perf_counter() - scoring_began,
+ "cpu_seconds": time.process_time() - scoring_cpu,
+ }
finally:
elapsed, cpu_seconds = time.perf_counter() - began, time.process_time() - cpu
- if native_acoustic is not None:
- operations["native_acoustic_successes"] = native_acoustic.successful_calls() - native_acoustic_before
matching._fit_ridge_uncached, Track.acoustic = original_fit, original_acoustic
+ tiled_search.score_candidate = original_score
if callable(native_fit):
matching._native_fit_irls = native_fit
- tiled_search.score_candidate = original_score
+ if native_acoustic is not None:
+ operations["native_acoustic_successes"] = native_acoustic.successful_calls() - native_acoustic_before
+ for owner, attribute, function in reversed(phase_patches):
+ setattr(owner, attribute, function)
expected_counts = fixture.get("expected_counts")
if expected_counts is not None:
actual_counts = {"cpa_rows": counts["cpa_rows"],
@@ -226,22 +343,37 @@ def worker(args):
if not operations["scored_candidates"]:
raise AssertionError("The benchmark did not score any real associations")
if args.require_native and not operations["native_fit_successes"]:
- raise AssertionError("The native fitter was available but no successful native fit was timed")
+ raise AssertionError("No successful native ridge fit occurred inside the timed scoring call")
if args.require_native_acoustic and not operations["native_acoustic_successes"]:
- raise AssertionError("The native acoustic solver was available but no successful native solve was timed")
+ raise AssertionError("No successful native acoustic solve occurred inside the timed scoring call")
with ledger.connect() as database:
retention = {name: database.execute(f"SELECT * FROM {name} ORDER BY 1,2").fetchall()
for name in ("records", "bounds", "stages")}
- write_new(args.output / "results.json", {"ranked": ranked, "counts": counts,
- "all_scored_candidates": all_rows, "retention": retention})
+ results = {"ranked": ranked, "counts": counts,
+ "all_scored_candidates": all_rows, "retention": retention}
+ if query_accounting is not None:
+ results["track_query_accounting"] = query_accounting
+ write_new(args.output / "results.json", results)
summary = {"wall_seconds": elapsed, "cpu_seconds": cpu_seconds,
"load_seconds_excluded": load_seconds, "operations": operations,
- "ridge_fit_backend": backend, "native_extension_available": callable(native_fit),
- "acoustic_backend": acoustic_backend,
+ "ridge_fit_backend": ridge_backend, "acoustic_backend": acoustic_backend,
+ "native_extension_available": callable(native_fit),
"peak_rss_mib": peak_rss_mib,
"peak_rss_scope": ("Worker lifetime through scoring, including imports and input loading"
if peak_rss_mib is not None else "Unavailable on this platform"),
"retained": len(ranked), "independent_groups": len(events), "scope": fixture["scope"]}
+ if "anchor_window_s" in fixture:
+ summary["anchor_window_s"] = fixture["anchor_window_s"]
+ if args.include_track_query:
+ summary["track_query_operations"] = query_operations
+ summary["execution_stage_seconds"] = execution_stages
+ summary["timing_scope"] = (
+ "Initial local index traversal and conservative distance sieve, followed by complete "
+ "CPA/association scoring and retention. Excludes imports, index metadata, event "
+ "preparation, downloads, terrain sampling, audio discovery, and refinement.")
+ if args.stage_timing:
+ summary["phase_seconds"] = phase_seconds
+ summary["phase_timing_scope"] = "Inclusive instrumented call times; overlapping phases must not be summed."
write_new(args.output / "summary.json", summary)
print(json.dumps(summary), flush=True)
@@ -261,6 +393,10 @@ def compare(args):
"--fixture", str(args.fixture.resolve()), "--output", str(output.resolve())]
if args.index:
command.extend(["--index", str(args.index.resolve())])
+ if args.stage_timing:
+ command.append("--stage-timing")
+ if args.include_track_query:
+ command.append("--include-track-query")
if name == "current" and args.require_current_native:
command.append("--require-native")
if name == "current" and args.require_current_native_acoustic:
@@ -270,7 +406,7 @@ def compare(args):
if expected is None:
expected = payload
elif payload != expected:
- raise AssertionError(f"Exact candidate, diagnostic, support, or retention mismatch: {output}")
+ raise AssertionError(f"Exact selected-track, candidate, diagnostic, support, or retention mismatch: {output}")
summary = json.loads((output / "summary.json").read_text())
runs.append({"source": name, "repetition": repetition + 1, **summary})
medians = {}
@@ -285,6 +421,16 @@ def compare(args):
"speedup": {metric: medians["baseline"][metric] / medians["current"][metric]
for metric in ("wall_seconds", "cpu_seconds")},
"scope": runs[0]["scope"], "runs": runs}
+ if args.include_track_query:
+ report["timing_scope"] = runs[0]["timing_scope"]
+ report["track_query_included"] = True
+ report["execution_stage_medians"] = {
+ name: {stage: {metric: statistics.median(
+ row["execution_stage_seconds"][stage][metric]
+ for row in runs if row["source"] == name)
+ for metric in ("wall_seconds", "cpu_seconds")}
+ for stage in runs[0]["execution_stage_seconds"]}
+ for name in sources}
write_new(args.output / "comparison.json", report)
print(json.dumps({key: report[key] for key in ("identical_results", "medians", "speedup", "scope")}), flush=True)
@@ -299,6 +445,10 @@ def main():
default=ROOT / "evidence/results/reference-local/report.json")
preparation.add_argument("--clock-half-width-s", type=float, default=120.0)
preparation.add_argument("--full-day", action="store_true", help="Use all indexed times with an unknown clock")
+ preparation.add_argument("--anchor-window-s", nargs=2, type=float, metavar=("LOW", "HIGH"),
+ help="Optional half-open association ownership partition; physical fit windows remain unchanged")
+ preparation.add_argument("--include-track-query", action="store_true",
+ help="Prepare the production query/sieve workload; budgets bound retained tracks, including overloaded queries")
preparation.add_argument("--max-tracks", type=int, default=5000)
preparation.add_argument("--max-samples", type=int, default=1000000)
preparation.add_argument("--max-candidates", type=int, default=20000)
@@ -312,21 +462,31 @@ def main():
comparison.add_argument("--repeats", type=int, default=3)
comparison.add_argument("--timeout-s", type=float, default=600.0)
comparison.add_argument("--output", required=True, type=Path)
+ comparison.add_argument("--stage-timing", action="store_true",
+ help="Collect inclusive phase times in a separate diagnostic run")
+ comparison.add_argument("--include-track-query", action="store_true",
+ help="Include initial index traversal and the production overload sieve in each timed worker")
comparison.add_argument("--require-current-native", action="store_true",
- help="Fail unless the timed current worker actually uses its native fitter")
+ help="Require successful native ridge fits inside each current scoring run")
comparison.add_argument("--require-current-native-acoustic", action="store_true",
- help="Fail unless the timed current worker actually uses its native acoustic solver")
+ help="Require successful native acoustic solves inside each current scoring run")
internal = commands.add_parser("_worker", help=argparse.SUPPRESS)
internal.add_argument("--source", required=True, type=Path)
internal.add_argument("--fixture", required=True, type=Path)
internal.add_argument("--index", type=Path)
internal.add_argument("--output", required=True, type=Path)
+ internal.add_argument("--stage-timing", action="store_true", help=argparse.SUPPRESS)
+ internal.add_argument("--include-track-query", action="store_true", help=argparse.SUPPRESS)
internal.add_argument("--require-native", action="store_true", help=argparse.SUPPRESS)
internal.add_argument("--require-native-acoustic", action="store_true", help=argparse.SUPPRESS)
args = parser.parse_args()
if args.command == "prepare":
if args.clock_half_width_s < 0:
parser.error("--clock-half-width-s must be nonnegative")
+ if (args.anchor_window_s is not None
+ and (not all(math.isfinite(value) for value in args.anchor_window_s)
+ or args.anchor_window_s[0] >= args.anchor_window_s[1])):
+ parser.error("--anchor-window-s requires finite increasing endpoints")
sys.path.insert(0, str(ROOT / "src"))
prepare(args)
elif args.command == "compare":
@@ -339,4 +499,3 @@ def main():
if __name__ == "__main__":
main()
-
diff --git a/src/dopplergeist/_native_ridge.c b/src/dopplergeist/_native_ridge.c
index c424624..6dfc964 100644
--- a/src/dopplergeist/_native_ridge.c
+++ b/src/dopplergeist/_native_ridge.c
@@ -267,8 +267,108 @@ done:
return result;
}
+/* This is a positive-only shortcut for the Python evidence validator. Any
+ * noncanonical or invalid value returns False, so Python retains its original
+ * error reason, ordering, subclass behavior and unbounded-integer semantics.
+ * No fit, score threshold or association arithmetic is performed here. */
+static int
+plain_string_dict(PyObject *object)
+{
+ if (!PyDict_CheckExact(object)) return 0;
+ PyObject *key, *value;
+ Py_ssize_t position = 0;
+ while (PyDict_Next(object, &position, &key, &value)) {
+ if (!PyUnicode_CheckExact(key)) return 0;
+ }
+ return 1;
+}
+
+static int
+exact_finite_float(PyObject *object, double *value)
+{
+ if (object == NULL || !PyFloat_CheckExact(object)) return 0;
+ *value = PyFloat_AS_DOUBLE(object);
+ return isfinite(*value);
+}
+
+static PyObject *
+valid_ridge_events(PyObject *self, PyObject *args)
+{
+ PyObject *diagnostics, *scores, *frequencies, *events;
+ if (!PyArg_ParseTuple(args, "OOOO", &diagnostics, &scores, &frequencies, &events)) {
+ return NULL;
+ }
+ if (!PyList_CheckExact(diagnostics) || !PyList_CheckExact(scores) ||
+ !PyList_CheckExact(frequencies) || !PyList_CheckExact(events)) Py_RETURN_FALSE;
+ Py_ssize_t count = PyList_GET_SIZE(events);
+ if (PyList_GET_SIZE(diagnostics) != count || PyList_GET_SIZE(scores) != count ||
+ PyList_GET_SIZE(frequencies) != count) Py_RETURN_FALSE;
+
+ static const char *fields[] = {
+ "support_fraction", "score", "f0_hz", "source_drift_hz", "source_drift_hz_s",
+ "source_reference_emission_time_s", "source_span_s", "drift_bound_hz",
+ "rmse_hz", "outlier_fraction"
+ };
+ for (Py_ssize_t index = 0; index < count; ++index) {
+ PyObject *diagnostic = PyList_GET_ITEM(diagnostics, index);
+ PyObject *event = PyList_GET_ITEM(events, index);
+ if (!plain_string_dict(diagnostic) || !plain_string_dict(event)) Py_RETURN_FALSE;
+ double values[10], score, frequency;
+ for (int field = 0; field < 10; ++field) {
+ if (!exact_finite_float(PyDict_GetItemString(diagnostic, fields[field]),
+ &values[field])) Py_RETURN_FALSE;
+ }
+ if (!exact_finite_float(PyList_GET_ITEM(scores, index), &score) ||
+ !exact_finite_float(PyList_GET_ITEM(frequencies, index), &frequency)) Py_RETURN_FALSE;
+ if (values[0] != 1.0 || values[6] <= 0.0 || values[7] < 0.0 || values[8] < 0.0 ||
+ values[9] < 0.0 || values[9] > 1.0 || values[2] <= 0.0 || values[1] > 0.0 ||
+ score != values[1] || frequency != values[2]) Py_RETURN_FALSE;
+
+ PyObject *samples = PyDict_GetItemString(diagnostic, "sample_count");
+ PyObject *independent = PyDict_GetItemString(diagnostic, "independent_sample_count");
+ if (samples == NULL || independent == NULL || !PyLong_CheckExact(samples) ||
+ !PyLong_CheckExact(independent)) Py_RETURN_FALSE;
+ Py_ssize_t n = PyLong_AsSsize_t(samples);
+ if (n == -1 && PyErr_Occurred()) {
+ PyErr_Clear();
+ Py_RETURN_FALSE;
+ }
+ Py_ssize_t k = PyLong_AsSsize_t(independent);
+ if (k == -1 && PyErr_Occurred()) {
+ PyErr_Clear();
+ Py_RETURN_FALSE;
+ }
+ if (n < 10 || k < 1 || k > n) Py_RETURN_FALSE;
+ PyObject *optimizer = PyDict_GetItemString(diagnostic, "optimizer_success");
+ if (optimizer != NULL && optimizer != Py_True) Py_RETURN_FALSE;
+
+ PyObject *bounds = PyDict_GetItemString(event, "f0_range_hz");
+ if (bounds == NULL || !(PyList_CheckExact(bounds) || PyTuple_CheckExact(bounds))) Py_RETURN_FALSE;
+ int tuple_bounds = PyTuple_CheckExact(bounds);
+ Py_ssize_t bounds_count = tuple_bounds ? PyTuple_GET_SIZE(bounds) : PyList_GET_SIZE(bounds);
+ if (bounds_count != 2) Py_RETURN_FALSE;
+ double lower, upper, fraction = 0.02;
+ if (!exact_finite_float(tuple_bounds ? PyTuple_GET_ITEM(bounds, 0) : PyList_GET_ITEM(bounds, 0), &lower) ||
+ !exact_finite_float(tuple_bounds ? PyTuple_GET_ITEM(bounds, 1) : PyList_GET_ITEM(bounds, 1), &upper)) Py_RETURN_FALSE;
+ PyObject *quality = PyDict_GetItemString(event, "quality");
+ if (quality != NULL) {
+ if (!plain_string_dict(quality)) Py_RETURN_FALSE;
+ PyObject *configured = PyDict_GetItemString(quality, "source_drift_fraction");
+ if (configured != NULL && !exact_finite_float(configured, &fraction)) Py_RETURN_FALSE;
+ }
+ if (!(0.0 < lower && lower <= upper && lower <= frequency && frequency <= upper &&
+ 0.0 <= fraction && fraction <= 0.1)) Py_RETURN_FALSE;
+ double drift = values[3], cap = values[7];
+ if (!(lower * fraction <= cap && cap <= upper * fraction) || fabs(drift) > cap) Py_RETURN_FALSE;
+ if (fabs(values[4] * values[6] - drift) > 1e-12 + 1e-12 * fabs(drift)) Py_RETURN_FALSE;
+ }
+ Py_RETURN_TRUE;
+}
+
static PyMethodDef methods[] = {
{"fit_irls", fit_irls, METH_VARARGS, "Run six robust ridge-fit iterations in C."},
+ {"valid_ridge_events", valid_ridge_events, METH_VARARGS,
+ "Recognize valid canonical ridge evidence; other inputs retain Python validation."},
{NULL, NULL, 0, NULL}
};
diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.py
index 4ca9830..a018a6b 100644
--- a/src/dopplergeist/data_access.py
+++ b/src/dopplergeist/data_access.py
@@ -63,6 +63,81 @@ def _frame_track(name, frame):
return track
+class _IndexColumn:
+ """The column operations used by the unchanged track normalization code."""
+
+ __slots__ = ("values",)
+
+ def __init__(self, values):
+ self.values = values
+
+ def to_numpy(self, dtype=None):
+ return np.asarray(self.values, dtype=dtype)
+
+ @property
+ def iloc(self):
+ return self.values
+
+ def tolist(self):
+ return self.values.tolist()
+
+
+class _IndexColumns:
+ """A bounded column view of one canonical decoded index segment."""
+
+ __slots__ = ("columns", "count")
+
+ def __init__(self, columns, count):
+ self.columns, self.count = columns, count
+
+ def __len__(self):
+ return self.count
+
+ def __contains__(self, name):
+ return name in self.columns
+
+ def __getitem__(self, name):
+ return self.columns[name]
+
+ def __getattr__(self, name):
+ try:
+ return self.columns[name]
+ except KeyError:
+ raise AttributeError(name) from None
+
+
+def _payload_frame(records):
+ """Avoid pandas construction for canonical float64/string index records.
+
+ Both paths still use _frame_track and _segment_velocity. Restrict this
+ shortcut to representations whose pandas column conversion is exact;
+ mixed schemas, integer position fields and unusual labels keep pandas'
+ original dtype inference and failure behavior.
+ """
+ if type(records) is not list or not records or any(type(row) is not dict for row in records):
+ return pd.DataFrame(records)
+ absent = object()
+ columns = {}
+ for name in ("time_s", "lat", "lon", "alt_m"):
+ values = [row.get(name, absent) for row in records]
+ if any(type(value) is not float for value in values):
+ return pd.DataFrame(records)
+ columns[name] = _IndexColumn(np.asarray(values, dtype=float))
+ for name in ("east_mps", "north_mps", "up_mps", "typecode", "altitude_basis", "aircraft_class"):
+ values = [row.get(name, absent) for row in records]
+ if all(value is absent for value in values):
+ continue
+ numeric = name in ("east_mps", "north_mps", "up_mps")
+ if numeric:
+ canonical = all(type(value) is float or value is None for value in values)
+ else:
+ canonical = all(type(value) is str for value in values)
+ if not canonical:
+ return pd.DataFrame(records)
+ columns[name] = _IndexColumn(np.asarray(values, dtype=float if numeric else object))
+ return _IndexColumns(columns, len(records))
+
+
@lru_cache(maxsize=1)
def _ecef_to_geographic():
from pyproj import Transformer
@@ -444,7 +519,7 @@ class FlightIndex:
cache_scan["used"].add(key)
if entry is not None:
return pickle.loads(cached)
- frame = pd.DataFrame(json.loads(gzip.decompress(payload)))
+ frame = _payload_frame(json.loads(gzip.decompress(payload)))
if max_samples is not None and len(frame) > max_samples:
raise DataUnavailable("Flight lookup sample budget exceeded")
track = _frame_track(name, frame)
diff --git a/src/dopplergeist/matching.py b/src/dopplergeist/matching.py
index d099d99..88afdcd 100644
--- a/src/dopplergeist/matching.py
+++ b/src/dopplergeist/matching.py
@@ -39,6 +39,8 @@ else:
_native_binary_hash = hashlib.sha256(Path(_native_ridge_module.__file__).read_bytes()).hexdigest()
RIDGE_FIT_BACKEND = f"native-c-irls-v2-pairwise:{_native_binary_hash}"
+_native_validate_ridge_events = getattr(_native_ridge_module, "valid_ridge_events", None)
+
_NATIVE_RIDGE_ERROR_POLICY = {"divide": "warn", "over": "warn", "under": "ignore", "invalid": "warn"}
diff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.py
index 9ef7604..54f689f 100644
--- a/src/dopplergeist/search.py
+++ b/src/dopplergeist/search.py
@@ -9,6 +9,7 @@ from scipy.signal import find_peaks
from scipy.spatial import cKDTree
from .geometry import AcousticSupportError, ecef
+from .matching import _native_validate_ridge_events
_CPA_COLUMNS = ["point_id", "flight_id", "emission_time", "receive_time", "distance_m", "speed_mps"]
_CPA_DTYPE = np.dtype([(name, "int64" if name == "point_id" else object if name == "flight_id" else "float64")
@@ -339,7 +340,13 @@ def _ridge_trial_classification(result, events, association):
return "unknown", "missing_event_fit_evidence"
fields = ("score", "f0_hz", "source_drift_hz", "source_drift_hz_s",
"source_reference_emission_time_s", "source_span_s", "drift_bound_hz", "rmse_hz", "outlier_fraction")
- for diagnostic, score, frequency, event in zip(diagnostics, scores, frequencies, events):
+ # The optional native path recognizes only valid evidence in plain core
+ # containers. Invalid or custom inputs retain the exact Python validation
+ # below, including error precedence and subclass/numeric-type behavior.
+ validated = (_native_validate_ridge_events is not None and
+ _native_validate_ridge_events(diagnostics, scores, frequencies, events))
+ pending = () if validated else zip(diagnostics, scores, frequencies, events)
+ for diagnostic, score, frequency, event in pending:
if (not isinstance(diagnostic, dict) or not finite(diagnostic.get("support_fraction"))
or diagnostic["support_fraction"] != 1.0):
return "unknown", "incomplete_acoustic_support"
@@ -408,6 +415,14 @@ def _association_event_ceiling(diagnostic, score, frequency, event, limit):
mutation checks while avoiding repeated validation of the same clock fit.
Noncanonical numeric types retain the original validation and arithmetic.
"""
+ if (_native_validate_ridge_events is not None
+ and _ridge_trial_classification is _CANONICAL_CEILING_FUNCTIONS[2]
+ and _association_event_ceiling_uncached is _CANONICAL_CEILING_FUNCTIONS[1]
+ and type(limit) is float and math.isfinite(limit) and limit > 0
+ and _native_validate_ridge_events([diagnostic], [score], [frequency], [event])):
+ if -score > limit:
+ return -np.inf
+ return float(np.nextafter(1 - (-score) / limit, np.inf))
from .matching import _ridge_fit_cache
cache = _ridge_fit_cache.get()
if (cache is None or cache.max_entries <= 0 or type(diagnostic) is not dict
diff --git a/tests/test_real_index_columns.py b/tests/test_real_index_columns.py
new file mode 100644
index 0000000..0371084
--- /dev/null
+++ b/tests/test_real_index_columns.py
@@ -0,0 +1,92 @@
+"""Index decoding parity on unchanged archived ADS-B observations."""
+from copy import deepcopy
+import gzip
+import json
+import os
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from dopplergeist import data_access
+
+
+def _assert_track_equal(got, expected):
+ assert vars(got).keys() == vars(expected).keys()
+ for name, value in vars(expected).items():
+ actual = getattr(got, name)
+ if isinstance(value, np.ndarray):
+ assert actual.dtype == value.dtype and actual.shape == value.shape
+ assert actual.tobytes() == value.tobytes(), name
+ elif name in {"spline", "derivative"}:
+ assert actual.c.tobytes() == value.c.tobytes(), name
+ assert actual.x.tobytes() == value.x.tobytes(), name
+ assert actual.axis == value.axis and actual.extrapolate == value.extrapolate
+ else:
+ assert actual == value, name
+
+
+def _recorded_segments():
+ root = Path(__file__).resolve().parents[1] / "research/trajectory-motion/frozen"
+ for source in sorted(root.glob("fresh-*.json")):
+ rows = sorted(json.loads(source.read_text()), key=lambda row: row["time_s"])
+ chunks = [[]]
+ for row in rows:
+ if chunks[-1] and row["time_s"] - chunks[-1][-1]["time_s"] > 30:
+ chunks.append([])
+ chunks[-1].append(row)
+ for number, chunk in enumerate(chunks):
+ if len(chunk) >= 3:
+ yield f"{source.stem}:{number}", chunk
+
+
+def test_recorded_columns_preserve_arrays_labels_and_velocity_inference():
+ checked, inferred = 0, 0
+ for name, records in _recorded_segments():
+ frame = data_access._payload_frame(records)
+ assert isinstance(frame, data_access._IndexColumns)
+ expected = data_access._frame_track(name, pd.DataFrame(records))
+ actual = data_access._frame_track(name, frame)
+ _assert_track_equal(actual, expected)
+ inferred += sum(actual.velocity_inferred_components.values())
+ checked += 1
+ assert checked >= 4 and inferred > 0
+
+
+def test_noncanonical_column_representations_keep_pandas():
+ name, recorded = next(_recorded_segments())
+ # Numeric representations and optional labels change; every observed
+ # position, time and velocity value remains the same.
+ numeric = deepcopy(recorded)
+ numeric[0]["time_s"] = np.float64(numeric[0]["time_s"])
+ absent_label = deepcopy(recorded)
+ del absent_label[0]["typecode"]
+ null_label = deepcopy(recorded)
+ null_label[0]["altitude_basis"] = None
+ for records in (numeric, absent_label, null_label):
+ actual = data_access._payload_frame(records)
+ assert isinstance(actual, pd.DataFrame)
+ pd.testing.assert_frame_equal(actual, pd.DataFrame(records))
+
+
+def test_indexed_recorded_payloads_preserve_all_track_fields():
+ source = os.environ.get("DOPPLERGEIST_REAL_CPA_FIXTURE")
+ if not source:
+ pytest.skip("set DOPPLERGEIST_REAL_CPA_FIXTURE to a recorded-input workload")
+ fixture = json.loads(Path(source).read_text())
+ index = data_access.FlightIndex(fixture["real_index"])
+ requested = set(fixture["track_ids"])
+ checked = fast = 0
+ with index.connect() as db:
+ for name, payload in db.execute("SELECT name,payload FROM segments ORDER BY name"):
+ if name not in requested:
+ continue
+ records = json.loads(gzip.decompress(payload))
+ frame = data_access._payload_frame(records)
+ fast += isinstance(frame, data_access._IndexColumns)
+ actual = data_access._frame_track(name, frame)
+ expected = data_access._frame_track(name, pd.DataFrame(records))
+ _assert_track_equal(actual, expected)
+ checked += 1
+ assert checked == len(requested) and checked > 0 and fast > 0
diff --git a/tests/test_real_native_ridge_long.py b/tests/test_real_native_ridge_long.py
new file mode 100644
index 0000000..29998f5
--- /dev/null
+++ b/tests/test_real_native_ridge_long.py
@@ -0,0 +1,72 @@
+"""Check the native reduction boundary on a long recorded Doppler ridge."""
+import json
+import os
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from dopplergeist import matching
+from dopplergeist.data_access import FlightIndex
+from dopplergeist.geometry import ecef
+from dopplergeist.search import filter_association_event
+
+
+def test_recorded_ridge_crosses_native_pairwise_reduction_boundary(monkeypatch):
+ native = matching._native_fit_irls
+ if native is None:
+ pytest.skip("optional native ridge extension is not installed")
+ source = os.environ.get("DOPPLERGEIST_REAL_CPA_FIXTURE")
+ if not source:
+ pytest.skip("set DOPPLERGEIST_REAL_CPA_FIXTURE to the real point120 diagnostic")
+ fixture = json.loads(Path(source).read_text())
+ events, measurements = matching.prepare_events(fixture["events"])
+ index = max(range(len(measurements)), key=lambda i: len(measurements[i][1]))
+ event, measurement = events[index], measurements[index]
+ if len(measurement[1]) < 141:
+ pytest.skip("this recorded fixture has no ridge crossing the 128-sample boundary")
+ rows = filter_association_event(pd.DataFrame(fixture["cpa_rows"]), event, 5., None, 5.)
+ tracks = {track.id: track for track in FlightIndex(fixture["real_index"]).lookup(
+ list(dict.fromkeys(rows.flight_id)), max_tracks=2000, max_samples=2000000)}
+ point = fixture["point"]
+ observer = ecef(point["lat"], point["lon"], point["elev_m"])
+ checked = passages = 0
+ native_successes = 0
+
+ def counted(*arguments):
+ nonlocal native_successes
+ result = native(*arguments)
+ native_successes += result is not None and all(np.isfinite(value) for value in result)
+ return result
+
+ monkeypatch.setattr(matching, "_native_fit_irls", counted)
+ for row in rows.itertuples():
+ # These are shared-clock hypotheses for actual measured observations;
+ # no recording or trajectory samples are generated.
+ starts = row.receive_time - event["cpa_receive_s"] + np.array([-2.5, 0., 2.5])
+ batch = matching.batch_acoustic_trials(
+ [tracks[row.flight_id]], [measurement], observer, starts)[0]
+ if batch is None or not all(np.all(np.isfinite(array)) for array in batch):
+ continue
+ for count in (127, 128, 129, 136, 141):
+ selected = (matching.RidgeTimes(measurement[0].time[:count]),
+ measurement[1][:count], measurement[2][:count], {})
+ for factor, emitted in zip(batch[0][:, :count], batch[1][:, :count]):
+ for sigma in (0.5, 1.0, 3.7, 18.001973492902533):
+ for fraction in (0.0, 0.02, 0.1):
+ arguments = (None, None, selected, 0., event["f0_range_hz"],
+ 343., sigma, fraction)
+ with monkeypatch.context() as patch:
+ patch.setattr(matching, "_native_fit_irls", None)
+ expected = matching._fit_ridge_uncached(
+ *arguments, acoustic_result=(factor, emitted))
+ actual = matching._fit_ridge_uncached(
+ *arguments, acoustic_result=(factor, emitted))
+ assert expected["support_fraction"] == 1. and np.isfinite(expected["score"])
+ assert json.dumps(actual, sort_keys=True) == json.dumps(expected, sort_keys=True)
+ checked += 1
+ passages += 1
+ if passages == 3:
+ break
+ assert passages == 3 and checked == native_successes == 540
diff --git a/tests/test_real_native_validation.py b/tests/test_real_native_validation.py
new file mode 100644
index 0000000..c115f0c
--- /dev/null
+++ b/tests/test_real_native_validation.py
@@ -0,0 +1,199 @@
+"""Validation parity on saved recorded-ridge fits, including corrupted evidence.
+
+These tests change fit metadata to exercise software guards; they do not create
+recordings, trajectories or location-accuracy cases.
+"""
+from copy import deepcopy
+import json
+from pathlib import Path
+import warnings
+
+import numpy as np
+import pytest
+
+from dopplergeist import matching, search
+
+
+def recorded():
+ fixture = json.loads((Path(__file__).parent / "data/real-ridge-batch-arithmetic.json").read_text())
+ return fixture["legacy_diagnostics"], fixture["event"]
+
+
+def native():
+ function = search._native_validate_ridge_events
+ if function is None:
+ pytest.skip("optional native ridge extension is not installed")
+ return function
+
+
+def outcome(function):
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ try:
+ value = ("result", function())
+ except Exception as error:
+ value = ("error", type(error), str(error))
+ return value, [(type(item.message), str(item.message)) for item in caught]
+
+
+def compare(monkeypatch, native, result, events, association=False):
+ with monkeypatch.context() as patch:
+ patch.setattr(search, "_native_validate_ridge_events", None)
+ expected = outcome(lambda: search._ridge_trial_classification(result, events, association))
+ assert outcome(lambda: search._ridge_trial_classification(result, events, association)) == expected
+ return expected[0]
+
+
+def test_recorded_fits_take_native_path_and_preserve_aggregate_checks(recorded, native, monkeypatch):
+ fits, event = recorded
+ for fit in fits:
+ result = search._ridge_result([fit])
+ assert native([fit], result["event_scores"], result["f0_hz"], [event]) is True
+ assert compare(monkeypatch, native, result, [event]) == ("result", ("evaluated", None))
+ association = matching.association_score(result, [event])
+ compare(monkeypatch, native, association, [event], True)
+ for key, value in (("score", 1.0), ("residual_score", 0.0),
+ ("score_kind", "unrecognized"), ("optimizer_success", False)):
+ changed = dict(association, **{key: value})
+ compare(monkeypatch, native, changed, [event], True)
+
+
+def test_recorded_field_mutations_keep_all_python_failure_reasons(recorded, native, monkeypatch):
+ fits, event = recorded
+ expected_reasons = {
+ "support_fraction": (0.0, "incomplete_acoustic_support"),
+ "source_reference_emission_time_s": (np.inf, "nonfinite_fit_evidence"),
+ "sample_count": (9, "invalid_fit_evidence"),
+ "optimizer_success": (False, "failed_optimizer"),
+ "score": (-123.0, "inconsistent_event_fit_evidence"),
+ "source_drift_hz": (fits[0]["drift_bound_hz"] + 1.0, "fit_outside_declared_drift_domain"),
+ "source_drift_hz_s": (123.0, "inconsistent_source_drift"),
+ }
+ for field, (value, reason) in expected_reasons.items():
+ result = search._ridge_result([deepcopy(fits[0])])
+ result["event_diagnostics"][0][field] = value
+ assert compare(monkeypatch, native, result, [event]) == ("result", ("unknown", reason))
+ result = search._ridge_result([deepcopy(fits[0])])
+ changed = dict(event, f0_range_hz=[1.0, 2.0])
+ assert compare(monkeypatch, native, result, [changed]) == (
+ "result", ("unknown", "fit_outside_declared_frequency_domain"))
+
+ # Invalid or noncanonical replacements must reach the original validator.
+ for field in search._CEILING_FLOAT_FIELDS:
+ for value in (None, True, 1, np.float32(1), np.float64(1), np.nan, np.inf, -np.inf):
+ result = search._ridge_result([deepcopy(fits[0])])
+ result["event_diagnostics"][0][field] = value
+ assert native(result["event_diagnostics"], result["event_scores"], result["f0_hz"], [event]) is False
+ compare(monkeypatch, native, result, [event])
+ result = search._ridge_result([deepcopy(fits[0])])
+ del result["event_diagnostics"][0][field]
+ compare(monkeypatch, native, result, [event])
+
+
+def test_native_fallback_preserves_integer_range_and_custom_types(recorded, native, monkeypatch):
+ fits, event = recorded
+
+ class Dictionary(dict):
+ pass
+
+ class Sequence(list):
+ pass
+
+ class Number(float):
+ pass
+
+ mutations = []
+ for field in ("sample_count", "independent_sample_count"):
+ for value in (True, 31.0, np.int64(31), 10**100, -1, None):
+ result = search._ridge_result([deepcopy(fits[0])])
+ result["event_diagnostics"][0][field] = value
+ mutations.append((result, [event]))
+ result = search._ridge_result([deepcopy(fits[0])])
+ result["event_diagnostics"][0]["sample_count"] = 10**100
+ result["event_diagnostics"][0]["independent_sample_count"] = 10**90
+ mutations.append((result, [event]))
+ for field in search._CEILING_FLOAT_FIELDS:
+ result = search._ridge_result([deepcopy(fits[0])])
+ result["event_diagnostics"][0][field] = Number(result["event_diagnostics"][0][field])
+ mutations.append((result, [event]))
+ result = search._ridge_result([Dictionary(fits[0])])
+ mutations.append((result, [event]))
+ for field in ("event_diagnostics", "event_scores", "f0_hz"):
+ result = search._ridge_result([deepcopy(fits[0])])
+ result[field] = Sequence(result[field])
+ mutations.append((result, [event]))
+ for changed in (Dictionary(event), dict(event, quality=Dictionary(event["quality"])),
+ dict(event, f0_range_hz=Sequence(event["f0_range_hz"]))):
+ mutations.append((search._ridge_result([deepcopy(fits[0])]), [changed]))
+ mutations.append((search._ridge_result([deepcopy(fits[0])]), Sequence([event])))
+ for result, events in mutations:
+ assert native(result["event_diagnostics"], result["event_scores"], result["f0_hz"], events) is False
+ compare(monkeypatch, native, result, events)
+
+
+def test_native_fallback_preserves_custom_key_and_getter_calls(recorded, native, monkeypatch):
+ fits, event = recorded
+ calls = []
+
+ class Dictionary(dict):
+ def get(self, *args):
+ calls.append(args)
+ return super().get(*args)
+
+ result = search._ridge_result([Dictionary(fits[0])])
+ assert native(result["event_diagnostics"], result["event_scores"], result["f0_hz"], [event]) is False
+ assert calls == []
+ compare(monkeypatch, native, result, [event])
+ midpoint = len(calls) // 2
+ assert calls[:midpoint] == calls[midpoint:]
+
+ class Key(str):
+ pass
+
+ for key in (17, Key("extra")):
+ result = search._ridge_result([dict(fits[0], extra=1)])
+ result["event_diagnostics"][0][key] = result["event_diagnostics"][0].pop("extra")
+ assert native(result["event_diagnostics"], result["event_scores"], result["f0_hz"], [event]) is False
+ compare(monkeypatch, native, result, [event])
+
+
+def test_default_fraction_and_current_mutable_values_are_rechecked(recorded, native, monkeypatch):
+ fits, event = recorded
+ result = search._ridge_result([deepcopy(fits[0])])
+ for changed in (event, {"f0_range_hz": tuple(event["f0_range_hz"])},
+ dict(event, quality={"source_drift_fraction": 0.02})):
+ assert native(result["event_diagnostics"], result["event_scores"], result["f0_hz"], [changed]) is True
+ compare(monkeypatch, native, result, [changed])
+ for field in ("score", "support_fraction", "f0_hz"):
+ original = result["event_diagnostics"][0][field]
+ result["event_diagnostics"][0][field] = np.nextafter(original, np.inf).item()
+ compare(monkeypatch, native, result, [event])
+ result["event_diagnostics"][0][field] = original
+ assert native(result["event_diagnostics"], result["event_scores"], result["f0_hz"], [event]) is True
+
+
+def test_native_ceiling_preserves_numpy_policies_and_custom_classifier(recorded, native, monkeypatch):
+ fits, event = recorded
+ fit = fits[0]
+ # Equality with the observed loss gives a zero contribution. nextafter
+ # then exercises the original NumPy underflow warning/exception behavior.
+ limits = (-fit["score"], np.nextafter(-fit["score"], np.inf).item(), 3.0,
+ np.float64(3.0), 3, 0.0, np.inf, np.nan)
+ for policy in ("ignore", "warn", "raise"):
+ for limit in limits:
+ arguments = (fit, fit["score"], fit["f0_hz"], event, limit)
+ with np.errstate(under=policy):
+ with monkeypatch.context() as patch:
+ patch.setattr(search, "_native_validate_ridge_events", None)
+ expected = outcome(lambda: search._association_event_ceiling(*arguments))
+ actual = outcome(lambda: search._association_event_ceiling(*arguments))
+ # NaN ceilings compare via their serialized value.
+ assert repr(actual) == repr(expected)
+ with monkeypatch.context() as patch:
+ patch.setattr(search, "_ridge_trial_classification", lambda *args: ("unknown", "custom"))
+ assert search._association_event_ceiling(fit, fit["score"], fit["f0_hz"], event, 3.0) == 1.0
+ with monkeypatch.context() as patch:
+ patch.setattr(search, "_association_event_ceiling_uncached", lambda *args: 0.125)
+ assert search._association_event_ceiling(fit, fit["score"], fit["f0_hz"], event, 3.0) == 0.125