Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Accelerate partitioned flight checks with exact native batches
Cinna Davisc5476bd0ff12
diff --git a/.github/workflows/real-performance.yml b/.github/workflows/real-performance.ymlindex d727775..afd1d3b 100644--- a/.github/workflows/real-performance.yml+++ b/.github/workflows/real-performance.yml@@ -11,7 +11,7 @@ on:type: booleandefault: truepush:- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103]+ branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103, perf/flight-sieve-94a6]paths:- .github/workflows/real-performance.yml- src/dopplergeist/**@@ -38,7 +38,7 @@ env:NUMEXPR_NUM_THREADS: '1'CC: gccPYTHONPATH: src- BASELINE_REF: ${{ inputs.baseline || '27da08a878ac2ec90cd7e2c07f43765ceeea609b' }}+ BASELINE_REF: ${{ inputs.baseline || (github.ref == 'refs/heads/perf/flight-sieve-94a6' && 'b5ff48fe0e9d6e6feac7a4faf5268630f0c85d8b') || '27da08a878ac2ec90cd7e2c07f43765ceeea609b' }}jobs:compare:if: github.event_name != 'push' || github.ref != 'refs/heads/perf/real-pipeline-throughput-20261103'@@ -56,12 +56,13 @@ jobs:python -m pip install 'setuptools>=68'python -m pip install --editable '.[test]'python - <<'PY'- from dopplergeist import matching, native_acoustic+ from dopplergeist import matching, native_acoustic, native_cpaassert callable(matching._native_fit_irls), "Native ridge extension did not build"assert callable(matching._native_validate_ridge_events), "Native evidence validator did not build"assert matching.RIDGE_FIT_BACKEND.startswith("native-c-")assert native_acoustic._native is not None, "Native acoustic extension did not build"assert native_acoustic.ACOUSTIC_BACKEND.startswith("native-c-")+ assert native_cpa._native is not None, "Native CPA extension did not build"print({"ridge_fit_backend": matching.RIDGE_FIT_BACKEND,"acoustic_backend": native_acoustic.ACOUSTIC_BACKEND})PY@@ -100,14 +101,17 @@ jobs:- name: Real-input regression testsenv:DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json+ DOPPLERGEIST_REAL_DENSE_QUERY: benchmark-inputs/dense-query.json+ DOPPLERGEIST_REAL_DEVELOPMENT_INDEX: benchmark-inputs/reference-region.sqliteDOPPLERGEIST_REAL_REFERENCE_ROOT: benchmark-inputs/reference-checkoutDOPPLERGEIST_REAL_REFERENCE_DISCOVERY: evidence/audio/w1-sensitivity-20261102/reference/discovery.json- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_native_ridge.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py+ run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_partition_query.py tests/test_real_native_cpa.py tests/test_real_native_ridge.py tests/test_real_native_ridge_batch.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py- name: Verify the explicit Python fallback on recorded arithmeticenv:DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.jsonDOPPLERGEIST_RIDGE_BACKEND: pythonDOPPLERGEIST_ACOUSTIC_BACKEND: python+ DOPPLERGEIST_CPA_BACKEND: pythonrun: python -m pytest -q tests/test_real_batch_scalar_parity.py tests/test_real_legacy_batch_cache.py- name: Compare real CPA preparationrun: python scripts/benchmark_real_cpa_rows.py --baseline-source "$RUNNER_TEMP/baseline/src" --output results/cpa-rows.json --repeats 5 --iterations 20@@ -121,11 +125,14 @@ jobs:if [ -f src/dopplergeist/_native_acoustic.c ]; thennative_option+=(--require-current-native-acoustic)fi+ if [ -f src/dopplergeist/_native_cpa.c ]; then+ native_option+=(--require-current-native-cpa)+ fifor width in 10 120; dotaskset -c "$benchmark_cpu" python scripts/benchmark_prepared_measurements.py compare --fixture "benchmark-inputs/point-$width.json" --baseline-source "$RUNNER_TEMP/baseline/src" --current-source src --repeats 3 --timeout-s 600 --output "results/performance-$width" "${native_option[@]}"donetaskset -c "$benchmark_cpu" python scripts/benchmark_prepared_measurements.py compare \- --fixture benchmark-inputs/dense-query.json --include-track-query \+ --fixture benchmark-inputs/dense-query.json --include-track-query --clock-partition-query \--baseline-source "$RUNNER_TEMP/baseline/src" --current-source src \--repeats 3 --timeout-s 600 --output results/performance-dense-query \"${native_option[@]}"diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 3a0d11e..71210f0 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -39,19 +39,24 @@ jobs:run: python -m compileall -q src scripts- name: CLI smoke checkrun: dopplergeist --help- - name: Check installed modules and packaged browser assets+ - name: Check installed native acceleration and packaged browser assetsrun: |python -I - <<'PY'from importlib import resourcesfrom pathlib import Pathimport dopplergeist- from dopplergeist import automatic, jobs, service+ from dopplergeist import _native_cpa, _native_ridge, automatic, jobs, servicepackage = Path(dopplergeist.__file__).resolve().parentassert "site-packages" in package.parts, package+ assert Path(_native_ridge.__file__).resolve().parent == package+ assert Path(_native_cpa.__file__).resolve().parent == package+ assert callable(_native_ridge.fit_irls_batch), "Native batch fitter did not build"+ assert callable(_native_ridge.fit_statistics), "Native fit counters did not build"+ assert callable(_native_cpa.select), "Native CPA reducer did not build"web = resources.files("dopplergeist.web")for name in ("index.html", "app.js", "style.css"):assert web.joinpath(name).is_file(), nameassert web.joinpath(name).read_bytes(), name- print("Installed package imports and browser assets passed; no audio or tracks generated.")+ print("Installed native acceleration and browser assets passed; no audio or tracks generated.")PYdiff --git a/docs/FLIGHT_SIEVE_PERFORMANCE.md b/docs/FLIGHT_SIEVE_PERFORMANCE.mdnew file mode 100644index 0000000..73c538e--- /dev/null+++ b/docs/FLIGHT_SIEVE_PERFORMANCE.md@@ -0,0 +1,149 @@+# Flight checking and sieve performance++## Measured result++The clock-partition query bound removes impossible whole flight segments before+JSON decoding, spline construction, distance checks, and closest-approach work.+The dense recorded replay queried **197 segments instead of 4,694**. It preserved+all **16,101 scored candidate payloads**, the **100 retained results**, and the+complete retention records and trial evidence.++Three alternating baseline/current pairs gave these median CPU times:++| Scope | Baseline | This change | Speedup |+| --- | ---: | ---: | ---: |+| Index query and conservative distance sieve | 6.413 s | 0.398 s | 16.11x |+| CPA, association scoring, and retention | 26.500 s | 24.964 s | 1.06x |+| Complete measured hot path | 34.073 s | 25.356 s | 1.34x |++Each row has its own median; stage medians need not sum to the total median.+Total wall-time medians were 34.146 and 25.400 seconds. The baseline is+`b5ff48fe0e9d6e6feac7a4faf5268630f0c85d8b`, which already includes native ridge and+acoustic solvers and numeric flight decoding. Earlier gains are not credited here.++This replay uses the original recording's 16 prepared event groups and 724 ridge+samples, one source-informed receiver, a full-day regional observation index, and one existing+348-second association ownership partition. It is a computation benchmark, not+an independent location-accuracy or nationwide-throughput result.++Timing includes index traversal, track decoding, the distance sieve, CPA+processing, association scoring, and retention. It excludes imports, index+metadata, event preparation, downloads, terrain sampling, audio discovery, and+refinement. Both sources used the same Python 3.12 runtime, dependencies,+single-thread numerical settings, CPU affinity, recorded inputs, and GCC flags.+The measured numerical functions were not replaced by counting wrappers.++## What changed++### Select possible source times before decoding tracks++Let the first matched passage imply recording start `A` inside the existing+half-open partition `[L, U)`. Its clock interval has tolerance `t_first`.+Each effective tolerance is `quality.cpa_tolerance_s + offset_tolerance_s / 2`.+A later passage with implied start `B` and tolerance `t_event` can participate+only if its interval intersects the first interval. Thus it lies within the+partition expanded by both tolerances. Any event may be first, so the bound uses+the maximum effective tolerance across every event.++Add each event's audio-relative CPA time to bound receive time. Subtract at most+`maximum_CPA_radius / sound_speed` to bound emission time. Every inverse operation+rounds outward, including subtraction at Unix timestamp precision.++The query selects **complete segments** touching this envelope. It never clips a+trajectory, removes an interior sample, or changes its spline. Serial and worker+paths derive the same anchor once and use it for both querying and scoring.+Silence filters, noncanonical settings, invalid tolerances, and disjoint query+windows keep the original broad query.++On the dense replay:++| Work | Baseline | This change |+| --- | ---: | ---: |+| Decoded flight segments | 4,694 | 197 |+| Decoded source samples | 713,100 | 40,690 |+| CPA rows | 944 | 95 |+| Association work units | 302,818 | 147,985 |+| Unsupported CPA track evaluations | 62 | 4 |++Every removed unsupported track's complete emission interval misses the proven+query envelope. All four retained unsupported reports remain identical. The+590,900 unknown trials and 15,777 unclassified candidate assessments also remain+unchanged. The replay remains incomplete where its original evidence was incomplete.++### Fuse closest-approach neighbor calculations++An optional C extension traverses the existing bounded spatial-neighbor graph.+It combines squared distances, local-minimum selection, and adjacent-segment+projection without creating expanded observer-index and coordinate arrays.+It preserves arithmetic order, asymmetric minima, first-edge ties, and row order.+Python still performs endpoint spline optimization and final row construction.++Five alternating trials of ten iterations on eight archived ADS-B segments gave+these full CPA-stage results, including preparation:++| Receiver points | Baseline CPU per iteration | Native CPA | Speedup |+| ---: | ---: | ---: | ---: |+| 1 | 1.411 ms | 1.091 ms | 1.29x |+| 32 | 3.598 ms | 2.767 ms | 1.30x |+| 128 | 4.977 ms | 3.572 ms | 1.39x |+| 1,024 | 15.017 ms | 10.183 ms | 1.47x |+| 4,096 | 48.600 ms | 38.321 ms | 1.27x |++All 205 baseline comparisons matched exactly. This small real-track workload+measures the CPA implementation, not complete search throughput.++The extension is optional. Unsupported layouts, floating-point policies, or+exceptional arithmetic retain the Python path. Set+`DOPPLERGEIST_CPA_BACKEND=python` before starting the process to select it explicitly.+Checkpoints record the active backend. Existing implementation-mismatch checks+require a new output directory for ordinary checkpoints from older code.+The separately supported legacy continuation requires Python ridge, acoustic,+and CPA backends.++### Batch the unchanged native ridge fit++The native ridge extension now accepts a matrix of clock trials. It reuses scratch+storage and releases the GIL once per batch. Each row keeps the existing six IRLS+iterations, pairwise reductions, box constraints, and output arithmetic.+Custom scalar hooks, old extension binaries, unusual inputs, and failed probes+retain the established fallback. Native output may not overlap its inputs.++The dense replay uses 5,421 native batches for 227,378 trial rows, plus 862 scalar+fits. Batching alone measured a modest 1.06x dense improvement; its sparse replay+was approximately neutral. The main sieve gain comes from doing less impossible+work before decoding.++## Reproduce and validate++Build the optional extensions with `CC=gcc python setup.py build_ext --inplace`.+Use the recorded-input preparation in `.github/workflows/real-performance.yml`;+it restores public observations and creates `benchmark-inputs/dense-query.json`.+Build the same existing native extensions in the pinned baseline checkout.++```bash+OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 \+python scripts/benchmark_prepared_measurements.py compare \+ --baseline-source /path/to/baseline/src --current-source src \+ --fixture benchmark-inputs/dense-query.json \+ --index benchmark-inputs/reference-region.sqlite \+ --include-track-query --clock-partition-query \+ --require-current-native --require-current-native-acoustic \+ --require-current-native-cpa --repeats 3 --output results/flight-sieve++python scripts/benchmark_real_cpa_rows.py \+ --baseline-source /path/to/baseline/src --points 1 32 128 1024 4096 \+ --repeats 5 --iterations 10 --skip-reuse-comparison \+ --output results/flight-cpa.json+```++Use one available CPU affinity for both arms when comparing elapsed time.+The normal comparison requires exact complete output equality. Partition-query+mode also requires exact scored outcomes, rankings, trial evidence, and retention.+Only declared early-work counts may decrease; each removed unsupported interval+must be outside the query. Repeated outputs must be deterministic within each arm.+Profiler runs and numerical wrapper instrumentation cannot produce speed claims.++The added recorded-input tests cover native arithmetic, buffer overlap, fallback+behavior, outward clock boundaries, complete-track preservation, serial/worker+query integration, and checkpoint backend identity. No generated recordings or+aircraft tracks are used. The remaining dominant cost is association scoring.diff --git a/scripts/benchmark_prepared_measurements.py b/scripts/benchmark_prepared_measurements.pyindex 038c87a..1f6720e 100644--- a/scripts/benchmark_prepared_measurements.py+++ b/scripts/benchmark_prepared_measurements.py@@ -7,6 +7,7 @@ and identical observed inputs. This script generates no audio or trajectories.from __future__ import annotationsimport argparse+from collections import Counterfrom dataclasses import asdictimport jsonimport math@@ -44,9 +45,15 @@ def canonical(value):allow_nan=True, default=json_scalar)-def query_tracks(index, points, events, budget, scoring, bounds, time_range):+def query_tracks(index, points, events, budget, scoring, bounds, time_range, *,+ anchor_window=None, clock_partition_query=False):"""Count actual index traversal while exercising the production overload path."""- from dopplergeist.tiled_search import _tracks_for_points+ from dopplergeist import tiled_search++ requested_time_range = list(time_range)+ narrow_query = getattr(tiled_search, "_partition_query_time_range", None)+ if clock_partition_query and callable(narrow_query):+ time_range = narrow_query(time_range, events, scoring, {}, anchor_window)if "elev_m" not in points:raise ValueError("The recorded receiver hypothesis must include its elevation")@@ -63,10 +70,13 @@ def query_tracks(index, points, events, budget, scoring, bounds, time_range):yield trackradius = max(event["slant_range_m"][1] for event in events)- tracks, points = _tracks_for_points(+ tracks, points = tiled_search._tracks_for_points([CountedIndex()], bounds, time_range, budget, points, None, radius,scoring.get("c", 343.0))accounting = {+ "requested_time_range_s": requested_time_range,+ "effective_time_range_s": list(time_range),+ "clock_partition_time_narrowed": list(time_range) != requested_time_range,"queried_track_ids": sorted(inventory),"queried_trajectory_samples": sum(inventory.values()),"selected_track_ids": [track.id for track in tracks],@@ -75,6 +85,72 @@ def query_tracks(index, points, events, budget, scoring, bounds, time_range):return tracks, points, operations, accounting+_PARTITION_WORK_COUNTS = {"cpa_rows", "per_event_survivors", "work_units", "unsupported_cpa_tracks"}+++def partition_outcomes(payload):+ """Project only the explicitly allowed early-work diagnostics out of parity.++ Scored candidate payloads, rankings, trial evidence, support during scoring,+ retention records/bounds/stages, and all other counts remain exact checks.+ The removed fields receive separate monotonicity and time-exclusion checks.+ """+ return {key: ({name: value for name, value in item.items()+ if name not in _PARTITION_WORK_COUNTS} if key == "counts" else item)+ for key, item in payload.items() if key != "track_query_accounting"}+++def verify_partition_work(baseline, current):+ """Explain every permitted diagnostic change; reject unexplained omissions."""+ before, after = (payload["track_query_accounting"] for payload in (baseline, current))+ if before["requested_time_range_s"] != after["requested_time_range_s"]:+ raise AssertionError("Partition comparison changed the requested source interval")+ lower, upper = after["effective_time_range_s"]+ broad_lower, broad_upper = before["effective_time_range_s"]+ if not broad_lower <= lower <= upper <= broad_upper:+ raise AssertionError("Partition query did not conservatively narrow the original query")+ if not set(after["queried_track_ids"]) <= set(before["queried_track_ids"]):+ raise AssertionError("Partition query introduced unobserved source tracks")+ if after["queried_trajectory_samples"] > before["queried_trajectory_samples"]:+ raise AssertionError("Partition query increased source sample traversal")+ old_counts, new_counts = baseline["counts"], current["counts"]+ for field in ("cpa_rows", "work_units"):+ if new_counts.get(field, 0) > old_counts.get(field, 0):+ raise AssertionError(f"Partition unexpectedly increased {field}")+ old_survivors = old_counts.get("per_event_survivors", [])+ new_survivors = new_counts.get("per_event_survivors", [])+ if len(old_survivors) != len(new_survivors) or any(+ new > old for old, new in zip(old_survivors, new_survivors)):+ raise AssertionError("Partition unexpectedly changed event filtering")+ old_support = {canonical(item): item for item in old_counts.get("unsupported_cpa_tracks", [])}+ old_multiplicity = Counter(canonical(item) for item in old_counts.get("unsupported_cpa_tracks", []))+ new_multiplicity = Counter(canonical(item) for item in new_counts.get("unsupported_cpa_tracks", []))+ if not new_multiplicity <= old_multiplicity:+ raise AssertionError("Partition introduced or changed an unsupported-track report")+ removed = old_multiplicity - new_multiplicity+ for identity in removed:+ lo, hi = old_support[identity]["emission_time_range_s"]+ if lo <= upper and hi >= lower:+ raise AssertionError("Partition dropped an unsupported track touching the feasible emission interval")+ return {+ "requested_time_range_s": before["requested_time_range_s"],+ "effective_time_range_s": after["effective_time_range_s"],+ "excluded_unsupported_tracks_proved_outside_time_range": removed.total(),+ "baseline": {"queried_tracks": len(before["queried_track_ids"]),+ "queried_samples": before["queried_trajectory_samples"],+ "selected_tracks": len(before["selected_track_ids"]),+ "cpa_rows": old_counts["cpa_rows"],+ "unsupported_cpa_tracks": old_multiplicity.total(),+ "association_work_units": old_counts.get("work_units")},+ "current": {"queried_tracks": len(after["queried_track_ids"]),+ "queried_samples": after["queried_trajectory_samples"],+ "selected_tracks": len(after["selected_track_ids"]),+ "cpa_rows": new_counts["cpa_rows"],+ "unsupported_cpa_tracks": new_multiplicity.total(),+ "association_work_units": new_counts.get("work_units")},+ }++def prepare(args):import pandas as pdfrom dopplergeist.data_access import FlightIndex@@ -182,16 +258,24 @@ def worker(args):if Path(dopplergeist.__file__).resolve().parent != source / "dopplergeist":raise RuntimeError("Comparison imported a different source tree")native_fit = getattr(matching, "_native_fit_irls", None)+ native_module = getattr(matching, "_native_ridge_module", None)+ native_statistics = getattr(native_module, "fit_statistics", None)ridge_backend = getattr(matching, "RIDGE_FIT_BACKEND", "python-irls-v1")acoustic_backend = getattr(geometry, "ACOUSTIC_BACKEND", "python-acoustic-v1")try:from dopplergeist.native_acoustic import _native as native_acousticexcept ImportError:native_acoustic = None+ try:+ from dopplergeist.native_cpa import _native as native_cpa+ except ImportError:+ native_cpa = Noneif args.require_native and not callable(native_fit):raise RuntimeError("The selected source's native ridge fitter is unavailable")if args.require_native_acoustic and native_acoustic is None:raise RuntimeError("The selected source's native acoustic solver is unavailable")+ if args.require_native_cpa and native_cpa is None:+ raise RuntimeError("The selected source's native CPA reducer is unavailable")fixture = json.loads(args.fixture.read_text())index = FlightIndex(args.index or fixture["real_index"])if index.report["input_hash"] != fixture["index_input_hash"]:@@ -278,11 +362,17 @@ def worker(args):all_rows.append(result)return result- matching._fit_ridge_uncached, Track.acoustic = counted_fit, counted_acoustic+ if args.instrument_numerics:+ matching._fit_ridge_uncached, Track.acoustic = counted_fit, counted_acoustic+ else:+ for name in ("scalar_fit_calls", "acoustic_calls", "native_fit_calls", "native_fit_successes"):+ operations[name] = Nonetiled_search.score_candidate = captured_score- if callable(native_fit):+ native_before = native_statistics() if callable(native_statistics) else None+ if args.instrument_numerics and callable(native_fit) and native_before is None:matching._native_fit_irls = counted_nativenative_acoustic_before = native_acoustic.successful_calls() if native_acoustic is not None else 0+ native_cpa_before = native_cpa.successful_calls() if native_cpa is not None else 0query_accounting, query_operations, execution_stages = None, None, {}began, cpu = time.perf_counter(), time.process_time()try:@@ -290,20 +380,23 @@ def worker(args):query_began, query_cpu = time.perf_counter(), time.process_time()tracks, points, query_operations, query_accounting = query_tracks(index, points, events, budget, fixture["scoring"],- fixture["query_bounds"], fixture["query_time_s"])+ fixture["query_bounds"], fixture["query_time_s"],+ anchor_window=fixture.get("anchor_window_s"),+ clock_partition_query=args.clock_partition_query)execution_stages["track_query_and_distance_sieve"] = {"wall_seconds": time.perf_counter() - query_began,"cpu_seconds": time.process_time() - query_cpu,}- if query_accounting["selected_track_ids"] != fixture["track_ids"]:- raise AssertionError("The real query changed the ordered selected-track inventory")- if query_accounting["selected_trajectory_samples"] != fixture["counts"]["trajectory_samples"]:- raise AssertionError("The real query changed the selected trajectory sample count")- prepared_query = fixture.get("track_query_preparation")- if prepared_query is not None:- if (len(query_accounting["queried_track_ids"]) != prepared_query["queried_trajectories"]- or query_accounting["queried_trajectory_samples"] != prepared_query["queried_trajectory_samples"]):- raise AssertionError("The real query changed the complete observed-track inventory")+ if not query_accounting["clock_partition_time_narrowed"]:+ if query_accounting["selected_track_ids"] != fixture["track_ids"]:+ raise AssertionError("The real query changed the ordered selected-track inventory")+ if query_accounting["selected_trajectory_samples"] != fixture["counts"]["trajectory_samples"]:+ raise AssertionError("The real query changed the selected trajectory sample count")+ prepared_query = fixture.get("track_query_preparation")+ if prepared_query is not None:+ if (len(query_accounting["queried_track_ids"]) != prepared_query["queried_trajectories"]+ or query_accounting["queried_trajectory_samples"] != prepared_query["queried_trajectory_samples"]):+ raise AssertionError("The real query changed the complete observed-track inventory")scoring_began, scoring_cpu = time.perf_counter(), time.process_time()partition = ({"anchor_window": fixture["anchor_window_s"]}if "anchor_window_s" in fixture else {})@@ -321,31 +414,59 @@ def worker(args):tiled_search.score_candidate = original_scoreif callable(native_fit):matching._native_fit_irls = native_fit+ if native_before is not None:+ native_after = native_statistics()+ delta = {key: native_after[key] - native_before[key] for key in native_before}+ operations["native_fit_calls"] = delta["scalar_calls"] + delta["batch_rows"]+ operations["native_fit_successes"] = delta["scalar_successes"] + delta["batch_successful_rows"]+ operations["native_batch_calls"] = delta["batch_calls"]+ operations["native_batch_rows"] = delta["batch_rows"]if native_acoustic is not None:operations["native_acoustic_successes"] = native_acoustic.successful_calls() - native_acoustic_before+ if native_cpa is not None:+ operations["native_cpa_successes"] = native_cpa.successful_calls() - native_cpa_beforefor owner, attribute, function in reversed(phase_patches):setattr(owner, attribute, function)expected_counts = fixture.get("expected_counts")if expected_counts is not None:actual_counts = {"cpa_rows": counts["cpa_rows"],"associations": counts["candidates"], "retained": len(ranked)}- if actual_counts != expected_counts:+ changed_work_only = (args.clock_partition_query+ and actual_counts["cpa_rows"] <= expected_counts["cpa_rows"]+ and all(actual_counts[key] == expected_counts[key] for key in ("associations", "retained")))+ if actual_counts != expected_counts and not changed_work_only:raise AssertionError(f"Recorded workload counts changed: {actual_counts}")- # macOS reports bytes and Linux reports KiB. Keep scoring usable where- # resource or its unit convention is unavailable, without guessing a value.+ # Linux ru_maxrss can include the comparison parent's high-water mark+ # inherited across fork/exec. VmHWM belongs to this worker's address space.+ # Keep unknown platforms usable without guessing a memory unit or scope.peak_rss_mib = None- if resource is not None and (sys.platform == "darwin" or sys.platform.startswith("linux")):- rss_divisor = 1024 * 1024 if sys.platform == "darwin" else 1024- peak_rss_mib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rss_divisor+ peak_rss_method = "Unavailable on this platform"+ if sys.platform.startswith("linux"):+ try:+ for line in Path("/proc/self/status").read_text().splitlines():+ if line.startswith("VmHWM:"):+ fields = line.split()+ if len(fields) == 3 and fields[2] == "kB":+ peak_rss_mib = int(fields[1]) / 1024+ peak_rss_method = "/proc/self/status VmHWM after exec"+ break+ except (OSError, ValueError):+ pass+ elif resource is not None and sys.platform == "darwin":+ peak_rss_mib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1024 * 1024)+ peak_rss_method = "resource.getrusage(RUSAGE_SELF).ru_maxrss"checksum = ledger.checksum(stage)if not ledger.stage_valid(stage, checksum):raise AssertionError("The scoring pass produced an invalid retention ledger")if not operations["scored_candidates"]:raise AssertionError("The benchmark did not score any real associations")- if args.require_native and not operations["native_fit_successes"]:+ if (args.require_native and (args.instrument_numerics or native_before is not None)+ and not operations["native_fit_successes"]):raise AssertionError("No successful native ridge fit occurred inside the timed scoring call")if args.require_native_acoustic and not operations["native_acoustic_successes"]:raise AssertionError("No successful native acoustic solve occurred inside the timed scoring call")+ if args.require_native_cpa and not operations.get("native_cpa_successes"):+ raise AssertionError("No successful native CPA reduction occurred inside the timed scoring call")with ledger.connect() as database:retention = {name: database.execute(f"SELECT * FROM {name} ORDER BY 1,2").fetchall()for name in ("records", "bounds", "stages")}@@ -354,11 +475,18 @@ def worker(args):if query_accounting is not None:results["track_query_accounting"] = query_accountingwrite_new(args.output / "results.json", results)+ diagnostic = args.instrument_numerics or args.stage_timing or sys.getprofile() is not Nonesummary = {"wall_seconds": elapsed, "cpu_seconds": cpu_seconds,"load_seconds_excluded": load_seconds, "operations": operations,+ "numerical_functions_canonical": not (args.instrument_numerics or args.stage_timing),+ "performance_timing_eligible": not diagnostic,+ "ridge_execution_check": ("In-module successful-call counters" if native_before is not None else+ "Wrapped successful-call counters" if args.instrument_numerics else+ "Backend availability; successful-call count unmeasured"),"ridge_fit_backend": ridge_backend, "acoustic_backend": acoustic_backend,"native_extension_available": callable(native_fit),"peak_rss_mib": peak_rss_mib,+ "peak_rss_measurement": peak_rss_method,"peak_rss_scope": ("Worker lifetime through scoring, including imports and input loading"if peak_rss_mib is not None else "Unavailable on this platform"),"retained": len(ranked), "independent_groups": len(events), "scope": fixture["scope"]}@@ -366,6 +494,7 @@ def worker(args):summary["anchor_window_s"] = fixture["anchor_window_s"]if args.include_track_query:summary["track_query_operations"] = query_operations+ summary["clock_partition_time_narrowed"] = query_accounting["clock_partition_time_narrowed"]summary["execution_stage_seconds"] = execution_stagessummary["timing_scope"] = ("Initial local index traversal and conservative distance sieve, followed by complete "@@ -384,7 +513,7 @@ def compare(args):environment = dict(os.environ)for variable in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS"):environment[variable] = "1"- runs, expected = [], None+ runs, expected, source_payloads, source_canonical = [], None, {}, {}for repetition in range(args.repeats):order = ("baseline", "current") if repetition % 2 == 0 else ("current", "baseline")for name in order:@@ -397,16 +526,28 @@ def compare(args):command.append("--stage-timing")if args.include_track_query:command.append("--include-track-query")+ if args.clock_partition_query:+ command.append("--clock-partition-query")+ if args.instrument_numerics:+ command.append("--instrument-numerics")if name == "current" and args.require_current_native:command.append("--require-native")if name == "current" and args.require_current_native_acoustic:command.append("--require-native-acoustic")+ if name == "current" and args.require_current_native_cpa:+ command.append("--require-native-cpa")subprocess.run(command, check=True, env=environment, timeout=args.timeout_s)- payload = canonical(json.loads((output / "results.json").read_text()))+ result = json.loads((output / "results.json").read_text())+ complete = canonical(result)+ if name in source_canonical and source_canonical[name] != complete:+ raise AssertionError(f"Repeated source output was not deterministic: {output}")+ source_canonical[name] = complete+ source_payloads[name] = result+ payload = canonical(partition_outcomes(result) if args.clock_partition_query else result)if expected is None:expected = payloadelif payload != expected:- raise AssertionError(f"Exact selected-track, candidate, diagnostic, support, or retention mismatch: {output}")+ raise AssertionError(f"Exact candidate, diagnostic, support, or retention mismatch: {output}")summary = json.loads((output / "summary.json").read_text())runs.append({"source": name, "repetition": repetition + 1, **summary})medians = {}@@ -415,12 +556,24 @@ def compare(args):for metric in ("wall_seconds", "cpu_seconds", "peak_rss_mib"):values = [row[metric] for row in runs if row["source"] == name and row[metric] is not None]medians[name][metric] = statistics.median(values) if values else None- report = {"identical_results": True, "repeats": args.repeats,+ report = {"identical_results": source_canonical["baseline"] == source_canonical["current"],+ "identical_scored_candidates_rankings_and_retention": True, "repeats": args.repeats,"fixture": str(args.fixture.resolve()),"sources": {name: str(path) for name, path in sources.items()}, "medians": medians,"speedup": {metric: medians["baseline"][metric] / medians["current"][metric]for metric in ("wall_seconds", "cpu_seconds")},"scope": runs[0]["scope"], "runs": runs}+ if args.clock_partition_query:+ report["clock_partition_query_verification"] = verify_partition_work(+ source_payloads["baseline"], source_payloads["current"])+ report["equality_scope"] = (+ "Exact scored candidate payloads, rankings, trial/support evidence and retention records. "+ "Early query/CPA/event/work counts may decrease; every removed unsupported track's "+ "complete emission interval must miss the proved partition interval.")+ report["performance_timing_eligible"] = all(row["performance_timing_eligible"] for row in runs)+ if not report["performance_timing_eligible"]:+ report["diagnostic_ratio"] = report["speedup"]+ report["speedup"] = Noneif args.include_track_query:report["timing_scope"] = runs[0]["timing_scope"]report["track_query_included"] = True@@ -466,19 +619,28 @@ def main():help="Collect inclusive phase times in a separate diagnostic run")comparison.add_argument("--include-track-query", action="store_true",help="Include initial index traversal and the production overload sieve in each timed worker")+ comparison.add_argument("--clock-partition-query", action="store_true",+ help="Exercise the production clock-partition query bound where available; verify exact outcomes and explain reduced work")+ comparison.add_argument("--instrument-numerics", action="store_true",+ help="Diagnostic only: wrap numerical calls for counters; changes execution overhead")comparison.add_argument("--require-current-native", action="store_true",help="Require successful native ridge fits inside each current scoring run")comparison.add_argument("--require-current-native-acoustic", action="store_true",help="Require successful native acoustic solves inside each current scoring run")+ comparison.add_argument("--require-current-native-cpa", action="store_true",+ help="Require successful native CPA reductions inside each current scoring run")internal = commands.add_parser("_worker", help=argparse.SUPPRESS)internal.add_argument("--source", required=True, type=Path)internal.add_argument("--fixture", required=True, type=Path)internal.add_argument("--index", type=Path)internal.add_argument("--output", required=True, type=Path)internal.add_argument("--stage-timing", action="store_true", help=argparse.SUPPRESS)+ internal.add_argument("--instrument-numerics", action="store_true", help=argparse.SUPPRESS)internal.add_argument("--include-track-query", action="store_true", help=argparse.SUPPRESS)+ internal.add_argument("--clock-partition-query", action="store_true", help=argparse.SUPPRESS)internal.add_argument("--require-native", action="store_true", help=argparse.SUPPRESS)internal.add_argument("--require-native-acoustic", action="store_true", help=argparse.SUPPRESS)+ internal.add_argument("--require-native-cpa", action="store_true", help=argparse.SUPPRESS)args = parser.parse_args()if args.command == "prepare":if args.clock_half_width_s < 0:@@ -492,6 +654,10 @@ def main():elif args.command == "compare":if args.repeats < 1 or args.timeout_s <= 0:parser.error("--repeats and --timeout-s must be positive")+ if args.clock_partition_query:+ fixture = json.loads(args.fixture.read_text())+ if not args.include_track_query or "anchor_window_s" not in fixture:+ parser.error("--clock-partition-query requires --include-track-query and a recorded anchor partition")compare(args)else:worker(args)diff --git a/scripts/benchmark_real_cpa_rows.py b/scripts/benchmark_real_cpa_rows.pyindex 03d74d5..bcbbb88 100644--- a/scripts/benchmark_real_cpa_rows.py+++ b/scripts/benchmark_real_cpa_rows.py@@ -139,18 +139,26 @@ def main():parser.add_argument("--output", type=Path, required=True)parser.add_argument("--repeats", type=int, default=5)parser.add_argument("--iterations", type=int, default=20)+ parser.add_argument("--points", type=int, nargs="+", default=[1, 32, 128])+ parser.add_argument("--skip-reuse-comparison", action="store_true",+ help="Skip the separate historical preparation-reuse comparison")args = parser.parse_args()if args.output.exists():parser.error("Use a new output path")if args.repeats < 1 or args.iterations < 1:parser.error("repeats and iterations must be positive")+ if any(count < 1 for count in args.points):+ parser.error("points must be positive")path = args.baseline_source / "dopplergeist/search.py"spec = importlib.util.spec_from_file_location("dopplergeist._cpa_before", path)baseline = importlib.util.module_from_spec(spec)sys.modules[spec.name] = baselinespec.loader.exec_module(baseline)tracks = recorded_tracks()- workloads = {str(count): recorded_points(tracks, count) for count in (1, 32, 128)}+ workloads = {str(count): recorded_points(tracks, count) for count in args.points}+ # Recent baselines already support the bounded row path. Comparing their+ # per-track DataFrames against current rows would remeasure an older change.+ baseline_row_mode = hasattr(baseline, "_cpa_rows")checks = 0for points in workloads.values():wide_ids = pd.concat([points.iloc[:1], points.iloc[:1]], ignore_index=True)@@ -175,12 +183,12 @@ def main():search.cpa_table(supported, candidate_points, radius, c, step, chunk_size),check_exact=True)checks += 1- timings = []+ timings, raw_timings = [], []for name, points in workloads.items():runs = {"baseline": [], "current": []}final = Nonefor repeat in range(args.repeats):- arms = [("baseline", baseline, False), ("current", search, True)]+ arms = [("baseline", baseline, baseline_row_mode), ("current", search, True)]if repeat % 2:arms.reverse()for label, module, row_mode in arms:@@ -194,8 +202,41 @@ def main():"unsupported_tracks": len(final[1]), "iterations_per_trial": args.iterations,"cpu_s": runs, "median_cpu_s": medians,"speedup": medians["baseline"]/medians["current"]})+ if hasattr(baseline, "_PreparedCPATrack"):+ observers = ecef(points.lat.to_numpy(), points.lon.to_numpy(), points.elev_m.to_numpy())+ point_ids = points.point_id.to_numpy(dtype=int)+ prepared = {"baseline": [], "current": []}+ for label, module in (("baseline", baseline), ("current", search)):+ for track in tracks:+ try:+ if not track.outside_observer_radius(observers, 20000., 343.):+ prepared[label].append(module._PreparedCPATrack(track, 343., 2.))+ except AcousticSupportError:+ pass++ def raw_rows(label):+ return [row for item in prepared[label]+ for row in item.raw_rows(observers, point_ids, 20000.)]++ # Warm each lazy tree once; sampling/tree preparation is included+ # in the complete-stage timing above, not this loop measurement.+ old_rows, new_rows = raw_rows("baseline"), raw_rows("current")+ assert old_rows == new_rows, "Raw CPA proposals or order changed"+ checks += 1+ raw_runs = {"baseline": [], "current": []}+ for repeat in range(args.repeats):+ for label in (("current", "baseline") if repeat % 2 else ("baseline", "current")):+ started = time.process_time()+ for _ in range(args.iterations):+ raw_rows(label)+ raw_runs[label].append(time.process_time()-started)+ raw_medians = {label: statistics.median(values) for label, values in raw_runs.items()}+ raw_timings.append({"receiver_points": int(name), "raw_cpa_rows": len(new_rows),+ "iterations_per_trial": args.iterations, "cpu_s": raw_runs,+ "median_cpu_s": raw_medians,+ "speedup": raw_medians["baseline"]/raw_medians["current"]})chunked_timings = []- for size in (1024, 4096):+ for size in (() if args.skip_reuse_comparison else (1024, 4096)):points = recorded_points(tracks, size)before, old_counts = counted_chunked_stage(tracks, points, False)after, new_counts = counted_chunked_stage(tracks, points, True)@@ -223,6 +264,9 @@ def main():"cpu_affinity": sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None},"observed_samples": sum(len(track.time) for track in tracks),"exact_comparisons": checks, "timings": timings,+ "baseline_row_mode": baseline_row_mode,+ "raw_rows_scope": "Same sorted spatial query, spline endpoint solves, speed evaluation and row assembly; sampled track and lazy tree already prepared; internal chunks of at most 512 observers",+ "raw_rows_timings": raw_timings,"chunked_comparison": "Same current code and recorded inputs, trajectory/tree reuse disabled versus enabled; normal 100000-row budget","chunked_timings": chunked_timings}args.output.parent.mkdir(parents=True, exist_ok=True)diff --git a/setup.py b/setup.pyindex d8f2687..d5b8184 100644--- a/setup.py+++ b/setup.py@@ -21,4 +21,13 @@ acoustic_extension = Extension(libraries=[] if os.name == "nt" else ["m"],)-setup(ext_modules=[extension, acoustic_extension])+cpa_extension = Extension(+ "dopplergeist._native_cpa",+ ["src/dopplergeist/_native_cpa.c"],+ optional=True,+ extra_compile_args=["/O2", "/fp:strict"] if os.name == "nt" else+ ["-O3", "-ffp-contract=off", "-fno-fast-math"],+ libraries=[] if os.name == "nt" else ["m"],+)++setup(ext_modules=[extension, acoustic_extension, cpa_extension])diff --git a/src/dopplergeist/_native_cpa.c b/src/dopplergeist/_native_cpa.cnew file mode 100644index 0000000..f768c7e--- /dev/null+++ b/src/dopplergeist/_native_cpa.c@@ -0,0 +1,184 @@+/* Optional exact-order CPA neighbour reduction. No NumPy C headers.+ * Keep the Python spatial query, endpoint spline solver and row construction.+ * This loop removes the expanded observer indices and temporary N-by-3 arrays.+ */+#define PY_SSIZE_T_CLEAN+#include <Python.h>+#include <fenv.h>+#include <math.h>+#include <stdint.h>+#include <string.h>++typedef struct { unsigned long long successful_calls; } module_state;++/* NumPy reduces the three coordinates in order. Do not reassociate or fuse+ * these additions and multiplications; setup.py disables both optimizations.+ */+static double sum3(double a, double b, double c) {+ return ((0.0 + a) + b) + c;+}++static double distance2(const double *position, const double *observer) {+ double x = position[0] - observer[0];+ double y = position[1] - observer[1];+ double z = position[2] - observer[2];+ return sum3(x*x, y*y, z*z);+}++static int project(const double *first, const double *last,+ const double *observer, double *fraction, double *squared) {+ double delta[3], residual[3], projected[3];+ for (int j = 0; j < 3; ++j) {+ delta[j] = last[j] - first[j];+ residual[j] = first[j] - observer[j];+ }+ double length2 = sum3(delta[0]*delta[0], delta[1]*delta[1], delta[2]*delta[2]);+ double dot = sum3(residual[0]*delta[0], residual[1]*delta[1], residual[2]*delta[2]);+ double denominator = length2 < 1e-12 ? 1e-12 : length2;+ double f = -dot / denominator;+ if (!isfinite(length2) || !isfinite(dot) || !isfinite(f)) return 0;+ if (f < 0.0) f = 0.0;+ else if (f > 1.0) f = 1.0;+ for (int j = 0; j < 3; ++j)+ projected[j] = residual[j] + f*delta[j];+ *squared = sum3(projected[0]*projected[0], projected[1]*projected[1],+ projected[2]*projected[2]);+ *fraction = f;+ return isfinite(*squared);+}++static int reduce_graph(const double *position, Py_ssize_t position_count,+ const double *observers, Py_ssize_t observer_count,+ const Py_ssize_t *counts, const Py_ssize_t *sample,+ Py_ssize_t sample_count, Py_ssize_t *endpoints,+ Py_ssize_t *indices, double *values, Py_ssize_t capacity,+ Py_ssize_t *sizes) {+ Py_ssize_t base = 0;+ for (Py_ssize_t oi = 0; oi < observer_count; ++oi) {+ Py_ssize_t count = counts[oi];+ if (count < 0 || count > sample_count-base) return 0;+ const Py_ssize_t *samples = sample + base;+ const double *observer = observers + 3*oi;+ for (int j = 0; j < 3; ++j) if (!isfinite(observer[j])) return 0;+ for (Py_ssize_t j = 0; j < count; ++j)+ if (samples[j] < 0 || samples[j] >= position_count) return 0;+ base += count;+ if (!count) continue;+ double previous = distance2(position + 3*samples[0], observer);+ if (!isfinite(previous)) return 0;+ if (count == 1) continue;+ double current = distance2(position + 3*samples[1], observer);+ if (!isfinite(current)) return 0;+ if (samples[0] == 0 && samples[1] == 1 && previous <= current)+ endpoints[sizes[0]++] = oi;+ for (Py_ssize_t j = 1; j < count-1; ++j) {+ double following = distance2(position + 3*samples[j+1], observer);+ if (!isfinite(following)) return 0;+ Py_ssize_t k = samples[j];+ if (samples[j-1] == k-1 && samples[j+1] == k+1+ && current <= previous && current < following) {+ double fraction[2], squared[2];+ if (!project(position + 3*(k-1), position + 3*k, observer,+ fraction, squared)+ || !project(position + 3*k, position + 3*(k+1), observer,+ fraction+1, squared+1)) return 0;+ /* np.argmin selects the first edge on an exact tie. */+ int edge = squared[1] < squared[0];+ Py_ssize_t out = sizes[2]++;+ if (out >= capacity) return 0;+ indices[out] = oi;+ indices[capacity+out] = edge ? k : k-1;+ values[out] = fraction[edge];+ values[capacity+out] = squared[edge];+ }+ previous = current;+ current = following;+ }+ if (samples[count-1] == position_count-1+ && samples[count-2] == position_count-2 && current <= previous)+ endpoints[observer_count+sizes[1]++] = oi;+ }+ return base == sample_count;+}++static int view_array(PyObject *object, Py_buffer *view, int integer, int writable) {+ int flags = PyBUF_FORMAT | PyBUF_ND | PyBUF_STRIDES;+ if (writable) flags |= PyBUF_WRITABLE;+ if (PyObject_GetBuffer(object, view, flags) < 0) return 0;+ size_t size = integer ? sizeof(Py_ssize_t) : sizeof(double);+ if (view->itemsize != (Py_ssize_t)size || !view->format+ || !PyBuffer_IsContiguous(view, 'C')+ || (uintptr_t)view->buf % size != 0) return 0;+ if (!integer) return strcmp(view->format, "d") == 0;+ return strlen(view->format) == 1 && strchr("nliq", view->format[0]) != NULL;+}++static int matrix_shape(const Py_buffer *view, Py_ssize_t rows, Py_ssize_t columns) {+ return view->ndim == 2 && view->shape[0] == rows && view->shape[1] == columns;+}++static int disjoint(const Py_buffer *a, const Py_buffer *b) {+ uintptr_t first = (uintptr_t)a->buf, second = (uintptr_t)b->buf;+ if (a->len < 0 || b->len < 0+ || (uintptr_t)a->len > UINTPTR_MAX-first+ || (uintptr_t)b->len > UINTPTR_MAX-second) return 0;+ return !a->len || !b->len || first+(uintptr_t)a->len <= second+ || second+(uintptr_t)b->len <= first;+}++static PyObject *select_edges(PyObject *self, PyObject *args) {+ PyObject *objects[7];+ Py_buffer views[7] = {{0}};+ Py_ssize_t sizes[3] = {0, 0, 0};+ int success = 0;+ if (!PyArg_ParseTuple(args, "OOOOOOO", &objects[0], &objects[1], &objects[2],+ &objects[3], &objects[4], &objects[5], &objects[6])) return NULL;+ if (fegetround() != FE_TONEAREST) Py_RETURN_NONE;+ for (int i = 0; i < 7; ++i)+ if (!view_array(objects[i], views+i, i >= 2 && i <= 5, i >= 4)) goto done;+ for (int i = 4; i < 7; ++i)+ for (int j = 0; j < i; ++j)+ if (!disjoint(views+i, views+j)) goto done;+ if (views[0].ndim != 2 || views[0].shape[0] < 2 || views[0].shape[1] != 3+ || views[1].ndim != 2 || views[1].shape[1] != 3+ || views[2].ndim != 1 || views[2].shape[0] != views[1].shape[0]+ || views[3].ndim != 1+ || !matrix_shape(views+4, 2, views[1].shape[0])+ || !matrix_shape(views+5, 2, views[3].shape[0]/2)+ || !matrix_shape(views+6, 2, views[3].shape[0]/2)) goto done;+ /* Failed probes leave NumPy's warning/exception behavior to the reference+ * path. Do not leak the probe's floating-point flags into that fallback.+ */+ fenv_t previous_environment;+ if (fegetenv(&previous_environment) != 0) goto done;+ Py_BEGIN_ALLOW_THREADS+ if (feclearexcept(FE_ALL_EXCEPT) == 0) {+ success = reduce_graph(views[0].buf, views[0].shape[0], views[1].buf,+ views[1].shape[0], views[2].buf, views[3].buf,+ views[3].shape[0], views[4].buf, views[5].buf,+ views[6].buf, views[3].shape[0]/2, sizes);+ if (fetestexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW)) success = 0;+ }+ if (fesetenv(&previous_environment) != 0) success = 0;+ Py_END_ALLOW_THREADS+done:+ for (int i = 0; i < 7; ++i) if (views[i].obj) PyBuffer_Release(views+i);+ if (PyErr_Occurred()) PyErr_Clear();+ if (!success) Py_RETURN_NONE;+ ((module_state *)PyModule_GetState(self))->successful_calls++;+ return Py_BuildValue("nnn", sizes[0], sizes[1], sizes[2]);+}++static PyObject *successful_calls(PyObject *self, PyObject *unused) {+ return PyLong_FromUnsignedLongLong(((module_state *)PyModule_GetState(self))->successful_calls);+}++static PyMethodDef methods[] = {+ {"select", select_edges, METH_VARARGS, "Reduce a canonical sorted CPA neighbour graph."},+ {"successful_calls", successful_calls, METH_NOARGS, "Count successful native CPA reductions."},+ {NULL, NULL, 0, NULL}+};+static struct PyModuleDef module = {+ PyModuleDef_HEAD_INIT, "_native_cpa", NULL, sizeof(module_state), methods+};+PyMODINIT_FUNC PyInit__native_cpa(void) { return PyModule_Create(&module); }diff --git a/src/dopplergeist/_native_ridge.c b/src/dopplergeist/_native_ridge.cindex 6dfc964..b343ab3 100644--- a/src/dopplergeist/_native_ridge.c+++ b/src/dopplergeist/_native_ridge.c@@ -1,8 +1,14 @@#define PY_SSIZE_T_CLEAN#include <Python.h>#include <math.h>+#include <stdint.h>#include <string.h>+typedef struct {+ unsigned long long scalar_calls, scalar_successes;+ unsigned long long batch_calls, batch_rows, batch_successful_rows;+} ridge_state;+static intget_vector(PyObject *object, Py_buffer *view){@@ -151,43 +157,13 @@ source_fit(const Py_buffer *factor, const Py_buffer *u, const Py_buffer *y,}}-static PyObject *-fit_irls(PyObject *self, PyObject *args)+/* One numerical implementation serves scalar and batch entry points. The caller+ * owns the six scratch vectors, so a batch does not allocate them per clock. */+static int+fit_row(const Py_buffer *views, Py_ssize_t count, double lower, double upper,+ double cap, double sigma, Py_ssize_t independent_samples,+ double *effective, double *output){- PyObject *objects[4];- double lower, upper, cap, sigma;- Py_ssize_t independent_samples;- Py_buffer views[4] = {{0}};- int acquired = 0;- double *effective = NULL;- PyObject *result = NULL;-- if (!PyArg_ParseTuple(args, "OOOOddddn", &objects[0], &objects[1],- &objects[2], &objects[3], &lower, &upper, &cap,- &sigma, &independent_samples)) {- return NULL;- }- for (int i = 0; i < 4; ++i) {- if (get_vector(objects[i], &views[i]) < 0) goto done;- ++acquired;- }- Py_ssize_t count = views[0].shape[0];- if (count == 0 || views[1].shape[0] != count || views[2].shape[0] != count ||- views[3].shape[0] != count || !isfinite(lower) || !isfinite(upper) ||- lower > upper || !isfinite(cap) || cap < 0.0 || !isfinite(sigma) ||- sigma <= 0.0 || independent_samples < 1) {- PyErr_SetString(PyExc_ValueError, "invalid robust ridge-fit inputs");- goto done;- }- if ((size_t)count > SIZE_MAX / (6 * sizeof(double))) {- PyErr_NoMemory();- goto done;- }- effective = PyMem_Malloc((size_t)count * 6 * sizeof(double));- if (effective == NULL) {- PyErr_NoMemory();- goto done;- }int valid = 1;for (Py_ssize_t i = 0; i < count; ++i) {double fi = at(&views[0], i), ui = at(&views[1], i);@@ -199,18 +175,13 @@ fit_irls(PyObject *self, PyObject *args)}effective[i] = wi;}- if (!valid) {- Py_INCREF(Py_None);- result = Py_None;- goto done;- }+ if (!valid) return 0;double f0 = NAN, drift = NAN, loss = NAN, rmse = NAN, outlier_fraction = NAN;double *scratch = effective + count;double prior = cap == 0.0 ? 0.0 : scalar_square(2.0 * sigma / cap);double loss_scale = 2.0 * scalar_square(sigma);valid = 1;- Py_BEGIN_ALLOW_THREADSfor (int iteration = 0; iteration < 6; ++iteration) {source_fit(&views[0], &views[1], &views[2], effective, scratch, count,lower, upper, cap, prior, &f0, &drift);@@ -253,9 +224,60 @@ fit_irls(PyObject *self, PyObject *args)loss = -sqrt(loss + penalty);valid = isfinite(loss) && isfinite(rmse) && isfinite(outlier_fraction);}- Py_END_ALLOW_THREADS+ output[0] = loss;+ output[1] = f0;+ output[2] = drift;+ output[3] = rmse;+ output[4] = outlier_fraction;+ return valid;+}- if (valid) result = Py_BuildValue("(ddddd)", loss, f0, drift, rmse, outlier_fraction);+static PyObject *+fit_irls(PyObject *self, PyObject *args)+{+ ridge_state *state = PyModule_GetState(self);+ ++state->scalar_calls;+ PyObject *objects[4];+ double lower, upper, cap, sigma, output[5];+ Py_ssize_t independent_samples;+ Py_buffer views[4] = {{0}};+ int acquired = 0;+ double *effective = NULL;+ PyObject *result = NULL;++ if (!PyArg_ParseTuple(args, "OOOOddddn", &objects[0], &objects[1],+ &objects[2], &objects[3], &lower, &upper, &cap,+ &sigma, &independent_samples)) return NULL;+ for (int i = 0; i < 4; ++i) {+ if (get_vector(objects[i], &views[i]) < 0) goto done;+ ++acquired;+ }+ Py_ssize_t count = views[0].shape[0];+ if (count == 0 || views[1].shape[0] != count || views[2].shape[0] != count ||+ views[3].shape[0] != count || !isfinite(lower) || !isfinite(upper) ||+ lower > upper || !isfinite(cap) || cap < 0.0 || !isfinite(sigma) ||+ sigma <= 0.0 || independent_samples < 1) {+ PyErr_SetString(PyExc_ValueError, "invalid robust ridge-fit inputs");+ goto done;+ }+ if ((size_t)count > SIZE_MAX / (6 * sizeof(double))) {+ PyErr_NoMemory();+ goto done;+ }+ effective = PyMem_Malloc((size_t)count * 6 * sizeof(double));+ if (effective == NULL) {+ PyErr_NoMemory();+ goto done;+ }+ int valid;+ Py_BEGIN_ALLOW_THREADS+ valid = fit_row(views, count, lower, upper, cap, sigma, independent_samples,+ effective, output);+ Py_END_ALLOW_THREADS+ if (valid) {+ ++state->scalar_successes;+ result = Py_BuildValue("(ddddd)", output[0], output[1], output[2], output[3], output[4]);+ }else {Py_INCREF(Py_None);result = Py_None;@@ -267,6 +289,146 @@ done:return result;}+static int+get_matrix(PyObject *object, Py_buffer *view, int writable)+{+ int flags = PyBUF_FORMAT | PyBUF_ND | PyBUF_STRIDES;+ if (writable) flags |= PyBUF_WRITABLE;+ if (PyObject_GetBuffer(object, view, flags) < 0) return -1;+ if (view->ndim != 2 || view->itemsize != (Py_ssize_t)sizeof(double) ||+ view->shape == NULL || view->strides == NULL || view->format == NULL ||+ strcmp(view->format, "d") != 0 || !PyBuffer_IsContiguous(view, 'C')) {+ PyBuffer_Release(view);+ view->obj = NULL;+ PyErr_SetString(PyExc_TypeError, "expected a contiguous native float64 matrix");+ return -1;+ }+ return 0;+}++/* Include the holes between strided elements. A conservative span is enough to+ * prove that writing a result cannot change a later row's input. */+static int+buffer_span(const Py_buffer *view, uintptr_t *begin, uintptr_t *end)+{+ uintptr_t low = (uintptr_t)view->buf, high = low;+ if (view->buf == NULL || view->itemsize <= 0) return 0;+ for (int axis = 0; axis < view->ndim; ++axis) {+ if (view->shape[axis] <= 0 ||+ (view->suboffsets != NULL && view->suboffsets[axis] >= 0)) return 0;+ Py_ssize_t stride = view->strides[axis];+ uintptr_t magnitude = stride >= 0 ? (uintptr_t)stride :+ (uintptr_t)(-(stride + 1)) + 1;+ uintptr_t steps = (uintptr_t)(view->shape[axis] - 1);+ if (magnitude != 0 && steps > UINTPTR_MAX / magnitude) return 0;+ uintptr_t offset = steps * magnitude;+ if (stride < 0) {+ if (low < offset) return 0;+ low -= offset;+ } else {+ if (high > UINTPTR_MAX - offset) return 0;+ high += offset;+ }+ }+ if (high > UINTPTR_MAX - (uintptr_t)view->itemsize) return 0;+ *begin = low;+ *end = high + (uintptr_t)view->itemsize;+ return 1;+}++static PyObject *+fit_irls_batch(PyObject *self, PyObject *args)+{+ ridge_state *state = PyModule_GetState(self);+ ++state->batch_calls;+ PyObject *objects[6];+ Py_buffer views[6] = {{0}};+ double lower, upper, sigma;+ Py_ssize_t independent_samples;+ int acquired = 0, valid = 1;+ double *scratch = NULL;+ PyObject *result = NULL;+ if (!PyArg_ParseTuple(args, "OOOOOdddnO", &objects[0], &objects[1], &objects[2],+ &objects[3], &objects[4], &lower, &upper, &sigma,+ &independent_samples, &objects[5])) return NULL;+ for (int i = 0; i < 6; ++i) {+ int status = (i < 2 || i == 5) ? get_matrix(objects[i], &views[i], i == 5) :+ get_vector(objects[i], &views[i]);+ if (status < 0) goto done;+ ++acquired;+ }+ Py_ssize_t rows = views[0].shape[0], count = views[0].shape[1];+ if (rows == 0 || count == 0 || views[1].shape[0] != rows || views[1].shape[1] != count ||+ views[2].shape[0] != count || views[3].shape[0] != count || views[4].shape[0] != rows ||+ views[5].shape[0] != rows || views[5].shape[1] != 5 ||+ !isfinite(lower) || !isfinite(upper) || lower > upper ||+ !isfinite(sigma) || sigma <= 0.0 || independent_samples < 1) {+ PyErr_SetString(PyExc_ValueError, "invalid robust ridge-fit batch inputs");+ goto done;+ }+ uintptr_t output_begin, output_end;+ if (!buffer_span(&views[5], &output_begin, &output_end)) {+ PyErr_SetString(PyExc_ValueError, "unsupported robust ridge-fit output buffer");+ goto done;+ }+ for (int i = 0; i < 5; ++i) {+ uintptr_t input_begin, input_end;+ if (!buffer_span(&views[i], &input_begin, &input_end) ||+ (output_begin < input_end && input_begin < output_end)) {+ PyErr_SetString(PyExc_ValueError, "robust ridge-fit output must not overlap inputs");+ goto done;+ }+ }+ if ((size_t)count > SIZE_MAX / (6 * sizeof(double))) {+ PyErr_NoMemory();+ goto done;+ }+ scratch = PyMem_Malloc((size_t)count * 6 * sizeof(double));+ if (scratch == NULL) {+ PyErr_NoMemory();+ goto done;+ }+ Py_buffer row_views[4] = {views[0], views[1], views[2], views[3]};+ row_views[0].strides = views[0].strides + 1;+ row_views[1].strides = views[1].strides + 1;+ state->batch_rows += (unsigned long long)rows;+ Py_BEGIN_ALLOW_THREADS+ for (Py_ssize_t row = 0; row < rows; ++row) {+ double cap = at(&views[4], row), output[5];+ if (!isfinite(cap) || cap < 0.0) {+ valid = 0;+ break;+ }+ row_views[0].buf = (char *)views[0].buf + row * views[0].strides[0];+ row_views[1].buf = (char *)views[1].buf + row * views[1].strides[0];+ if (!fit_row(row_views, count, lower, upper, cap, sigma,+ independent_samples, scratch, output)) {+ valid = 0;+ break;+ }+ memcpy((char *)views[5].buf + row * views[5].strides[0], output, sizeof(output));+ }+ Py_END_ALLOW_THREADS+ if (valid) state->batch_successful_rows += (unsigned long long)rows;+ result = PyBool_FromLong(valid);+done:+ PyMem_Free(scratch);+ for (int i = 0; i < acquired; ++i) PyBuffer_Release(&views[i]);+ return result;+}++static PyObject *+fit_statistics(PyObject *self, PyObject *unused)+{+ ridge_state *state = PyModule_GetState(self);+ return Py_BuildValue("{sK,sK,sK,sK,sK}",+ "scalar_calls", state->scalar_calls,+ "scalar_successes", state->scalar_successes,+ "batch_calls", state->batch_calls,+ "batch_rows", state->batch_rows,+ "batch_successful_rows", state->batch_successful_rows);+}+/* This is a positive-only shortcut for the Python evidence validator. Any* noncanonical or invalid value returns False, so Python retains its original* error reason, ordering, subclass behavior and unbounded-integer semantics.@@ -367,13 +529,16 @@ valid_ridge_events(PyObject *self, PyObject *args)static PyMethodDef methods[] = {{"fit_irls", fit_irls, METH_VARARGS, "Run six robust ridge-fit iterations in C."},+ {"fit_irls_batch", fit_irls_batch, METH_VARARGS,+ "Run the same robust fit for contiguous clock rows with shared scratch storage."},+ {"fit_statistics", fit_statistics, METH_NOARGS, "Read native fit operation counters."},{"valid_ridge_events", valid_ridge_events, METH_VARARGS,"Recognize valid canonical ridge evidence; other inputs retain Python validation."},{NULL, NULL, 0, NULL}};static struct PyModuleDef module = {- PyModuleDef_HEAD_INIT, "_native_ridge", NULL, -1, methods+ PyModuleDef_HEAD_INIT, "_native_ridge", NULL, sizeof(ridge_state), methods};PyMODINIT_FUNCdiff --git a/src/dopplergeist/batch_ridge.py b/src/dopplergeist/batch_ridge.pyindex d9bc57a..f1caf0f 100644--- a/src/dopplergeist/batch_ridge.py+++ b/src/dopplergeist/batch_ridge.py@@ -185,12 +185,19 @@ def _first_event_fits_uncached(measurement, event, factors, emissions, *, legacyand factors.flags.aligned and emissions.flags.aligned):native_rows = []try:- for factor, normalized_time, bound in zip(factors, u, cap):- fit = native(factor, normalized_time, y, w, low, high,- float(bound), sigma, independent)- if fit is None:- break- native_rows.append(fit)+ batch = matching._native_fit_irls_batch+ if batch is not None and native is matching._CANONICAL_NATIVE_FIT_IRLS:+ output = np.empty((len(factors), 5), dtype=float)+ if batch(factors, u, y, w, cap, low, high, sigma, independent, output):+ native_rows = output+ else:+ # Preserve custom scalar hooks and older installed binaries.+ for factor, normalized_time, bound in zip(factors, u, cap):+ fit = native(factor, normalized_time, y, w, low, high,+ float(bound), sigma, independent)+ if fit is None:+ break+ native_rows.append(fit)if len(native_rows) == len(factors):score, alpha, beta, rmse, outlier = np.asarray(native_rows, dtype=float).Tif all(np.all(np.isfinite(value)) for value indiff --git a/src/dopplergeist/checkpoint_resume.py b/src/dopplergeist/checkpoint_resume.pyindex 5ff63b4..0e58d94 100644--- a/src/dopplergeist/checkpoint_resume.py+++ b/src/dopplergeist/checkpoint_resume.py@@ -33,7 +33,7 @@ def upgrade_legacy_job(directory):staged batching with the saved first-group arithmetic and fallback checks."""from .automatic import _audio_identity, _digest- from . import geometry, matching+ from . import geometry, matching, native_cpadirectory = Path(directory)search = directory / "search"@@ -47,6 +47,8 @@ def upgrade_legacy_job(directory):raise ValueError("Legacy checkpoint continuation requires the Python ridge fitter")if geometry.ACOUSTIC_BACKEND != "python-acoustic-v1":raise ValueError("Legacy checkpoint continuation requires the Python acoustic solver")+ if native_cpa.CPA_BACKEND != "python-cpa-v1":+ raise ValueError("Legacy checkpoint continuation requires Python closest-approach processing")if (old["scoring"].get("mode") != "robust_ridge" or old.get("coarse_floor_policy") != "per_tile"or old["scoring"].get("time_origin") is not Noneor old.get("filters", {}).get("association") != "subsets"):diff --git a/src/dopplergeist/matching.py b/src/dopplergeist/matching.pyindex 88afdcd..f3baa7c 100644--- a/src/dopplergeist/matching.py+++ b/src/dopplergeist/matching.py@@ -40,6 +40,8 @@ else:RIDGE_FIT_BACKEND = f"native-c-irls-v2-pairwise:{_native_binary_hash}"_native_validate_ridge_events = getattr(_native_ridge_module, "valid_ridge_events", None)+_native_fit_irls_batch = getattr(_native_ridge_module, "fit_irls_batch", None)+_CANONICAL_NATIVE_FIT_IRLS = _native_fit_irls_NATIVE_RIDGE_ERROR_POLICY = {"divide": "warn", "over": "warn", "under": "ignore", "invalid": "warn"}diff --git a/src/dopplergeist/native_cpa.py b/src/dopplergeist/native_cpa.pynew file mode 100644index 0000000..6eaad94--- /dev/null+++ b/src/dopplergeist/native_cpa.py@@ -0,0 +1,52 @@+"""Optional exact-order reduction of a bounded CPA neighbour graph."""+import os++import numpy as np+++_backend_choice = os.environ.get("DOPPLERGEIST_CPA_BACKEND", "auto")+if _backend_choice not in {"auto", "python"}:+ raise ValueError("DOPPLERGEIST_CPA_BACKEND must be 'auto' or 'python'")+_native = None+if _backend_choice == "auto":+ try:+ from . import _native_cpa as _native+ except ImportError:+ pass++CPA_BACKEND = "python-cpa-v1" if _native is None else "native-c-cpa-v1"+_ERROR_POLICY = {"divide": "warn", "over": "warn", "under": "ignore", "invalid": "warn"}+++def try_cpa_edges(position, observers, counts, sample):+ """Return endpoint candidates and projected minima, or use the reference.++ The caller owns one bounded, sorted radius-neighbour graph. The extension+ streams its distances and projects only the sampled minima; it does not+ change the spatial query, spline endpoint solver, or row selection rules.+ """+ if _native is None or np.geterr() != _ERROR_POLICY:+ return None+ for array, dtype in ((position, np.float64), (observers, np.float64),+ (counts, np.intp), (sample, np.intp)):+ if (type(array) is not np.ndarray or array.dtype != dtype+ or not array.flags.c_contiguous or not array.flags.aligned):+ return None+ if (position.ndim != 2 or position.shape[1] != 3+ or observers.ndim != 2 or observers.shape[1] != 3+ or counts.ndim != 1 or len(counts) != len(observers)+ or sample.ndim != 1):+ return None+ # Two consecutive samples cannot both satisfy d[k] < d[k+1]. Each+ # interior minimum therefore consumes at least two neighbour entries.+ capacity = sample.size // 2+ endpoints = np.empty((2, len(observers)), dtype=np.intp)+ indices = np.empty((2, capacity), dtype=np.intp)+ values = np.empty((2, capacity), dtype=np.float64)+ sizes = _native.select(position, observers, counts, sample, endpoints, indices, values)+ if sizes is None:+ return None+ first_count, last_count, minimum_count = sizes+ return ((endpoints[0, :first_count], endpoints[1, :last_count]),+ indices[0, :minimum_count], indices[1, :minimum_count],+ values[0, :minimum_count], values[1, :minimum_count])diff --git a/src/dopplergeist/parallel_tiles.py b/src/dopplergeist/parallel_tiles.pyindex 37e10df..05c66fb 100644--- a/src/dopplergeist/parallel_tiles.py+++ b/src/dopplergeist/parallel_tiles.py@@ -8,7 +8,7 @@ import pandas as pdfrom .data_access import CachedTerrain, DataUnavailable, FlightIndexfrom .finite_search import RetentionLedger-from .tiled_search import (_expanded_bounds, _sample_tile_terrain, _score, _tile_anchor_window,+from .tiled_search import (_expanded_bounds, _partition_query_time_range, _sample_tile_terrain, _score, _tile_anchor_window,_tile_coordinates, _tracks_for_points)@@ -146,10 +146,14 @@ def score_tile(stage, cells, score_floor=None):radius = max(event["slant_range_m"][1] for event in state["events"])points = _tile_coordinates(cells, budget.spacing_m, state["bounds"], state["crs"])tracks = []+ anchor = _tile_anchor_window(stage.removeprefix("coarse:"), state["start_window"],+ state["parameters"], state["filters"], state["events"])+ query_time = _partition_query_time_range(state["time_range"], state["events"], state["parameters"],+ state["filters"], anchor)if len(points):tracks, points = _tracks_for_points(state["indexes"], _expanded_bounds(cells, budget.spacing_m, radius, state["crs"]),- state["time_range"], budget, points, terrain, radius, state["parameters"].get("c", 343.0))+ query_time, budget, points, terrain, radius, state["parameters"].get("c", 343.0))if tracks and "elev_m" not in points:points = _sample_tile_terrain(points, terrain)sampled = "elev_m" in points@@ -161,8 +165,6 @@ def score_tile(stage, cells, score_floor=None):ledger = RetentionLedger(path, state["run_hash"], state["score_kind"])if len(points):kwargs = {"ledger": ledger, "stage": stage}- anchor = _tile_anchor_window(stage.removeprefix("coarse:"), state["start_window"],- state["parameters"], state["filters"], state["events"])if anchor is not None:kwargs["anchor_window"] = anchorif score_floor is not None:diff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.pyindex 54f689f..45430f1 100644--- a/src/dopplergeist/search.py+++ b/src/dopplergeist/search.py@@ -10,6 +10,7 @@ from scipy.spatial import cKDTreefrom .geometry import AcousticSupportError, eceffrom .matching import _native_validate_ridge_events+from .native_cpa import try_cpa_edges_CPA_COLUMNS = ["point_id", "flight_id", "emission_time", "receive_time", "distance_m", "speed_mps"]_CPA_DTYPE = np.dtype([(name, "int64" if name == "point_id" else object if name == "flight_id" else "float64")@@ -146,24 +147,29 @@ class _PreparedCPATrack:counts=np.fromiter((len(a) for a in neighbours),dtype=int,count=len(neighbours))if not counts.sum():continuesample=np.concatenate(neighbours).astype(int)- observer_index=np.repeat(np.arange(len(obs)),counts)- distance2=np.sum((pos[sample]-obs[observer_index])**2,axis=1)- previous=np.r_[False,(observer_index[1:]==observer_index[:-1])&(sample[1:]==sample[:-1]+1)]- following=np.r_[(observer_index[1:]==observer_index[:-1])&(sample[1:]==sample[:-1]+1),False]+ projected = try_cpa_edges(pos, obs, counts, sample)+ if projected is None:+ observer_index=np.repeat(np.arange(len(obs)),counts)+ distance2=np.sum((pos[sample]-obs[observer_index])**2,axis=1)+ previous=np.r_[False,(observer_index[1:]==observer_index[:-1])&(sample[1:]==sample[:-1]+1)]+ following=np.r_[(observer_index[1:]==observer_index[:-1])&(sample[1:]==sample[:-1]+1),False]+ eligible = np.flatnonzero(counts >= 2)+ ends = np.cumsum(counts)[eligible]+ starts = ends - counts[eligible]# An endpoint sample can be the sampled minimum even when the# continuous spline turns strictly inside its adjacent interval.# Check the endpoint range-rate signs before optimizing the actual# spline; an endpoint alone is not evidence of a complete pass.- eligible = np.flatnonzero(counts >= 2)- ends = np.cumsum(counts)[eligible]- starts = ends - counts[eligible]for end in (False, True):- edge_indices = ends - 1 if end else starts- adjacent = edge_indices - 1 if end else edge_indices + 1- mask = ((sample[edge_indices] == (len(t)-1 if end else 0))- & (previous[edge_indices] if end else following[edge_indices])- & (distance2[edge_indices] <= distance2[adjacent]))- oi_edge = eligible[mask]+ if projected is None:+ edge_indices = ends - 1 if end else starts+ adjacent = edge_indices - 1 if end else edge_indices + 1+ mask = ((sample[edge_indices] == (len(t)-1 if end else 0))+ & (previous[edge_indices] if end else following[edge_indices])+ & (distance2[edge_indices] <= distance2[adjacent]))+ oi_edge = eligible[mask]+ else:+ oi_edge = projected[0][end]if not len(oi_edge):continuelo, hi = (t[-2], t[-1]) if end else (t[0], t[1])@@ -184,25 +190,31 @@ class _PreparedCPATrack:rows.append((int(point_ids[chosen[oi_edge[j]]]), track.id,float(when), float(when + distance/c), float(distance),float(np.linalg.norm(track.speed_vector(when)))))- minima=(previous&following&(distance2<=np.r_[np.inf,distance2[:-1]])&- (distance2<np.r_[distance2[1:],np.inf]))- indices=np.flatnonzero(minima)- if not len(indices):continue- k=sample[indices]- oi=observer_index[indices]- # Project onto both neighbouring polyline edges, never across a- # missing interval or between unrelated flight passes.- first=np.stack((pos[k-1],pos[k]),axis=1)- last=np.stack((pos[k],pos[k+1]),axis=1)- delta=last-first- residual=first-obs[oi,None,:]- fraction=np.clip(-np.sum(residual*delta,axis=2)/np.maximum(np.sum(delta**2,axis=2),1e-12),0,1)- d2=np.sum((residual+fraction[:,:,None]*delta)**2,axis=2)- edge=np.argmin(d2,axis=1)- n=np.arange(len(k))- distance=np.sqrt(d2[n,edge])- start=np.where(edge==0,k-1,k)- when=t[start]+fraction[n,edge]*(t[start+1]-t[start])+ if projected is None:+ minima=(previous&following&(distance2<=np.r_[np.inf,distance2[:-1]])&+ (distance2<np.r_[distance2[1:],np.inf]))+ indices=np.flatnonzero(minima)+ if not len(indices):continue+ k=sample[indices]+ oi=observer_index[indices]+ # Project onto both neighbouring polyline edges, never across a+ # missing interval or between unrelated flight passes.+ first=np.stack((pos[k-1],pos[k]),axis=1)+ last=np.stack((pos[k],pos[k+1]),axis=1)+ delta=last-first+ residual=first-obs[oi,None,:]+ fraction=np.clip(-np.sum(residual*delta,axis=2)/np.maximum(np.sum(delta**2,axis=2),1e-12),0,1)+ d2=np.sum((residual+fraction[:,:,None]*delta)**2,axis=2)+ edge=np.argmin(d2,axis=1)+ n=np.arange(len(k))+ distance=np.sqrt(d2[n,edge])+ start=np.where(edge==0,k-1,k)+ fraction = fraction[n, edge]+ else:+ _, oi, start, fraction, d2 = projected+ if not len(oi):continue+ distance=np.sqrt(d2)+ when=t[start]+fraction*(t[start+1]-t[start])speed=np.linalg.norm(track.speed_vector(when),axis=1)for j in np.flatnonzero(distance<=max_distance):rows.append((int(point_ids[chosen[oi[j]]]), track.id,diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.pyindex a502613..7c2ba9d 100644--- a/src/dopplergeist/tiled_search.py+++ b/src/dopplergeist/tiled_search.py@@ -25,6 +25,7 @@ from .finite_search import RetentionLedger, bounded_assessment, classified_assesfrom .resource_usage import process_peak_rss_kibfrom .geometry import ACOUSTIC_BACKEND, AcousticSupportError, eceffrom .matching import RIDGE_FIT_BACKEND, _cache_ridge_fits+from .native_cpa import CPA_BACKENDfrom .search import (_CPA_DTYPE, _PreparedCPATrack, _ordered_cpa_rows,filter_association_event, pair_candidates, score_candidate)@@ -383,6 +384,92 @@ def _tile_anchor_window(tile_id, start_window, parameters, filters, events):return lo, hi+def _partition_query_time_range(time_range, events, parameters, filters, anchor_window):+ """Bound whole-track queries by an association partition, not a clock prior.++ The first matched passage has an implied start inside ``anchor_window``.+ Every later matched passage must overlap its clock interval. Expanding by+ both passages' tolerances therefore contains every participating receive+ time. Subtract the largest possible acoustic delay to bound emission time.+ FlightIndex uses these bounds only to select complete original segments.+ """+ if (anchor_window is None or parameters.get("mode") != "robust_ridge"+ or filters.get("association", "subsets") != "subsets" or filters.get("silence")):+ return time_range++ def finite(value):+ # Keep unusual scalar/container semantics on the established path.+ if type(value) not in (int, float):+ return False+ try:+ return math.isfinite(value)+ except OverflowError:+ return False++ if (not isinstance(anchor_window, (tuple, list)) or len(anchor_window) != 2+ or not all(finite(value) for value in anchor_window)+ or anchor_window[0] >= anchor_window[1]+ or not isinstance(time_range, (tuple, list)) or len(time_range) != 2+ or not all(finite(value) for value in time_range) or time_range[0] > time_range[1]+ or not isinstance(events, (tuple, list)) or not events):+ return time_range+ c = parameters.get("c", 343.0)+ offset = filters.get("offset_tolerance_s", 5.0)+ if not finite(c) or c <= 0 or not finite(offset) or offset < 0:+ return time_range+ tolerances, centers, radii = [], [], []+ for event in events:+ if not isinstance(event, dict) or not isinstance(event.get("quality", {}), dict):+ return time_range+ quality = event.get("quality", {})+ proposal_tolerances = [quality.get(name, 0) for name in+ ("cpa_tolerance_s", "speed_tolerance_mps", "slant_tolerance_m")]+ # Do not let an empty narrowed query hide a filter_event validation error.+ if not all(finite(value) and value >= 0 for value in proposal_tolerances):+ return time_range+ slant = event.get("slant_range_m")+ speed = event.get("speed_range_mps")+ center = event.get("cpa_receive_s")+ if (not isinstance(slant, (tuple, list)) or len(slant) != 2+ or not all(finite(value) for value in slant) or not 0 <= slant[0] <= slant[1]+ or not isinstance(speed, (tuple, list)) or len(speed) != 2+ or not all(finite(value) for value in speed) or not 0 <= speed[0] <= speed[1]+ or not finite(center)):+ return time_range+ tolerance = proposal_tolerances[0] + offset / 2+ if not finite(tolerance):+ return time_range+ tolerances.append(tolerance)+ centers.append(center)+ radii.append(slant[1])+ if max(radii) <= 0:+ return time_range++ down = lambda value: math.nextafter(value, -math.inf)+ up = lambda value: math.nextafter(value, math.inf)+ # Any event can be the first matched one after unmatched proposals. Use+ # the maximum tolerance over all events, not the first event in the list.+ margin = max(tolerances)+ start_lo, start_hi = down(anchor_window[0] - margin), up(anchor_window[1] + margin)+ receive_lo, receive_hi = math.inf, -math.inf+ for center, tolerance in zip(centers, tolerances):+ implied_lo = down(down(start_lo) - tolerance)+ implied_hi = up(up(start_hi) + tolerance)+ receive_lo = min(receive_lo, down(down(implied_lo) + center))+ receive_hi = max(receive_hi, up(up(implied_hi) + center))+ # Extra outward steps before inverse operations cover rounded start =+ # receive - event CPA and receive = emission + distance / c arithmetic.+ delay = up(max(radii) / c)+ lower = down(down(receive_lo) - delay)+ upper = up(receive_hi)+ if not all(math.isfinite(value) for value in (start_lo, start_hi, receive_lo, receive_hi, delay, lower, upper)):+ return time_range+ lower, upper = max(time_range[0], lower), min(time_range[1], upper)+ # A segment can span two disjoint query windows. Keep the original path+ # rather than treating their empty intersection as no matching segments.+ return (lower, upper) if lower <= upper else time_range++def _clock_children(tile_id, cells, start_window, parameters, filters, events):if len([p for p in tile_id.split("~")[1:] if p.startswith("t")]) >= 24:return []@@ -758,6 +845,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outruntime_identity["ridge_fit_backend"] = RIDGE_FIT_BACKENDif ACOUSTIC_BACKEND != "python-acoustic-v1":runtime_identity["acoustic_backend"] = ACOUSTIC_BACKEND+ if CPA_BACKEND != "python-cpa-v1":+ runtime_identity["cpa_backend"] = CPA_BACKENDidentity = {"indexes": [{"input_hash": i.report["input_hash"], "file_sha256": cached_sha256(i.path)} for i in indexes],"terrain": terrain_signature, "additional_input_hashes": input_hashes or {},"events": events, "acoustic_hashes": acoustic, "bounds": list(bounds), "time_range_s": list(time_range_s),@@ -909,9 +998,11 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outelse:points = _tile_coordinates(cells, budget.spacing_m, bounds, crs)tracks = []+ anchor = _tile_anchor_window(tile_id, start_window, parameters, filters, events)+ query_time = _partition_query_time_range(time_range_s, events, parameters, filters, anchor)if len(points):tracks, points = _tracks_for_points(- indexes, _expanded_bounds(cells, budget.spacing_m, radius, crs), time_range_s,+ indexes, _expanded_bounds(cells, budget.spacing_m, radius, crs), query_time,budget, points, terrain, radius, parameters.get("c", 343.0))if tracks and "elev_m" not in points:points = _sample_tile_terrain(points, terrain)@@ -920,7 +1011,6 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outpoints = pd.DataFrame()if len(points):flush_empty_tiles()- anchor = _tile_anchor_window(tile_id, start_window, parameters, filters, events)rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,ledger=ledger, stage=stage,anchor_window=anchor,diff --git a/tests/test_real_native_cpa.py b/tests/test_real_native_cpa.pynew file mode 100644index 0000000..486cb21--- /dev/null+++ b/tests/test_real_native_cpa.py@@ -0,0 +1,185 @@+"""Native CPA parity on the repository's frozen real ADS-B observations."""+import importlib.util+import os+from pathlib import Path+import subprocess+import sys+from unittest.mock import patch++import numpy as np+import pytest++from dopplergeist import native_cpa, search+from dopplergeist.geometry import AcousticSupportError, ecef+++ROOT = Path(__file__).resolve().parents[1]+SPEC = importlib.util.spec_from_file_location(+ "benchmark_real_cpa_rows", ROOT / "scripts/benchmark_real_cpa_rows.py")+BENCHMARK = importlib.util.module_from_spec(SPEC)+SPEC.loader.exec_module(BENCHMARK)++[email protected](scope="module")+def tracks():+ return BENCHMARK.recorded_tracks()+++def backend():+ if native_cpa._native is None:+ pytest.skip("optional CPA extension is not built")+ return native_cpa._native+++def assert_exact(before, after):+ BENCHMARK.compare(before, after)+ for column in ("emission_time", "receive_time", "distance_m", "speed_mps"):+ # Include signed-zero bits, not just pandas' numerical equality.+ assert before[0][column].to_numpy().tobytes() == after[0][column].to_numpy().tobytes()++[email protected]("count", [1, 32, 128, 1024])[email protected]("radius,c,step,chunk", [+ (1000., 343., 2., 512), (20000., 343., 2., 512),+ (20000., 338., 1., 7), (20000., 348., 8., 1),+ (20000., 343., np.float32(2.), 512),+])+def test_recorded_tables_and_unsupported_accounting(tracks, backend, count, radius, c, step, chunk):+ points = BENCHMARK.recorded_points(tracks, count)+ with patch.object(native_cpa, "_native", None):+ expected = BENCHMARK.cpa_stage(search, tracks, points, radius, c, step, chunk, True)+ actual = BENCHMARK.cpa_stage(search, tracks, points, radius, c, step, chunk, True)+ assert_exact(expected, actual)+++def test_raw_endpoint_and_interior_order_with_repeated_wide_ids(tracks, backend):+ points = BENCHMARK.recorded_points(tracks, 1024).iloc[::-1].copy()+ points["point_id"] = np.arange(len(points), dtype=np.int64)//2 + 2**53+ observers = ecef(points.lat.to_numpy(), points.lon.to_numpy(), points.elev_m.to_numpy())+ calls = backend.successful_calls()+ retained = 0+ with patch.object(search, "minimize_scalar", wraps=search.minimize_scalar) as endpoint:+ for track in tracks:+ try:+ prepared = search._PreparedCPATrack(track, 343., 2.)+ except AcousticSupportError:+ continue+ with patch.object(native_cpa, "_native", None):+ expected = prepared.raw_rows(observers, points.point_id.to_numpy(), 20000.)+ actual = prepared.raw_rows(observers, points.point_id.to_numpy(), 20000.)+ assert [(row[:2], np.asarray(row[2:]).tobytes()) for row in actual] == [+ (row[:2], np.asarray(row[2:]).tobytes()) for row in expected]+ assert search._ordered_cpa_rows(actual, 2.) == search._ordered_cpa_rows(expected, 2.)+ retained += len(actual)+ assert retained > 0+ assert endpoint.call_count > 0+ assert backend.successful_calls() > calls++[email protected]("policy", ["raise", "ignore", "warn"])+def test_nondefault_error_policy_uses_reference(tracks, backend, policy):+ points = BENCHMARK.recorded_points(tracks, 32)+ calls = backend.successful_calls()+ with np.errstate(all=policy):+ with patch.object(native_cpa, "_native", None):+ expected = BENCHMARK.cpa_stage(search, tracks, points, row_mode=True)+ actual = BENCHMARK.cpa_stage(search, tracks, points, row_mode=True)+ assert_exact(expected, actual)+ assert backend.successful_calls() == calls+++def graph(tracks):+ supported = []+ for track in tracks:+ try:+ track.validate_motion_support()+ supported.append(track)+ except AcousticSupportError:+ pass+ track = max(supported, key=lambda item: item.time[-1]-item.time[0])+ prepared = search._PreparedCPATrack(track, 343., 2.)+ points = BENCHMARK.recorded_points([track], 32)+ observers = np.ascontiguousarray(ecef(points.lat.to_numpy(), points.lon.to_numpy(),+ points.elev_m.to_numpy()))+ neighbours = search.cKDTree(prepared.position).query_ball_point(+ observers, 20000.+2*prepared.max_step, return_sorted=True)+ counts = np.fromiter(map(len, neighbours), dtype=np.intp)+ sample = np.concatenate(neighbours).astype(np.intp)+ return prepared.position, observers, counts, sample++[email protected]("index", range(4))+def test_strided_same_observations_use_reference(graph, backend, index):+ inputs = list(graph)+ inputs[index] = np.repeat(inputs[index], 2, axis=0)[::2]+ assert not inputs[index].flags.c_contiguous+ calls = backend.successful_calls()+ assert native_cpa.try_cpa_edges(*inputs) is None+ assert backend.successful_calls() == calls+++def test_readonly_buffers_and_empty_neighbour_groups(graph, backend):+ expected = native_cpa.try_cpa_edges(*graph)+ assert expected is not None+ for array in graph:+ array.flags.writeable = False+ actual = native_cpa.try_cpa_edges(*graph)+ for left, right in zip((*actual[0], *actual[1:]), (*expected[0], *expected[1:])):+ assert left.dtype == right.dtype+ assert left.shape == right.shape+ assert left.tobytes() == right.tobytes()+ # An empty receiver chunk must return no candidate proposals.+ far_observers = graph[1][:0]+ empty = native_cpa.try_cpa_edges(graph[0], far_observers, graph[2][:0], graph[3][:0])+ assert empty is not None+ assert all(array.size == 0 for array in (*empty[0], *empty[1:]))+++def test_native_rejects_invalid_graph_metadata_and_output_aliases(graph, backend):+ position, observers, counts, sample = graph+ capacity = sample.size//2+ endpoints = np.empty((2, len(observers)), dtype=np.intp)+ indices = np.empty((2, capacity), dtype=np.intp)+ values = np.empty((2, capacity), dtype=np.float64)+ changed = counts.copy()+ changed[0] = -1+ assert backend.select(position, observers, changed, sample, endpoints, indices, values) is None+ changed = sample.copy()+ changed[0] = len(position)+ assert backend.select(position, observers, counts, changed, endpoints, indices, values) is None+ assert backend.select(position, observers, counts, sample[:-1], endpoints, indices, values) is None+ assert backend.select(position, observers, counts, sample, endpoints, indices[:, :-1], values) is None+ # Writable output must never overwrite neighbour indices while C reads them.+ storage = np.empty(max(sample.size, endpoints.size), dtype=np.intp)+ storage[:sample.size] = sample+ alias = storage[:endpoints.size].reshape(endpoints.shape)+ before = storage.tobytes()+ assert backend.select(position, observers, counts, storage[:sample.size], alias, indices, values) is None+ assert storage.tobytes() == before+ before = values.tobytes()+ assert backend.select(position, observers, counts, sample, endpoints,+ values.view(np.intp), values) is None+ assert values.tobytes() == before+++def test_backend_selection_and_source_only_import():+ code = "from dopplergeist.native_cpa import CPA_BACKEND; print(CPA_BACKEND)"+ environment = dict(os.environ, DOPPLERGEIST_CPA_BACKEND="python")+ result = subprocess.run([sys.executable, "-c", code], env=environment,+ text=True, capture_output=True, check=True)+ assert result.stdout.strip() == "python-cpa-v1"+ environment["DOPPLERGEIST_CPA_BACKEND"] = "auto"+ result = subprocess.run([sys.executable, "-c", code], env=environment,+ text=True, capture_output=True, check=True)+ assert result.stdout.strip() in {"python-cpa-v1", "native-c-cpa-v1"}+ missing = "import sys; sys.modules['dopplergeist._native_cpa'] = None; " + code+ result = subprocess.run([sys.executable, "-c", missing], env=environment,+ text=True, capture_output=True, check=True)+ assert result.stdout.strip() == "python-cpa-v1"+ environment["DOPPLERGEIST_CPA_BACKEND"] = "unknown"+ result = subprocess.run([sys.executable, "-c", code], env=environment,+ text=True, capture_output=True)+ assert result.returncode != 0+ assert "DOPPLERGEIST_CPA_BACKEND must be" in result.stderrdiff --git a/tests/test_real_native_ridge_batch.py b/tests/test_real_native_ridge_batch.pynew file mode 100644index 0000000..e8dae3b--- /dev/null+++ b/tests/test_real_native_ridge_batch.py@@ -0,0 +1,152 @@+"""Batched native fitting on frozen, recorded acoustic factors and ridges."""+from copy import deepcopy+import json+from pathlib import Path++import numpy as np+import pytest++from dopplergeist import batch_ridge, matching+++def recorded_batch():+ if matching._native_fit_irls_batch is None:+ pytest.skip("optional native batch fitter is not installed")+ source = Path(__file__).parent / "data/real-ridge-batch-arithmetic.json"+ data = json.loads(source.read_text())+ measurement = (matching.RidgeTimes(np.asarray(data["times"], dtype=float)),+ np.asarray(data["ridge"], dtype=float), np.asarray(data["weight"], dtype=float), {})+ return measurement, deepcopy(data["event"]), np.asarray(data["factors"]), np.asarray(data["emissions"])+++def canonical(value):+ return json.dumps(value, sort_keys=True, allow_nan=True)++[email protected]("order", ["C", "F", "reversed"])[email protected]("policy", ["default", "raise"])+def test_recorded_native_batch_is_exact_and_reduces_dispatch(recorded_batch, monkeypatch, order, policy):+ measurement, event, factors, emissions = recorded_batch+ if order == "reversed":+ factors, emissions = factors[::-1], emissions[::-1]+ else:+ factors, emissions = np.array(factors, order=order), np.array(emissions, order=order)+ settings = {} if policy == "default" else {"all": "raise"}+ with monkeypatch.context() as patch:+ patch.setattr(matching, "_native_fit_irls_batch", None)+ with np.errstate(**settings):+ expected = batch_ridge._first_event_fits_uncached(measurement, event, factors, emissions)+ before = matching._native_ridge_module.fit_statistics()+ with np.errstate(**settings):+ actual = batch_ridge._first_event_fits_uncached(measurement, event, factors, emissions)+ after = matching._native_ridge_module.fit_statistics()+ assert canonical(actual) == canonical(expected)+ assert after["scalar_calls"] == before["scalar_calls"]+ assert after["batch_calls"] - before["batch_calls"] == int(order == "C")+ assert after["batch_successful_rows"] - before["batch_successful_rows"] == (len(factors) if order == "C" else 0)++[email protected]("failure", ["false", "type_error"])+def test_recorded_batch_failure_retains_vectorized_fallback(recorded_batch, monkeypatch, failure):+ with monkeypatch.context() as patch:+ patch.setattr(matching, "_native_fit_irls", None)+ expected = batch_ridge._first_event_fits_uncached(*recorded_batch)++ def unavailable(*args):+ # A partial native output must never enter the final fit diagnostics.+ args[-1][:] = 12345.+ if failure == "type_error":+ raise TypeError("Unsupported native buffers")+ return False++ monkeypatch.setattr(matching, "_native_fit_irls_batch", unavailable)+ actual = batch_ridge._first_event_fits_uncached(*recorded_batch)+ assert canonical(actual) == canonical(expected)+++def native_arguments(recorded_batch):+ measurement, event, factors, emissions = recorded_batch+ times, ridge, weight = matching._ridge_fit_inputs(measurement, matching._frequency_scale(event), .02)+ measured = np.isfinite(times) & np.isfinite(ridge) & (weight > 0)+ y, w = ridge[measured], weight[measured]+ independent = len(np.unique(times[measured]))+ w = w * independent / w.sum()+ span = np.maximum(np.ptp(emissions, axis=1), 1e-9)+ reference = (emissions.min(axis=1) + emissions.max(axis=1)) / 2+ u = (emissions - reference[:, None]) / span[:, None]+ low, high = event["f0_range_hz"]+ cap = np.clip(np.median(y[None, :] / factors, axis=1), low, high) * .02+ return [factors, u, y, w, cap, low, high, matching._frequency_scale(event),+ independent, np.empty((len(factors), 5))]++[email protected]("invalid", ["shape", "dtype", "readonly", "strided"])+def test_native_output_buffers_are_checked(recorded_batch, invalid):+ args = native_arguments(recorded_batch)+ if invalid == "shape":+ args[-1] = args[-1][:, :4].copy()+ elif invalid == "dtype":+ args[-1] = args[-1].astype(np.float32)+ elif invalid == "readonly":+ args[-1].setflags(write=False)+ else:+ args[-1] = args[-1][::-1]+ with pytest.raises((TypeError, ValueError, BufferError)):+ matching._native_fit_irls_batch(*args)+++def test_native_batch_reuses_scalar_arithmetic_for_recorded_rows(recorded_batch):+ args = native_arguments(recorded_batch)+ expected = [matching._native_fit_irls(factor, u, args[2], args[3], args[5], args[6],+ float(cap), args[7], args[8])+ for factor, u, cap in zip(args[0], args[1], args[4])]+ assert matching._native_fit_irls_batch(*args)+ assert np.array_equal(args[-1], np.asarray(expected))++[email protected]("source", ["factors", "time", "ridge", "weight", "caps", "reversed_caps"])+def test_native_batch_rejects_overlapping_output_before_writing(recorded_batch, source):+ args = native_arguments(recorded_batch)+ rows = len(args[0])+ if source in {"caps", "reversed_caps"}:+ storage = np.empty((rows, 5))+ caps = storage[:, 0] if source == "caps" else storage[::-1, 0]+ caps[:] = args[4]+ args[4], args[-1] = caps, storage+ else:+ index = {"factors": 0, "time": 1, "ridge": 2, "weight": 3}[source]+ args[-1] = args[index].reshape(-1)[:rows * 5].reshape(rows, 5)+ before = [value.copy() for value in (*args[:5], args[-1])]+ with pytest.raises(ValueError, match="must not overlap"):+ matching._native_fit_irls_batch(*args)+ for actual, saved in zip((*args[:5], args[-1]), before):+ assert np.array_equal(actual, saved, equal_nan=True)+++def test_native_batch_accepts_disjoint_strided_recorded_inputs(recorded_batch):+ args = native_arguments(recorded_batch)+ expected = [matching._native_fit_irls(factor, u, args[2], args[3], args[5], args[6],+ float(cap), args[7], args[8])+ for factor, u, cap in zip(args[0], args[1], args[4])]+ for index in (2, 3, 4):+ storage = np.empty(len(args[index]) * 2)+ vector = storage[-2::-2]+ vector[:] = args[index]+ args[index] = vector+ assert matching._native_fit_irls_batch(*args)+ assert np.array_equal(args[-1], np.asarray(expected))+++def test_native_batch_accepts_adjacent_input_and_output(recorded_batch):+ args = native_arguments(recorded_batch)+ expected = [matching._native_fit_irls(factor, u, args[2], args[3], args[5], args[6],+ float(cap), args[7], args[8])+ for factor, u, cap in zip(args[0], args[1], args[4])]+ output_size = len(args[0]) * 5+ storage = np.empty(output_size + len(args[2]))+ storage[output_size:] = args[2]+ args[2] = storage[output_size:]+ args[-1] = storage[:output_size].reshape(len(args[0]), 5)+ assert matching._native_fit_irls_batch(*args)+ assert np.array_equal(args[-1], np.asarray(expected))diff --git a/tests/test_real_partition_query.py b/tests/test_real_partition_query.pynew file mode 100644index 0000000..bd84f4a--- /dev/null+++ b/tests/test_real_partition_query.py@@ -0,0 +1,187 @@+"""Clock-query checks on recorded audio settings and historical flight segments.++The numeric boundary controls vary floating-point endpoints, not aircraft/audio.+The optional dense replay uses the real fixture from benchmark_prepared_measurements.+"""+from copy import deepcopy+import json+import math+import os+from pathlib import Path+from types import SimpleNamespace++import numpy as np+import pandas as pd+import pytest++from dopplergeist.data_access import FlightIndex+from dopplergeist.matching import association_candidates, prepare_events+from dopplergeist.tiled_search import _partition_query_time_range+++def observed_settings():+ fixture = json.loads((Path(__file__).parent / "data/reference-cpa-real.json").read_text())+ events, rows = fixture["events"], fixture["cpa_rows"]+ day = math.floor(rows[0]["emission_time"] / 86400) * 86400+ anchor = [row["receive_time"] - events[0]["cpa_receive_s"] for row in rows[:2]]+ return [day, day + 86400], events, {"mode": "robust_ridge"}, {}, anchor+++def test_recorded_clock_query_defaults_and_fallbacks(observed_settings):+ original, events, parameters, filters, anchor = observed_settings+ narrowed = _partition_query_time_range(*observed_settings)+ assert original[0] < narrowed[0] < narrowed[1] < original[1]+ assert _partition_query_time_range(original, events, parameters, {"association": "subsets"}, anchor) == narrowed+ for changes in (+ {4: None}, {4: [anchor[0], anchor[0]]}, {4: [math.nan, anchor[1]]},+ {4: [anchor[0], math.inf]}, {4: [anchor[0], 10 ** 1000]},+ {2: {"mode": "ridge"}}, {2: {"mode": "robust_ridge", "c": 0}},+ {2: {"mode": "robust_ridge", "c": math.inf}},+ {3: {"association": "full"}}, {3: {"silence": {"edge_margin_s": -1}}},+ {3: {"offset_tolerance_s": -1}}, {3: {"offset_tolerance_s": math.inf}},+ {1: []},+ ):+ args = list(observed_settings)+ for index, value in changes.items():+ args[index] = value+ assert _partition_query_time_range(*args) is original+ for name in ("cpa_tolerance_s", "speed_tolerance_mps", "slant_tolerance_m"):+ for value in (-1, math.nan, math.inf, "invalid", 10 ** 1000):+ changed = deepcopy(events)+ changed[0].setdefault("quality", {})[name] = value+ assert _partition_query_time_range(original, changed, parameters, filters, anchor) is original+ malformed = deepcopy(events)+ del malformed[0]["speed_range_mps"]+ assert _partition_query_time_range(original, malformed, parameters, filters, anchor) is original+ # Complete segments can bridge disjoint windows, so retain the original query.+ disjoint = [original[0], math.nextafter(narrowed[0], -math.inf)]+ assert _partition_query_time_range(disjoint, events, parameters, filters, anchor) is disjoint+++def test_recorded_clock_query_contains_rounded_interval_boundaries(observed_settings):+ original, events, parameters, filters, anchor = observed_settings+ lower, upper = _partition_query_time_range(*observed_settings)+ tolerances = [event["quality"]["cpa_tolerance_s"] + 2.5 for event in events]+ radius = max(event["slant_range_m"][1] for event in events)+ overlaps = 0+ for first in (anchor[0], math.nextafter(anchor[0], math.inf), math.nextafter(anchor[1], -math.inf)):+ for first_tolerance in tolerances:+ for event, tolerance in zip(events, tolerances):+ for boundary in (first - first_tolerance - tolerance, first + first_tolerance + tolerance):+ for start in (math.nextafter(boundary, -math.inf), boundary, math.nextafter(boundary, math.inf)):+ if max(first - first_tolerance, start - tolerance) > min(first + first_tolerance, start + tolerance):+ continue+ for distance in (0.0, radius):+ received = start + event["cpa_receive_s"]+ for emitted in (math.nextafter(received - distance / 343.0, -math.inf),+ received - distance / 343.0,+ math.nextafter(received - distance / 343.0, math.inf)):+ assert lower <= emitted <= upper+ overlaps += 1+ assert overlaps > 10000+++def dense_recorded_input():+ fixture_path = os.environ.get("DOPPLERGEIST_REAL_DENSE_QUERY")+ index_path = os.environ.get("DOPPLERGEIST_REAL_DEVELOPMENT_INDEX")+ if not fixture_path or not index_path:+ pytest.skip("set DOPPLERGEIST_REAL_DENSE_QUERY and DOPPLERGEIST_REAL_DEVELOPMENT_INDEX")+ fixture = json.loads(Path(fixture_path).read_text())+ index = FlightIndex(index_path, track_cache_bytes=0)+ assert index.report["input_hash"] == fixture["index_input_hash"]+ events, measurements = prepare_events(fixture["events"])+ return fixture, index, events, measurements+++def test_dense_recorded_query_preserves_candidates_and_complete_segments(dense_recorded_input):+ fixture, index, events, _ = dense_recorded_input+ narrowed = _partition_query_time_range(fixture["query_time_s"], events, fixture["scoring"], {},+ fixture["anchor_window_s"])+ tracks = list(index.query(fixture["query_bounds"], narrowed))+ ids = {track.id for track in tracks}+ original_count = fixture["track_query_preparation"]["queried_trajectories"]+ assert 0 < len(tracks) < original_count / 10+ assert any(track.time[0] < narrowed[0] or track.time[-1] > narrowed[1] for track in tracks)+ complete = {track.id: track for track in index.lookup(ids)}+ for track in tracks:+ np.testing.assert_array_equal(track.time, complete[track.id].time)+ np.testing.assert_array_equal(track.xyz, complete[track.id].xyz)+ np.testing.assert_array_equal(track.spline.c, complete[track.id].spline.c)+ table = pd.DataFrame(fixture["cpa_rows"])+ selected = table.loc[table.flight_id.isin(ids)]+ options = dict(anchor_window=fixture["anchor_window_s"], max_candidates=fixture["budget"]["max_candidates_per_tile"])+ before, before_counts = association_candidates(table, events, **options)+ after, after_counts = association_candidates(selected, events, **options)+ assert before and before == after+ assert after_counts["hypotheses_examined"] == before_counts["hypotheses_examined"]+ assert after_counts["work_units"] < before_counts["work_units"]+ missing = {item["track_id"] for item in fixture["unsupported_cpa_tracks"]} - ids+ assert missing+ with index.connect() as connection:+ intervals = dict((name, (start, end)) for name, start, end in+ connection.execute("SELECT name, start, end FROM segments"))+ assert all(intervals[name][1] < narrowed[0] or intervals[name][0] > narrowed[1] for name in missing)+ print(f"real dense query: {original_count} -> {len(tracks)} complete segments; "+ f"{len(table)} -> {len(selected)} CPA rows; {len(after)} identical association payloads")++[email protected]("driver", ["serial", "worker"])+def test_recorded_partition_driver_queries_before_scoring(dense_recorded_input, tmp_path, monkeypatch, driver):+ """Exercise each driver up to scoring with the recorded receiver and index."""+ import dopplergeist.parallel_tiles as parallel+ import dopplergeist.tiled_search as tiled+ from dopplergeist.native_cpa import CPA_BACKEND++ fixture, index, events, measurements = dense_recorded_input+ parameters, filters = fixture["scoring"], {"association": "subsets"}+ start_window = fixture["anchor_window_s"]+ budget = tiled.SearchBudget(**fixture["budget"])+ bounds = fixture["query_bounds"]+ tile, cells = next(tiled.grid_tiles(bounds, budget.spacing_m, budget.tile_cells, "EPSG:5070"))+ tile += "~t0"+ anchor = tiled._tile_anchor_window(tile, start_window, parameters, filters, events)+ expected = tiled._partition_query_time_range(fixture["query_time_s"], events, parameters, filters, anchor)+ point = pd.DataFrame([fixture["point"]])+ terrain = SimpleNamespace(last_sample_files=set(), last_sample_ocean_tiles=set(), failures=[])+ queries = []+ original_query = index.query++ def query(query_bounds, query_time):+ queries.append(query_time)+ yield from original_query(query_bounds, query_time)++ class ReachedScoring(RuntimeError):+ pass++ def score(points, tracks, *args, **kwargs):+ assert queries == [expected]+ assert kwargs["anchor_window"] == anchor+ assert 0 < len(tracks) < fixture["track_query_preparation"]["queried_trajectories"] / 10+ assert points.equals(point)+ raise ReachedScoring++ monkeypatch.setattr(index, "query", query)+ target = tiled if driver == "serial" else parallel+ monkeypatch.setattr(target, "_tile_coordinates", lambda *args: point.copy())+ monkeypatch.setattr(target, "_expanded_bounds", lambda *args: bounds)+ monkeypatch.setattr(target, "_score", score)+ if driver == "serial":+ monkeypatch.setattr(tiled, "grid_tiles", lambda *args: iter([(tile, cells)]))+ with pytest.raises(ReachedScoring):+ tiled.search_national([index], terrain, events, measurements,+ time_range_s=fixture["query_time_s"], output_dir=tmp_path, bounds=bounds, budget=budget,+ scoring=dict(parameters, recording_start_window_s=start_window), filters=filters, workers=1)+ runtime = json.loads((tmp_path / "manifest.json").read_text())["configuration"]["runtime"]+ if CPA_BACKEND == "python-cpa-v1":+ assert "cpa_backend" not in runtime+ else:+ assert runtime["cpa_backend"] == CPA_BACKEND+ else:+ monkeypatch.setattr(parallel, "_worker", dict(indexes=[index], terrain=terrain, events=events,+ measurements=measurements, bounds=bounds, time_range=fixture["query_time_s"], budget=budget,+ parameters=parameters, filters=filters, start_window=start_window, crs="EPSG:5070",+ ledger_directory=tmp_path, run_hash="recorded-query-test", score_kind="matched_group_support_minus_normalized_residual"))+ with pytest.raises(ReachedScoring):+ parallel.score_tile(f"coarse:{tile}", cells)