Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Integrate exact native search, prepared flight loading, and pipeline reuse
Cinna Davis49c90338f7be
diff --cc .github/workflows/real-performance.ymlindex d727775,95b8a1c,a552267..fc1d21f--- a/.github/workflows/real-performance.yml+++ b/.github/workflows/real-performance.yml@@@@ -4,14 -4,14 -4,14 +4,14 @@@@ oninputs:baseline:description: Exact baseline commit for both real-input comparisons- default: 27da08a878ac2ec90cd7e2c07f43765ceeea609b- default: b4cd00ad46f0299baa716950fabe062d74f019f0- default: 994d1ee48e25df6d30b23a356bba8f6502d8b7f6+++ default: ae7721b7d3099980b884175cde1a3b07fc3fb1b5required: trueregional:description: Also compare the complete regional HTTP workflowtype: booleandefault: truepush:- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103]- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103, perf/flight-sieve-94a6, perf/flight-sieve-combined-94a6, perf/prepared-flight-scoring-20261103]- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103, perf/prepared-flight-search-20261103]+++ branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103, perf/flight-sieve-94a6, perf/flight-sieve-combined-94a6, perf/prepared-flight-scoring-20261103, perf/prepared-flight-search-20261103, perf/integrated-search-47383f75]paths:- .github/workflows/real-performance.yml- src/dopplergeist/**@@@@ -26,9 -26,9 -31,11 +31,13 @@@@- scripts/check_real_upload_arrays.py- pyproject.toml- tests/test_real_*.py+++ - tests/test_refinement_geographic_bounds.py- tests/data/real-ridge-batch-arithmetic.json- tests/data/reference-empty-tiles-real.json- tests/data/reference-retention-real.json+++ - tests/data/reference-cpa-real.json++ - tests/data/real-refinement-profile.json++ - tests/fixtures/real_discovery_normalization.jsonpermissions:contents: readenv:@@@@ -38,7 -38,7 -45,7 +47,7 @@@@NUMEXPR_NUM_THREADS: '1'CC: gccPYTHONPATH: src- BASELINE_REF: ${{ inputs.baseline || '27da08a878ac2ec90cd7e2c07f43765ceeea609b' }}- BASELINE_REF: ${{ inputs.baseline || 'b4cd00ad46f0299baa716950fabe062d74f019f0' }}- BASELINE_REF: ${{ inputs.baseline || '994d1ee48e25df6d30b23a356bba8f6502d8b7f6' }}+++ BASELINE_REF: ${{ inputs.baseline || 'ae7721b7d3099980b884175cde1a3b07fc3fb1b5' }}jobs:compare:if: github.event_name != 'push' || github.ref != 'refs/heads/perf/real-pipeline-throughput-20261103'@@@@ -100,9 -103,11 -113,9 +118,49 @@@@- name: Real-input regression testsenv:DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json+ + DOPPLERGEIST_REAL_DENSE_QUERY: benchmark-inputs/dense-query.json+ + DOPPLERGEIST_REAL_DEVELOPMENT_INDEX: benchmark-inputs/reference-region.sqliteDOPPLERGEIST_REAL_REFERENCE_ROOT: benchmark-inputs/reference-checkoutDOPPLERGEIST_REAL_REFERENCE_DISCOVERY: evidence/audio/w1-sensitivity-20261102/reference/discovery.json- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_native_ridge.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_partition_query.py tests/test_real_native_cpa.py tests/test_real_native_ridge.py tests/test_real_native_ridge_batch.py tests/test_real_native_prefix_integration.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py tests/test_real_scalar_terrain.py tests/test_real_scoring_inputs.py tests/test_real_trial_prefix.py tests/test_real_candidate_setup.py tests/test_real_retention_batch.py- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_native_ridge.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py tests/test_real_native_convergence.py tests/test_real_flight_payload.py tests/test_real_prepared_index.py tests/test_real_trial_vectors.py tests/test_real_refinement_reuse.py tests/test_real_discovery_normalization.py+++ run: >-+++ python -m pytest -q+++ tests/test_real_prepared_measurements.py+++ tests/test_real_geometry_kinematics.py+++ tests/test_real_batch_scalar_parity.py+++ tests/test_real_motion_support_batch.py+++ tests/test_real_single_pass_sieve.py+++ tests/test_real_refinement_lookup.py+++ tests/test_real_cpa_preparation.py+++ tests/test_real_partition_query.py+++ tests/test_real_native_cpa.py+++ tests/test_real_native_ridge.py+++ tests/test_real_native_ridge_batch.py+++ tests/test_real_native_prefix_integration.py+++ tests/test_real_native_ridge_long.py+++ tests/test_real_ridge_backend.py+++ tests/test_real_legacy_batch_cache.py+++ tests/test_real_native_acoustic.py+++ tests/test_real_native_acoustic_pipeline.py+++ tests/test_real_batch_ceilings.py+++ tests/test_real_acoustic_screen_cache.py+++ tests/test_real_cpa_spatial_sieve.py+++ tests/test_real_native_validation.py+++ tests/test_real_index_columns.py+++ tests/test_real_track_cache.py+++ tests/test_real_scalar_terrain.py+++ tests/test_real_scoring_inputs.py+++ tests/test_real_trial_prefix.py+++ tests/test_real_candidate_setup.py+++ tests/test_real_retention_batch.py+++ tests/test_real_native_convergence.py+++ tests/test_real_flight_payload.py+++ tests/test_real_prepared_index.py+++ tests/test_real_trial_vectors.py+++ tests/test_real_refinement_reuse.py+++ tests/test_refinement_geographic_bounds.py+++ tests/test_real_discovery_normalization.py+++ tests/test_real_association_budget.py+++ tests/test_real_association_clock_window.py- name: Verify the explicit Python fallback on recorded arithmeticenv:DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.jsondiff --cc README.mdindex 0b843f1,0b843f1,0b843f1..f382184--- a/README.md+++ b/README.md@@@@ -37,7 -37,7 -37,7 +37,7 @@@@ python -m venv .venTo install the CLI without keeping a checkout, create a virtual environment and run `.venv/bin/python -m pip install "git+https://github.com/1337isnot1337/DOPPLERGEIST.git"`.---Installation builds optional acoustic and ridge acceleration when a compatible C compiler is available. Python fallbacks remain available. See [native backend checks and reproducible timing](docs/PERFORMANCE.md#reproduce-the-optional-native-backends) to verify which implementations are active and compare complete recorded-input jobs.+++Installation builds optional acoustic, ridge, and closest-approach (CPA) acceleration when a compatible C compiler is available. Python fallbacks remain available. See [native backend checks and reproducible timing](docs/PERFORMANCE.md#reproduce-the-optional-native-backends) to verify which implementations are active and compare complete recorded-input jobs.Native Windows execution is unverified.diff --cc docs/PERFORMANCE.mdindex 9d99f2d,f9765f8,492b841..4406e5a--- a/docs/PERFORMANCE.md+++ b/docs/PERFORMANCE.md@@@@ -1,5 -1,5 -1,27 +1,34 @@@@# Measured pipeline performance++ ## Prepared flight inputs and exact search reuse++++ [PR262](https://github.com/1337isnot1337/DOPPLERGEIST/pull/262) adds an optional++ prepared trajectory representation, vectorized conservative trial bounds, and++ exact native acoustic reuse. Against `994d1ee`, which already includes PR258,-the measured dense query, sieve, scoring, and retention workload falls from-**33.986 to 26.794 seconds: 1.268×, or 21.16% less elapsed time**.-Final-runtime CI at `b98a0fa3` repeats the gain at **38.374 to 30.399 seconds,-or 1.262× / 20.78% less time**, and passes 168 recorded-input and fallback checks.+++the historical instrumented dense workload falls from+++**33.986 to 26.794 seconds: a 1.268× diagnostic ratio, or 21.16% less elapsed time**.+++Final-runtime CI at `b98a0fa3` records **38.374 to 30.399 seconds,+++or a 1.262× diagnostic ratio / 20.78% less time**, and passes 168 recorded-input and fallback checks.++ All compared scientific outputs and logical retention records remain equal.+++++The original prepared-flight harness wrapped numerical functions during these+++dense measurements. The wrappers add overhead and can disable optimizations+++guarded by function identity in the combined runtime. These figures retain+++diagnostic and exact-output value. They do not measure canonical production throughput.+++The [historical timing scope](PERFORMANCE_PREPARED_SEARCH.md#historical-timing-scope)+++explains the qualification and the corrected harness controls.+++++ The final implementation also reuses local terrain pixels and profile coordinate++ transforms, and arranges spectral median inputs contiguously. A completed++ intermediate HTTP series improves 9.56%. The final source has two complete exact++ HTTP pairs, both faster, followed by an interrupted third pair. The report keeps++ that incomplete series separate and makes no three-pair final HTTP claim.++++ The [prepared-search guide](PERFORMANCE_PREPARED_SEARCH.md) gives the complete++ samples, preparation cost, storage tradeoff, HTTP measurements, operator steps,++ and reproduction commands. Its pinned baseline separates these gains from the++ earlier changes documented below. Component speedups are not multiplied.++## Indexed flight queries and repeated evidence checks[PR258](https://github.com/1337isnot1337/DOPPLERGEIST/pull/258) reduces repeated work in indexed flight loading and ridge-evidence validation.@@@@ -1096,8 -1099,8 -1118,8 +1128,8 @@@@ requires its validated checkpoint compa## Reproduce the optional native backends---Installation builds the acoustic and ridge extensions when a compatible C compiler is available.---The package remains usable when either optional extension cannot build.+++Installation builds the acoustic, ridge, and closest-approach (CPA) extensions when a compatible C compiler is available.+++The package remains usable when an optional extension cannot build.A source checkout used for timing must contain its own built extensions; an installed copy elsewhere does not establish native use.```bash@@@@ -1106,25 -1109,25 -1128,25 +1138,36 @@@@ CC=gcc python -m pip install --editablpython - <<'PY'from dopplergeist.geometry import ACOUSTIC_BACKENDfrom dopplergeist.matching import RIDGE_FIT_BACKEND+++from dopplergeist.native_cpa import CPA_BACKENDprint("Acoustic:", ACOUSTIC_BACKEND)print("Ridge:", RIDGE_FIT_BACKEND)+++print("CPA:", CPA_BACKEND)assert ACOUSTIC_BACKEND.startswith("native-c-")assert RIDGE_FIT_BACKEND.startswith("native-c-")+++assert CPA_BACKEND.startswith("native-c-")PY```The native acoustic solver retains polynomial evaluation order, global Newton stopping, and Python motion-support diagnostics.The ridge extension retains the six IRLS iterations and reduction order.---Both extensions disable floating-point reassociation and contraction.+++The CPA extension preserves the existing closest-approach selection and arithmetic order.+++All three extensions disable floating-point reassociation and contraction.Incompatible numeric representations use the existing Python implementations.Acoustic and scalar ridge calls also fall back for unsupported NumPy error policies.Batched ridge arithmetic retains its existing ignored-error block and restores the caller's policy for diagnostics.The optional ridge implementation is integrated from [PR228](https://github.com/1337isnot1337/DOPPLERGEIST/pull/228).---Set `DOPPLERGEIST_ACOUSTIC_BACKEND=python` and `DOPPLERGEIST_RIDGE_BACKEND=python` before starting a process to force both Python implementations.+++Set all three selectors before starting a process to force the Python implementations:++++++```bash+++export DOPPLERGEIST_ACOUSTIC_BACKEND=python+++export DOPPLERGEIST_RIDGE_BACKEND=python+++export DOPPLERGEIST_CPA_BACKEND=python+++```+++The default for each selector is `auto`; invalid selector values fail explicitly.---The legacy checkpoint upgrade requires both Python backends and its original runtime and input controls.---Ordinary checkpoints include the selected native binary identities and reject an implementation mismatch.+++The legacy checkpoint upgrade requires all three Python backends and its original runtime and input controls.+++Ordinary checkpoints include the selected backend and implementation identities and reject an implementation mismatch.### Recorded-input scoring comparisondiff --cc docs/PERFORMANCE_PREPARED_SEARCH.mdindex 0000000,0000000,356bf8d..234d18emode 000000,000000,100644..100644--- a/docs/PERFORMANCE_PREPARED_SEARCH.md+++ b/docs/PERFORMANCE_PREPARED_SEARCH.md@@@@ -1,0 -1,0 -1,404 +1,426 @@@@++ # Prepared flight data and exact pipeline optimization++-The final runtime's dense-search CI workload falls from **38.374 to 30.399-seconds**, or **20.78% less elapsed time**. The earlier local search comparison+++## Historical timing scope++++++The dense-search figures in this document are **instrumented diagnostic timings**.+++The original `benchmark_prepared_flight_search.py` always wrapped numerical functions,+++even for its default comparison. Those wrappers add timing overhead and change+++function identities. In the combined runtime, they can also disable guarded input+++and prefix reuse and native batch dispatch.++++++The original harness wrapped `_fit_ridge_uncached`, `Track.acoustic`,+++`_native_fit_irls`, and `flight_payload.decode_track`. The archived dense results+++retain exact-output evidence, but do not measure canonical production throughput.+++The separate HTTP, refinement, and discovery comparisons use other harnesses.+++This qualification does not change their declared measurement scopes.++++++The final runtime's dense-search CI diagnostic falls from **38.374 to 30.399+++seconds**, or **20.78% less elapsed time**. The earlier local diagnostic++ falls from **33.986 to 26.794 seconds**, or **21.16% less time**.++ A completed intermediate HTTP series falls from **58.233 to 52.668++ seconds**, or **9.56% less time**, after exact refinement reuse is added.++ These are separate measured workloads. Their speedup ratios must not be multiplied.++++ The finalized runtime is `b98a0fa3e9dc8a204ffddf00f0bc411c9206d868`.++ Earlier measurements retain their original source identities:++++ - Baseline: `994d1ee48e25df6d30b23a356bba8f6502d8b7f6`, including PR #258.++ - Dense search and initial HTTP: `ecb1726fb9872d0060265014aa830e598b4dd3b4`.++ - Independent CI and portable component baseline: `c0c9e798f47b547ab581fa473912e7218e3b4c3e`.++++ The [measurement record](../benchmarks/results/prepared-flight-search-20261103.json)++ contains individual samples, source identities, preparation costs, rejected++ experiments, and comparison scope. Later changes on `main` are outside these pins.++-## Dense search and independent replication+++## Historical instrumented dense-search comparisons++-| Timed work | Baseline median | Current median | Speedup |+++| Timed work | Baseline median | Current median | Diagnostic ratio |++ | --- | ---: | ---: | ---: |-| Complete measured search, wall | 33.985746 s | 26.793586 s | 1.268× |-| Complete measured search, CPU | 33.968096 s | 26.784584 s | 1.268× |+++| Complete instrumented search, wall | 33.985746 s | 26.793586 s | 1.268× |+++| Complete instrumented search, CPU | 33.968096 s | 26.784584 s | 1.268× |++ | Flight query and distance sieve, wall | 6.569945 s | 2.237149 s | 2.937× |++ | CPA, associations, scoring and retention, wall | 27.846495 s | 24.058635 s | 1.157× |++++ Stage medians do not sum to the median of the complete search.++ The query traverses 4,694 observed trajectories and 713,100 samples.++ The unchanged sieve selects 1,354 trajectories and 357,324 samples.++ The scorer evaluates 16,101 candidates and retains 100.++ The recording supplies 16 independent event groups and 724 ridge samples.++ All six runs return exactly equal complete outputs and logical retention records.++++ This workload uses one source-informed receiver point and one association-anchor++ partition, without a recording-start clock prior. The regional archive has++ incomplete coverage. The result does not establish independent location accuracy,++ complete daily search, or national completion time.++++ The timer includes the initial flight query, distance sieve, associations,++ scoring, and retention writes. It excludes imports, index metadata, event++ preparation, downloads, terrain sampling, audio discovery, and refinement.++ Both arms read the same prepared database; baseline reads its original payloads.++++ [Independent CI](https://github.com/1337isnot1337/DOPPLERGEIST/actions/runs/41701995581)-repeats the dense comparison on another machine: **38.451 to 30.292 seconds**,-or **1.269×**. Its three alternating pairs produce exactly equal complete outputs.+++repeats the instrumented dense comparison on another machine: **38.451 to 30.292 seconds**,+++or a **1.269× diagnostic ratio**. Its three alternating pairs produce exactly equal complete outputs.++ CI pins one available CPU and rebuilds the original index with baseline code.++ That earlier workflow passes 133 recorded-input checks and four fallback checks,++ without skips. Separate installation and static checks pass on Python 3.11 and 3.14.++++ [Final-runtime CI](https://github.com/1337isnot1337/DOPPLERGEIST/actions/runs/41708828025)-repeats all comparisons at `b98a0fa3`. Prepared dense-search wall medians are-**38.373940 versus 30.398725 seconds**, or **1.262× / 20.78% less time**.+++repeats all comparisons at `b98a0fa3`. Instrumented dense-search wall medians are+++**38.373940 versus 30.398725 seconds**, or a **1.262× diagnostic ratio / 20.78% less time**.++ CPU medians are 38.329208 versus 30.363599 seconds. Worker-lifetime peak RSS++ medians are 541.410 versus 518.855 MiB. Query/sieve medians are 8.150010 versus++ 2.962375 seconds; scoring/retention medians are 30.179885 versus 27.415653 seconds.++ All three pairs preserve complete outputs. The final workflow passes **164++ recorded-input checks and four Python-fallback checks**, without skips.++ [Final installation checks](https://github.com/1337isnot1337/DOPPLERGEIST/actions/runs/41708837646)++ also pass on Python 3.11 and 3.14. HTTP CI jobs remain outside this branch's push gate.++++ ## Full HTTP and refinement measurements++++ | Complete HTTP series | Baseline median | Current median | Result |++ | --- | ---: | ---: | --- |++ | Initial search changes | 54.020778 s | 54.855469 s | 1.55% more time |++ | Search plus refinement reuse | 58.233276 s | 52.668349 s | 9.56% less time |++++ Each series uses three fresh alternating pairs. The initial series establishes++ **no complete-pipeline speedup**. Its narrower request has little dense-query cost;++ producer search timing is essentially unchanged, at 22.759 versus 22.739 seconds.++ Each current run in the later refinement series is faster than its paired baseline.++++ Every pair passes exact scientific JSON, logical SQLite, decoded audio, CSV-array,++ and PNG-pixel comparisons. All uploads traverse intake, discovery, search, and++ refinement, returning the same 23 candidates and 17 event records.++ The request remains `inconclusive / search_incomplete`: 40 refinement points are++ searched and 80 are skipped. All 15 independent event groups are considered.++ Faster execution does not establish complete coverage or an accepted location.++++ The later series adds these exact files to the earlier runtime:++++ - `matching.py`: `8d7c6b8f7bf83f06203640df32136d1dbe625cd558270bc7b4184305f9024826`.++ - `terrain.py`: `9390f39b25f41ee03e567b3c9c83fce9bf0a0285dc90f842124a39c93e43c8b2`.++++ The request, complete recording, index, terrain, and budgets remain unchanged.++ Each verifier checks source stability. Verifier elapsed time includes service++ startup, polling, shutdown, and artifact checks. Phase observations have one-second++ polling uncertainty. Unchanged intake code also varies between runs, so the entire++ HTTP saving cannot be attributed to terrain alone. Compare paired arms within a++ series; absolute times across separate experiments have different host conditions.++++ A separate complete sensitivity-profile comparison isolates refinement reuse.++ Median wall time falls from **9.064 to 8.178 seconds**, or **9.78% less time**.++ Median CPU time falls from 9.054 to 8.175 seconds.++ All six runs return exactly equal complete profiles and retain all **46 solves**.++ Each performs 36,835 native acoustic calls. Input loading and earlier finalist++ refinement are outside this component timer.++++ Pixel reuse applies to `terrain.Terrain` with local operator GeoTIFF files.++ The online provider's `data_access.CachedTerrain` is unchanged. Profile-local++ transformer reuse applies with either provider. The HTTP measurement uses local++ rasters and makes no online-provider throughput claim.++++ The final `b98a0fa3` HTTP series has **two complete exact pairs and five verified++ upload receipts**. Its third pair was interrupted before the last receipt.++++ | Completed final-runtime pair | Baseline | Current | Paired ratio |++ | --- | ---: | ---: | ---: |++ | 1, baseline then current | 58.389875 s | 51.997905 s | 1.122927× |++ | 2, current then baseline | 57.154603 s | 54.008309 s | 1.058256× |++++ The unpaired third baseline took 59.612323 seconds and remains in the record.++ The interrupted current attempt has no valid elapsed-time sample. The incomplete++ series has `speedup: null`; no three-pair aggregate is reported.++ The session's final exit status could not be recovered. These partial observations++ do not establish a three-pair speedup for the final runtime. The completed++ refinement series above remains the measured full-HTTP result.++++ ## Discovery experiment and final correctness++++ The discovery normalization prototype reduces median stage wall time from++ **6.156 to 5.761 seconds**, or **6.43% less time**, across three alternating pairs.++ It processes the complete 3,600.1845-second saved PCM recording and writes the++ normal JSON, 17 CSV files, and summary PNG. Decoding and search are excluded.++ Pre-call imports are excluded; lazy plotting imports inside discovery are included.++ Median peak RSS increases from 159.953 to 165.883 MiB, a 5.930 MiB tradeoff.++++ This timing precedes the final guards for C-contiguous arrays and canonical NumPy++ functions. It is **not a separate timing claim for the final runtime**.++ The final implementation passes a new full-recording comparison: all 60 spectra++ match in bytes, dtype, shape, strides, and mutability. Complete event records,++ CSV bytes, and diagnostic pixels also match. Eighteen portable real-PCM tests++ cover odd/even medians, custom hooks, data types, layouts, and short inputs.++ The final combined HTTP series supplies two exact integration pairs; its planned++ three-pair timing series remains incomplete.++++ ## What changes++++ - **Prepared trajectory inputs:** indexes store numeric constructor inputs beside++ the original compressed observations. Values follow original decoding and++ normalization, preserving rounding, inferred velocities, and altitude labels.++ - **Vectorized trial bounds:** masks and conservative bounds operate across++ eligible clocks. Event order, upward rounding, strict rejection, diagnostics,++ and custom-function fallbacks remain unchanged.++ - **Native acoustic reuse:** the solver reuses exact spline intervals and exact++ fixed points. Retired samples still contribute to the global stopping condition.++ Tolerances and iteration limits remain unchanged.++ - **Refinement reuse:** each profile shares its unchanged coordinate transformers.++ Each local sampler retains at most 1,024 exact single-pixel masked reads.++ Pixel selection, raster precedence, missing data, geoid correction, and returned++ array mutability remain unchanged. Changed or custom readers use original reads.++ - **Discovery normalization:** contiguous frequency columns accelerate median++ reductions for eligible finite float32 arrays. Returned layout and every++ arithmetic expression remain unchanged. Other inputs use the original path.++++ Prepared data uses little-endian float64 arrays and bounded JSON metadata, with++ no executable object format. Tracks remain independently mutable. Original++ payload integrity checks remain mandatory, including cache hits. Prepared hashes++ and implementation identity must match; incompatible optional data uses original++ payloads. Runtime identity includes normalization code, dependencies, Python, and++ CPU capabilities. Operators can prepare a new copy after changing the runtime.++ The acoustic scratch allocation is bounded at 2 MiB; larger requests keep the++ original native calculation. No optimization changes search budgets or resolution.++++ Local validation includes 70 focused search tests, 13 refinement/terrain tests,++ and 18 discovery tests, without skips. The refinement cases cover closed,++ replaced, changed, and custom readers, masked pixels, bounded eviction, and hooks.++++ ## Measurement method++++ 1. Freeze real observations, request settings, source hashes, and workload counts.++ Keep complete outputs so comparisons can detect changed scientific work.++ 2. Measure the full workflow first. Profile separately to locate substantial costs;++ exclude profiler instrumentation from reported timing pairs.++ 3. Optimize repeated work while preserving operations, bounds, and scenario counts.++ Reject faster results that omit candidates, diagnostics, arrays, or coverage records.++ 4. Compare full values exactly before accepting timings. Include ordered inventory,++ candidate fields, logical retention rows, and array bytes and layouts where relevant.++ 5. Use fresh interpreters or services in alternating baseline/current order.++ Use one numerical thread and pause other team benchmarks during each comparison.++ 6. Record wall time, CPU time, memory, preparation, and storage separately.++ Report workload-specific medians and all samples; never multiply component ratios.+++++The current prepared-flight harness leaves numerical functions unchanged during+++default comparisons. It uses passive native counters when those counters exist.+++Unmeasured counters are null.+++`--instrument-numerics`, `--stage-timing`, or active Python profiling set+++`performance_timing_eligible=false`. These diagnostic reports use `diagnostic_ratio`+++and leave production speedup and preparation amortization null.+++++ Local runs do not pin CPU affinity or flush operating-system caches. Unrelated++ host load is uncontrolled. CI timing pins one available CPU. Three pairs are a++ small performance sample, so retain the workload and variability qualifications.++ Native backends and prepared decoding must actually execute when required.++ Every measured current dense run records 4,694 successful prepared decodes.++++ ## Rejected and deferred experiments++++ | Experiment | Observed result | Decision |++ | --- | --- | --- |++ | FFmpeg `filter_threads=1`, three pairs | 8.195 → 8.644 s median | Slower; rejected |++ | FFmpeg `flush_packets=0`, one pair | 8.444 → 7.751 s | Below the declared 10% time-reduction gate; stopped |++ | FFmpeg direct-file output, one pair | 7.689 → 7.915 s | Slower; rejected |++ | Native ridge IRLS fixed points, five pairs | 0.212 → 0.226 s median | Slower; rejected |++++ Decoder trials preserve all 57,602,952 PCM bytes, sample counts, and peak amplitude.++ The IRLS trial preserves every result byte from 36,835 actual calls, but this++ kernel is too small to dominate the full refinement profile. These changes are++ not integrated. Single-pair results are screening observations, not stable medians.++ An earlier nominal-solve reuse prototype was simplified to transformer reuse;++ all 46 final solves remain, with fewer custom-hook guards.++++ Raw-STFT overlap reuse is **unimplemented and unmeasured**. The real recording has++ 14,100 spectral frames but 7,197 distinct aligned windows: 6,903 transforms repeat.++ However, raw spectrogram work occupies only 0.693 profiled seconds. At equal++ per-frame cost, perfect reuse of the repeated 48.96% would save about 0.339 seconds.++ This idealized estimate helped prioritize normalization instead. It is not a cache++ speedup. A future cache must preserve windows and recompute chunk-dependent statistics.++ Bounded chunk concurrency also remains unimplemented; results must enter merging++ and harmonic grouping in original order.++++ An earlier decoding-only experiment measured 7.929× on 4,694 trajectories against++ `8bd72aed`. It excludes SQL reads and cache hits, and does not describe full search.++++ ## Preparation cost and operator use++++ | One-time preparation | Result |++ | --- | ---: |++ | Wall / CPU time | 7.418266 / 7.370144 s |++ | Original SQLite file | 41,385,984 bytes |++ | Prepared SQLite copy | 85,929,984 bytes |++ | Added storage | 44,544,000 bytes |++++ Preparation copies the index, decodes canonical observations, writes the optional++ representation, and commits. The original remains unchanged. Input identity,++ source records, and coverage declarations stay equal. Original payloads remain++ available for compatibility and fallback.++ The timer excludes imports and separate whole-file integrity verification.++ That verification reads the source before timing, warming its file pages.-Two comparable searches recover the measured preparation cost. This estimate-uses the combined code-and-format saving, not an isolated format benefit.+++The historical two-search break-even estimate used the instrumented dense timings.+++It does not establish production amortization. A new estimate requires+++uninstrumented comparisons in the integrated runtime.++++ Prepare an existing index in the serving runtime. Choose a new destination:++++ ```bash++ python scripts/prepare_flight_index.py \++ --index /absolute/path/original.sqlite \++ --output /absolute/path/prepared.sqlite++ ```++++ Set the operator configuration's index path:++++ ```json++ {"flight_indexes": ["/absolute/path/prepared.sqlite"]}++ ```++++ Keep the existing terrain, coverage, and search configuration. Start the service++ as described in [PRODUCT.md](PRODUCT.md). New indexes include prepared inputs by++ default. Unprepared indexes remain usable; preparation does not expand coverage.++++ ## Reproduce the comparisons++++ Use the final checkout containing these scripts, Python 3.12, GCC, and the same++ dependencies for all source trees. Build the pinned dense runtime and baseline:++++ ```bash++ export PYTHONPATH=src++ export CC=gcc++ export OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1++ export MKL_NUM_THREADS=1 NUMEXPR_NUM_THREADS=1++ python -m pip install 'setuptools>=68'++ python -m pip install --editable '.[test]'++ git worktree add --detach ../dopplergeist-baseline \++ 994d1ee48e25df6d30b23a356bba8f6502d8b7f6++ git worktree add --detach ../dopplergeist-dense \++ ecb1726fb9872d0060265014aa830e598b4dd3b4++ (cd ../dopplergeist-baseline && python setup.py build_ext --inplace)++ (cd ../dopplergeist-dense && python setup.py build_ext --inplace)++ ```++++ Restore public observations. Build the original index with baseline code, then++ prepare a common copy with the measured dense runtime:++++ ```bash++ python scripts/restore_artifacts.py --bundle data-adsb-derived --bundle data-terrain++ PYTHONPATH=../dopplergeist-baseline/src \++ python scripts/prepare_reference_upload.py --data-root data --output benchmark-inputs++ python scripts/benchmark_prepared_measurements.py prepare \++ --discovery evidence/audio/w1-sensitivity-20261102/reference/discovery.json \++ --index benchmark-inputs/reference-region.sqlite \++ --reference-report evidence/results/reference-local/report.json \++ --full-day --max-tracks 2000 --max-samples 500000 --max-candidates 20000 \++ --anchor-window-s 1788864888.7151732 1788865236.7214916 \++ --include-track-query --output benchmark-inputs/dense-query.json++ python scripts/benchmark_prepared_flight_search.py prepare-index \++ --source ../dopplergeist-dense/src --index benchmark-inputs/reference-region.sqlite \++ --destination benchmark-inputs/reference-region-prepared.sqlite \++ --output results/flight-preparation.json++ python scripts/benchmark_prepared_flight_search.py compare \++ --fixture benchmark-inputs/dense-query.json \++ --baseline-source ../dopplergeist-baseline/src --current-source ../dopplergeist-dense/src \++ --baseline-index benchmark-inputs/reference-region-prepared.sqlite \++ --current-index benchmark-inputs/reference-region-prepared.sqlite \++ --preparation-report results/flight-preparation.json \++ --repeats 3 --timeout-s 600 --output results/performance-prepared-flight-search \++ --require-native --require-native-acoustic --require-current-prepared++ ```++++ ### Bounded HTTP replay++++ Use the existing `benchmark-inputs` directory from the preparation steps above.++ The measured upload is the archived **43,202,925-byte MP3**, with SHA-256++ `707083f0344823eeea7f130faec6c22b6628b8b9fea04d026915d93df9a07b15`.++ Place that exact file at `benchmark-inputs/recording.mp3`. The public WAV or a new++ MP3 encoding is a **new workload**: decoding cost, decoded samples, and downstream++ results can differ. Do not attribute the archived timing results to that substitute.++++ Set the exact bounded request and one-worker operator configuration. Both arms++ must read the same prepared index. Keep the restored terrain paths and the++ preparation helper's search budgets unchanged:++++ ```bash++ cp /absolute/path/archived-recording.mp3 benchmark-inputs/recording.mp3++ python - <<'PY'++ import hashlib++ import json++ from pathlib import Path++++ inputs = Path("benchmark-inputs")++ recording = inputs / "recording.mp3"++ assert recording.stat().st_size == 43202925++ with recording.open("rb") as stream:++ assert hashlib.file_digest(stream, "sha256").hexdigest() == "707083f0344823eeea7f130faec6c22b6628b8b9fea04d026915d93df9a07b15"++ request = {++ "schema_version": 1,++ "search": {"coverage": "conus", "bounds_wsen": [-73.545, 40.865, -73.505, 40.895]},++ "time_window": {++ "start_utc": "2026-09-08T20:17:20Z", "end_utc": "2026-09-08T20:17:40Z",++ "source": "user",++ "source_detail": "Source-informed regional runtime control using the historical fitted clock and the published four-point receiver region; not independent timing evidence or a location-accuracy evaluation.",++ },++ }++ operator = json.loads((inputs / "operator.json").read_text())++ operator["flight_indexes"] = [str((inputs / "reference-region-prepared.sqlite").resolve(strict=True))]++ operator["search_workers"] = 1++ assert operator["search_budget"] == {"spacing_m": 2000, "tile_cells": 16, "top_k": 20,++ "refine_spacing_m": 500, "refine_finalists": 3}++ assert operator["refine_finalists"] == 3 and operator["offline"] is True++ for name, value in (("request.json", request), ("operator.json", operator)):++ (inputs / name).write_text(json.dumps(value, indent=2) + "\n")++ PY++ python scripts/benchmark_real_bounded_comparison.py \++ --baseline-source ../dopplergeist-baseline/src --current-source src \++ --inputs benchmark-inputs --output results/performance-bounded-http \++ --repeats 3 --timeout-s 900++ ```++++ This command runs three alternating pairs with fresh services and jobs. It++ requires `inconclusive / search_incomplete`, complete workflow receipts, and exact++ scientific and array comparisons. Preparation is outside the timer. The controller++ preserves logs and receipts, and reports a speedup only after all requested pairs++ complete successfully. An interrupted run must remain an incomplete series.++++ ### Refinement and discovery++++ For refinement, build the public component baseline. The committed fixture++ contains the real finalist, unchanged discovery observations, and provenance:++++ ```bash++ git worktree add --detach ../dopplergeist-component-baseline \++ c0c9e798f47b547ab581fa473912e7218e3b4c3e++ (cd ../dopplergeist-component-baseline && python setup.py build_ext --inplace)++ python scripts/benchmark_real_refinement.py compare \++ --baseline-source ../dopplergeist-component-baseline/src --current-source src \++ --index benchmark-inputs/reference-region-prepared.sqlite --terrain-root . \++ --repeats 3 --require-native --require-native-acoustic \++ --output results/performance-refinement++ ```++++ This timer covers the complete profile and its exact-fit cache, excluding++ imports, index lookup, terrain construction, event preparation, and earlier++ finalist refinement. Every arm writes its full result and source hashes.++ The portable wrapper has AST, CLI-help, and static validation. The reported++ component timings came from the earlier tested harness on these same recorded++ inputs; the optional portable-wrapper timing smoke was canceled.++++ Discovery requires a saved real service job with decoded PCM, intake metadata,++ and discovery artifacts. Replace the job path with that existing directory:++++ ```bash++ python scripts/benchmark_real_discovery_normalization.py compare \++ --baseline-source ../dopplergeist-component-baseline/src --current-source src \++ --job /absolute/path/saved-job --pairs 3 \++ --output results/performance-discovery++ ```++++ The discovery script checks all spectra outside timing. It continues three-pair++ screening only when its first exact pair clears the declared improvement threshold.++ The completed HTTP comparisons supply separate integration checks.++ Use new output directories. Commands fail on unequal outputs or unused required++ backends. Their reports contain samples and medians; workers preserve complete++ comparison outputs. Keep profiling separate and avoid competing benchmarks.++++ The [performance workflow](../.github/workflows/real-performance.yml) contains++ recorded-input regression commands and the dense comparison. It uploads the++ preparation report and pins one CPU for timing. Local full HTTP replays remain++ separate from the branch's automatic dense and static CI gates.diff --cc scripts/benchmark_prepared_flight_search.pyindex 0000000,0000000,b70e45e..67aa440mode 000000,000000,100644..100644--- a/scripts/benchmark_prepared_flight_search.py+++ b/scripts/benchmark_prepared_flight_search.py@@@@ -1,0 -1,0 -1,484 +1,529 @@@@++ """Measure real flight-query, overload-sieve and scoring work with exact parity.++++ Use a frozen recorded-data fixture and two pinned src/ directories. Each arm runs++ in a fresh interpreter. An optional prepared index changes representation only;++ its one-time construction is measured separately by the prepare-index command.++ The source-informed regional receiver and anchor partition are performance++ workloads, not independent localization or complete national-search evidence.+++Default timings leave numerical and decoder functions unchanged. Use+++--instrument-numerics or --stage-timing for separate diagnostics; their ratios+++are never reported as performance speedups or preparation amortization.++ """++ from __future__ import annotations++++ import argparse++ import hashlib++ import json++ import math++ import os++ from pathlib import Path++ import statistics++ import subprocess++ import sys++ import time++++ try:++ import resource++ except ImportError:++ resource = None++++ ROOT = Path(__file__).resolve().parents[1]++++ # Query and scientific-output capture adapted from the project's PR #258++ # benchmark_prepared_measurements.py; keep the original scoring harness intact.++++++ def json_scalar(value):++ if hasattr(value, "tolist"):++ return value.tolist()++ raise TypeError(f"Cannot serialize {type(value).__name__}")++++ def write_new(path, value):++ path = Path(path)++ path.parent.mkdir(parents=True, exist_ok=True)++ with path.open("x") as output:++ json.dump(value, output, indent=2, allow_nan=True, default=json_scalar)++ output.write("\n")++++ def canonical(value):++ return json.dumps(value, sort_keys=True, separators=(",", ":"),++ allow_nan=True, default=json_scalar)++++ def query_tracks(index, points, events, budget, scoring, bounds, time_range):++ """Count actual index traversal while exercising the production overload path."""++ from dopplergeist.tiled_search import _tracks_for_points++++ if "elev_m" not in points:++ raise ValueError("The recorded receiver hypothesis must include its elevation")++ operations = {"query_calls": 0, "decoded_track_yields": 0, "decoded_sample_yields": 0}++ inventory = {}++ ordered_inventory = []++++ class CountedIndex:++ def query(self, query_bounds, query_time):++ operations["query_calls"] += 1++ for track in index.query(query_bounds, query_time):++ operations["decoded_track_yields"] += 1++ operations["decoded_sample_yields"] += len(track.time)++ inventory[track.id] = len(track.time)++ ordered_inventory.append([track.id, len(track.time)])++ yield track++++ radius = max(event["slant_range_m"][1] for event in events)++ tracks, points = _tracks_for_points(++ [CountedIndex()], bounds, time_range, budget, points, None, radius,++ scoring.get("c", 343.0))++ accounting = {++ "queried_tracks_in_order": ordered_inventory,++ "queried_track_ids": sorted(inventory),++ "queried_trajectory_samples": sum(inventory.values()),++ "selected_track_ids": [track.id for track in tracks],++ "selected_trajectory_samples": sum(len(track.time) for track in tracks),++ }++ return tracks, points, operations, accounting++++ def worker(args):++ source = args.source.resolve()++ sys.path.insert(0, str(source))++ import dopplergeist++ import pandas as pd++ from dopplergeist import geometry, matching, tiled_search++ from dopplergeist.data_access import FlightIndex++ from dopplergeist.finite_search import RetentionLedger++ from dopplergeist.geometry import AcousticSupportError, Track++++ if Path(dopplergeist.__file__).resolve().parent != source / "dopplergeist":++ raise RuntimeError("Comparison imported a different source tree")++ native_fit = getattr(matching, "_native_fit_irls", None)+++ native_module = getattr(matching, "_native_ridge_module", None)+++ native_statistics = getattr(native_module, "fit_statistics", None)++ ridge_backend = getattr(matching, "RIDGE_FIT_BACKEND", "python-irls-v1")++ acoustic_backend = getattr(geometry, "ACOUSTIC_BACKEND", "python-acoustic-v1")++ try:++ from dopplergeist.native_acoustic import _native as native_acoustic++ except ImportError:++ native_acoustic = None++ if args.require_native and not callable(native_fit):++ raise RuntimeError("The selected source's native ridge fitter is unavailable")++ if args.require_native_acoustic and native_acoustic is None:++ raise RuntimeError("The selected source's native acoustic solver is unavailable")++ fixture = json.loads(args.fixture.read_text())++ index = FlightIndex(args.index or fixture["real_index"],++ track_cache_bytes=args.track_cache_bytes)++ if index.report["input_hash"] != fixture["index_input_hash"]:++ raise ValueError("The real trajectory index no longer matches the fixture")++ if args.require_prepared and not getattr(index, "_prepared_tracks", False):++ raise RuntimeError("The selected source cannot use this prepared index")++ budget = tiled_search.SearchBudget(**fixture["budget"])++ loaded_at = time.perf_counter()++ tracks = None++ if not args.include_track_query:++ tracks = list(index.lookup(fixture["track_ids"], max_tracks=budget.max_tracks_per_tile,++ max_samples=budget.max_track_samples_per_tile))++ if sorted(track.id for track in tracks) != sorted(fixture["track_ids"]):++ raise ValueError("The index is missing a recorded trajectory")++ events, measurements = matching.prepare_events(fixture["events"])++ points = pd.DataFrame([fixture["point"]])++ load_seconds = time.perf_counter() - loaded_at++ args.output.mkdir(parents=True, exist_ok=False)++ ledger = RetentionLedger(args.output / "retention.sqlite", "recorded-point-benchmark",++ "matched_group_support_minus_normalized_residual")++ stage = "coarse:recorded-point"++ operations = {"scalar_fit_calls": 0, "acoustic_calls": 0, "scored_candidates": 0,++ "native_fit_calls": 0, "native_fit_successes": 0,++ "native_acoustic_successes": 0, "prepared_decode_calls": 0,++ "prepared_decode_successes": 0}++ all_rows = []++ original_fit, original_acoustic = matching._fit_ridge_uncached, Track.acoustic++ original_score = tiled_search.score_candidate++ phase_seconds, phase_patches = {}, []++ original_decoder, payload_module = None, None- if getattr(index, "_prepared_tracks", False):+++ if args.instrument_numerics and getattr(index, "_prepared_tracks", False):+++ # Leave data_access._frame_track unchanged even in diagnostics: its+++ # identity guards the prepared decoder. Count the selected decoder only.++ from dopplergeist import flight_payload as payload_module++ original_decoder = payload_module.decode_track++++ def counted_decode(*values, **options):++ operations["prepared_decode_calls"] += 1++ track = original_decoder(*values, **options)++ operations["prepared_decode_successes"] += 1++ return track++++ payload_module.decode_track = counted_decode++++ def timed_phase(name, function):++ def run(*values, **options):++ began, cpu = time.perf_counter(), time.process_time()++ try:++ return function(*values, **options)++ finally:++ phase = phase_seconds.setdefault(name, {"wall_seconds": 0.0, "cpu_seconds": 0.0, "calls": 0})++ phase["wall_seconds"] += time.perf_counter() - began++ phase["cpu_seconds"] += time.process_time() - cpu++ phase["calls"] += 1++ return run++++ if args.stage_timing:++ phase_targets = [++ (Track, "outside_observer_radius", "distance_sieve"),++ (matching, "association_candidates", "association_enumeration"),++ (tiled_search, "merge_candidates", "retention"),++ ]++ if callable(getattr(tiled_search, "cpa_table", None)):++ phase_targets.append((tiled_search, "cpa_table", "closest_approach"))++ else:++ prepared_cpa = getattr(tiled_search, "_PreparedCPATrack", None)++ if prepared_cpa is not None:++ phase_targets.extend([++ (prepared_cpa, "__init__", "closest_approach_preparation"),++ (prepared_cpa, "raw_rows", "closest_approach"),++ ])++ for owner, attribute, name in phase_targets:++ function = getattr(owner, attribute, None)++ if function is not None:++ phase_patches.append((owner, attribute, function))++ setattr(owner, attribute, timed_phase(name, function))++ score_function = timed_phase("candidate_scoring", original_score) if args.stage_timing else original_score++++ def counted_fit(*values, **options):++ operations["scalar_fit_calls"] += 1++ return original_fit(*values, **options)++++ def counted_acoustic(*values, **options):++ operations["acoustic_calls"] += 1++ return original_acoustic(*values, **options)++++ def counted_native(*values, **options):++ operations["native_fit_calls"] += 1++ result = native_fit(*values, **options)++ if result is not None and all(math.isfinite(value) for value in result):++ operations["native_fit_successes"] += 1++ return result++++ def captured_score(*values, **options):++ operations["scored_candidates"] += 1++ try:++ result = score_function(*values, **options)++ except AcousticSupportError as error:++ all_rows.append({"support_failure": tiled_search._support_error(error)})++ raise++ all_rows.append(result)++ return result++- matching._fit_ridge_uncached, Track.acoustic = counted_fit, counted_acoustic+++ if args.instrument_numerics:+++ matching._fit_ridge_uncached, Track.acoustic = counted_fit, counted_acoustic+++ else:+++ for name in ("scalar_fit_calls", "acoustic_calls", "native_fit_calls", "native_fit_successes",+++ "prepared_decode_calls", "prepared_decode_successes"):+++ operations[name] = None++ tiled_search.score_candidate = captured_score- if callable(native_fit):+++ native_before = native_statistics() if callable(native_statistics) else None+++ if args.instrument_numerics and callable(native_fit) and native_before is None:++ matching._native_fit_irls = counted_native++ native_acoustic_before = native_acoustic.successful_calls() if native_acoustic is not None else 0++ query_accounting, query_operations, execution_stages = None, None, {}++ began, cpu = time.perf_counter(), time.process_time()++ try:++ if args.include_track_query:++ query_began, query_cpu = time.perf_counter(), time.process_time()++ tracks, points, query_operations, query_accounting = query_tracks(++ index, points, events, budget, fixture["scoring"],++ fixture["query_bounds"], fixture["query_time_s"])++ execution_stages["track_query_and_distance_sieve"] = {++ "wall_seconds": time.perf_counter() - query_began,++ "cpu_seconds": time.process_time() - query_cpu,++ }++ if query_accounting["selected_track_ids"] != fixture["track_ids"]:++ raise AssertionError("The real query changed the ordered selected-track inventory")++ if query_accounting["selected_trajectory_samples"] != fixture["counts"]["trajectory_samples"]:++ raise AssertionError("The real query changed the selected trajectory sample count")++ prepared_query = fixture.get("track_query_preparation")++ if prepared_query is not None:++ if (len(query_accounting["queried_track_ids"]) != prepared_query["queried_trajectories"]++ or query_accounting["queried_trajectory_samples"] != prepared_query["queried_trajectory_samples"]):++ raise AssertionError("The real query changed the complete observed-track inventory")++ scoring_began, scoring_cpu = time.perf_counter(), time.process_time()++ partition = ({"anchor_window": fixture["anchor_window_s"]}++ if "anchor_window_s" in fixture else {})++ ranked, counts = tiled_search._score(++ points, tracks, events, measurements, budget, fixture["scoring"], {},++ start_window=fixture["recording_start_window_s"], ledger=ledger, stage=stage, **partition)++ if args.include_track_query:++ execution_stages["cpa_association_scoring_and_retention"] = {++ "wall_seconds": time.perf_counter() - scoring_began,++ "cpu_seconds": time.process_time() - scoring_cpu,++ }++ finally:++ elapsed, cpu_seconds = time.perf_counter() - began, time.process_time() - cpu- matching._fit_ridge_uncached, Track.acoustic = original_fit, original_acoustic++ tiled_search.score_candidate = original_score- if callable(native_fit):- matching._native_fit_irls = native_fit+++ if args.instrument_numerics:+++ matching._fit_ridge_uncached, Track.acoustic = original_fit, original_acoustic+++ if callable(native_fit) and native_before is None:+++ matching._native_fit_irls = native_fit+++ if native_before is not None:+++ native_after = native_statistics()+++ delta = {key: native_after[key] - native_before[key] for key in native_before}+++ operations["native_fit_calls"] = delta["scalar_calls"] + delta["batch_rows"]+++ operations["native_fit_successes"] = delta["scalar_successes"] + delta["batch_successful_rows"]+++ operations["native_batch_calls"] = delta["batch_calls"]+++ operations["native_batch_rows"] = delta["batch_rows"]++ if native_acoustic is not None:++ operations["native_acoustic_successes"] = native_acoustic.successful_calls() - native_acoustic_before++ for owner, attribute, function in reversed(phase_patches):++ setattr(owner, attribute, function)++ if payload_module is not None:++ payload_module.decode_track = original_decoder++ expected_counts = fixture.get("expected_counts")++ if expected_counts is not None:++ actual_counts = {"cpa_rows": counts["cpa_rows"],++ "associations": counts["candidates"], "retained": len(ranked)}++ if actual_counts != expected_counts:++ raise AssertionError(f"Recorded workload counts changed: {actual_counts}")++ # macOS reports bytes and Linux reports KiB. Keep scoring usable where++ # resource or its unit convention is unavailable, without guessing a value.++ peak_rss_mib = None++ if resource is not None and (sys.platform == "darwin" or sys.platform.startswith("linux")):++ rss_divisor = 1024 * 1024 if sys.platform == "darwin" else 1024++ peak_rss_mib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rss_divisor++ checksum = ledger.checksum(stage)++ if not ledger.stage_valid(stage, checksum):++ raise AssertionError("The scoring pass produced an invalid retention ledger")++ if not operations["scored_candidates"]:++ raise AssertionError("The benchmark did not score any real associations")- if args.require_native and not operations["native_fit_successes"]:+++ if (args.require_native and operations["native_fit_successes"] is not None+++ and not operations["native_fit_successes"]):++ raise AssertionError("No successful native ridge fit occurred inside the timed scoring call")++ if args.require_native_acoustic and not operations["native_acoustic_successes"]:++ raise AssertionError("No successful native acoustic solve occurred inside the timed scoring call")- if args.require_prepared and not operations["prepared_decode_successes"]:+++ if (args.require_prepared and operations["prepared_decode_successes"] is not None+++ and not operations["prepared_decode_successes"]):++ raise AssertionError("No successful prepared decode occurred inside the timed query")++ with ledger.connect() as database:++ retention = {name: database.execute(f"SELECT * FROM {name} ORDER BY 1,2").fetchall()++ for name in ("records", "bounds", "stages")}++ results = {"ranked": ranked, "counts": counts,++ "all_scored_candidates": all_rows, "retention": retention}++ if query_accounting is not None:++ results["track_query_accounting"] = query_accounting++ results["workload"] = {key: fixture.get(key) for key in (++ "index_input_hash", "point", "events", "query_bounds", "query_time_s",++ "recording_start_window_s", "anchor_window_s", "budget", "scoring")}++ results["index_coverage"] = index.report["coverage"]++ write_new(args.output / "results.json", results)+++ diagnostic = args.instrument_numerics or args.stage_timing or sys.getprofile() is not None++ summary = {"wall_seconds": elapsed, "cpu_seconds": cpu_seconds,++ "load_seconds_excluded": load_seconds, "operations": operations,+++ "numerical_functions_canonical": not (args.instrument_numerics or args.stage_timing),+++ "performance_timing_eligible": not diagnostic,+++ "measurement_mode": "diagnostic" if diagnostic else "canonical_timing",+++ "ridge_execution_check": ("In-module successful-call counters" if native_before is not None else+++ "Wrapped successful-call counters" if args.instrument_numerics else+++ "Backend availability; successful-call count unmeasured"),+++ "prepared_execution_check": ("Wrapped successful-call counters" if args.instrument_numerics else+++ "Compatible representation availability; successful-call count unmeasured"),++ "ridge_fit_backend": ridge_backend, "acoustic_backend": acoustic_backend,++ "native_extension_available": callable(native_fit),++ "peak_rss_mib": peak_rss_mib,++ "peak_rss_scope": ("Worker lifetime through scoring, including imports and input loading"++ if peak_rss_mib is not None else "Unavailable on this platform"),++ "retained": len(ranked), "independent_groups": len(events), "scope": fixture["scope"]}++ summary["index_prepared_tracks"] = index.report.get("prepared_tracks")++ summary["track_cache_bytes"] = args.track_cache_bytes++ if "anchor_window_s" in fixture:++ summary["anchor_window_s"] = fixture["anchor_window_s"]++ if args.include_track_query:++ summary["track_query_operations"] = query_operations++ summary["execution_stage_seconds"] = execution_stages++ summary["timing_scope"] = (++ "Initial local index traversal and conservative distance sieve, followed by complete "++ "CPA/association scoring and retention. Excludes imports, index metadata, event "++ "preparation, downloads, terrain sampling, audio discovery, and refinement.")++ if args.stage_timing:++ summary["phase_seconds"] = phase_seconds++ summary["phase_timing_scope"] = "Inclusive instrumented call times; overlapping phases must not be summed."++ write_new(args.output / "summary.json", summary)++ print(json.dumps(summary), flush=True)++++++ def file_digest(path):++ with Path(path).open("rb") as stream:++ return hashlib.file_digest(stream, "sha256").hexdigest()++++++ def prepare_index(args):++ """Time a new optional representation without changing the original index."""++ sys.path.insert(0, str(args.source.resolve()))++ import dopplergeist++ from dopplergeist.data_access import FlightIndex++++ if Path(dopplergeist.__file__).resolve().parent != args.source.resolve() / "dopplergeist":++ raise RuntimeError("Preparation imported a different source tree")++ if args.output.exists() or args.destination.exists():++ raise FileExistsError("Choose new preparation output and index paths")++ before = file_digest(args.index)++ began, cpu = time.perf_counter(), time.process_time()++ original = FlightIndex(args.index)++ prepared = original.prepare(args.destination)++ elapsed, cpu_seconds = time.perf_counter() - began, time.process_time() - cpu++ if before != file_digest(args.index):++ raise AssertionError("Preparation modified the original observed index")++ for key in ("input_hash", "coverage", "sources"):++ if original.report.get(key) != prepared.report.get(key):++ raise AssertionError(f"Preparation changed the original {key}")++ if not getattr(prepared, "_prepared_tracks", False):++ raise AssertionError("The optional representation is unavailable to its builder")++ report = {++ "wall_seconds": elapsed, "cpu_seconds": cpu_seconds,++ "source": str(args.source.resolve()), "index": str(args.index.resolve()),++ "destination": str(args.destination.resolve()),++ "input_hash": original.report["input_hash"],++ "original_unchanged": True,++ "source_bytes": args.index.stat().st_size,++ "prepared_bytes": args.destination.stat().st_size,++ "prepared_tracks": prepared.report["prepared_tracks"],++ "timing_scope": "One-time index copy, canonical decoding, optional representation and commit; excludes imports and verification.",++ "cache_scope": "Source integrity verification reads the original index before timing; operating-system file caches are not flushed.",++ }++ write_new(args.output, report)++ print(json.dumps(report), flush=True)++++++ def compare(args):++ args.output.mkdir(parents=True, exist_ok=False)++ sources = {"baseline": args.baseline_source.resolve(), "current": args.current_source.resolve()}++ fixture = json.loads(args.fixture.read_text())++ indexes = {"baseline": Path(args.baseline_index or fixture["real_index"]).resolve(),++ "current": Path(args.current_index or fixture["real_index"]).resolve()}++ environment = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")++ for variable in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS"):++ environment[variable] = "1"++ runs, expected = [], None++ for repetition in range(args.repeats):++ order = ("baseline", "current") if repetition % 2 == 0 else ("current", "baseline")++ for name in order:++ output = args.output / f"{repetition + 1}-{name}"++ command = [sys.executable, str(Path(__file__).resolve()), "_worker",++ "--source", str(sources[name]), "--fixture", str(args.fixture.resolve()),++ "--index", str(indexes[name]), "--output", str(output.resolve()),++ "--track-cache-bytes", str(args.track_cache_bytes)]- for flag in ("stage_timing", "require_native", "require_native_acoustic"):+++ for flag in ("stage_timing", "instrument_numerics", "require_native", "require_native_acoustic"):++ if getattr(args, flag):++ command.append("--" + flag.replace("_", "-"))++ if name == "current" and args.require_current_prepared:++ command.append("--require-prepared")++ print(f"Starting pair {repetition + 1}: {name}", flush=True)++ subprocess.run(command, check=True, env=environment, timeout=args.timeout_s)++ payload = canonical(json.loads((output / "results.json").read_text()))++ if expected is None:++ expected = payload++ elif payload != expected:++ raise AssertionError(f"Exact inventory, scientific-output or retention mismatch: {output}")++ summary = json.loads((output / "summary.json").read_text())++ runs.append({"source": name, "repetition": repetition + 1, **summary})++ medians = {name: {} for name in sources}++ for name in sources:++ for metric in ("wall_seconds", "cpu_seconds", "peak_rss_mib"):++ values = [row[metric] for row in runs if row["source"] == name and row[metric] is not None]++ medians[name][metric] = statistics.median(values) if values else None++ report = {++ "identical_results": True, "repeats": args.repeats,++ "fixture": str(args.fixture.resolve()),++ "sources": {name: str(path) for name, path in sources.items()},++ "indexes": {name: str(path) for name, path in indexes.items()},++ "medians": medians,++ "speedup": {metric: medians["baseline"][metric] / medians["current"][metric]++ for metric in ("wall_seconds", "cpu_seconds")},++ "execution_stage_medians": {++ name: {stage: {metric: statistics.median(++ row["execution_stage_seconds"][stage][metric]++ for row in runs if row["source"] == name)++ for metric in ("wall_seconds", "cpu_seconds")}++ for stage in runs[0]["execution_stage_seconds"]}++ for name in sources},++ "scope": runs[0]["scope"], "timing_scope": runs[0]["timing_scope"],++ "cache_scope": "Fresh interpreter and FlightIndex per arm; operating-system file caches are not flushed.",++ "runs": runs,++ }+++ report["performance_timing_eligible"] = (+++ not (args.instrument_numerics or args.stage_timing or sys.getprofile() is not None)+++ and all(row["performance_timing_eligible"] for row in runs))+++ if not report["performance_timing_eligible"]:+++ report["diagnostic_ratio"] = report["speedup"]+++ report["speedup"] = None++ if args.preparation_report:++ preparation = json.loads(args.preparation_report.read_text())++ if (Path(preparation["destination"]).resolve() != indexes["current"]++ or preparation["input_hash"] != fixture["index_input_hash"]):++ raise ValueError("Preparation report does not describe the compared current index")++ report["preparation"] = preparation++ report["preparation_amortization_scope"] = (++ "Preparation cost divided by the complete baseline/current query-and-score saving. "++ "This includes other code improvements and measures the combined upgrade, "++ "not the isolated prepared-format contribution.")++ report["queries_to_amortize_preparation"] = {}++ for metric in ("wall_seconds", "cpu_seconds"):++ saved = medians["baseline"][metric] - medians["current"][metric]++ report["queries_to_amortize_preparation"][metric] = (- max(1, math.ceil(preparation[metric] / saved)) if saved > 0 else None)+++ max(1, math.ceil(preparation[metric] / saved))+++ if report["performance_timing_eligible"] and saved > 0 else None)+++ if not report["performance_timing_eligible"]:+++ report["preparation_amortization_scope"] += " Unavailable for diagnostic timings."++ write_new(args.output / "comparison.json", report)++ print(json.dumps({key: report[key] for key in (++ "identical_results", "medians", "speedup", "execution_stage_medians", "scope")}), flush=True)++++++ def main():++ parser = argparse.ArgumentParser(description=__doc__)++ commands = parser.add_subparsers(dest="command", required=True)++ preparation = commands.add_parser("prepare-index", help="Measure one-time creation of an optional index representation")++ preparation.add_argument("--source", type=Path, default=ROOT / "src")++ preparation.add_argument("--index", type=Path, required=True)++ preparation.add_argument("--destination", type=Path, required=True)++ preparation.add_argument("--output", type=Path, required=True)++ comparison = commands.add_parser("compare", help="Compare equal real query/sieve/scoring workloads")++ comparison.add_argument("--fixture", required=True, type=Path)++ comparison.add_argument("--baseline-source", required=True, type=Path)++ comparison.add_argument("--current-source", type=Path, default=ROOT / "src")++ comparison.add_argument("--baseline-index", type=Path)++ comparison.add_argument("--current-index", type=Path)++ comparison.add_argument("--preparation-report", type=Path)++ comparison.add_argument("--repeats", type=int, default=3)++ comparison.add_argument("--timeout-s", type=float, default=900.0)++ comparison.add_argument("--output", required=True, type=Path)- comparison.add_argument("--require-current-prepared", action="store_true")+++ comparison.add_argument("--require-current-prepared", action="store_true",+++ help="Require a compatible prepared index; count successful decodes in diagnostics")++ internal = commands.add_parser("_worker", help=argparse.SUPPRESS)++ internal.add_argument("--source", required=True, type=Path)++ internal.add_argument("--fixture", required=True, type=Path)++ internal.add_argument("--index", type=Path)++ internal.add_argument("--output", required=True, type=Path)- internal.add_argument("--require-prepared", action="store_true")+++ internal.add_argument("--require-prepared", action="store_true",+++ help="Require a compatible prepared index; count successful decodes in diagnostics")++ for command in (comparison, internal):++ command.add_argument("--track-cache-bytes", type=int, default=32 * 1024 * 1024)++ command.add_argument("--stage-timing", action="store_true",- help="Collect inclusive call times in a separate diagnostic run")+++ help="Diagnostic phase timings; do not report a performance speedup")+++ command.add_argument("--instrument-numerics", action="store_true",+++ help="Diagnostic numerical/decoder counters; wrappers may disable guarded optimizations")++ command.add_argument("--require-native", action="store_true",- help="Require successful native ridge fits in every compared source")+++ help="Require native ridge backends; verify successful calls when counters are available")++ command.add_argument("--require-native-acoustic", action="store_true",++ help="Require successful native acoustic solves in every compared source")++ command.set_defaults(include_track_query=True)++ args = parser.parse_args()++ if hasattr(args, "track_cache_bytes") and args.track_cache_bytes < 0:++ parser.error("--track-cache-bytes must be nonnegative")++ if args.command == "prepare-index":++ prepare_index(args)++ elif args.command == "compare":++ if args.repeats < 1 or not math.isfinite(args.timeout_s) or args.timeout_s <= 0:++ parser.error("--repeats and --timeout-s must be positive")++ compare(args)++ else:++ worker(args)++++++ if __name__ == "__main__":++ main()diff --cc scripts/benchmark_real_association.pyindex e0d3f37,e0d3f37,e0d3f37..866423c--- a/scripts/benchmark_real_association.py+++ b/scripts/benchmark_real_association.py@@@@ -5,6 -5,6 -5,6 +5,7 @@@@ flight observations or proposal measurejoin, not whole-search runtime or localization accuracy."""import argparse+++import gcimport jsonfrom pathlib import Pathimport statistics@@@@ -39,7 -39,7 -39,7 +40,13 @@@@ def main()sys.modules[baseline.__name__] = baselineexec(compile(source, "<baseline matching.py>", "exec"), baseline.__dict__)fixture = json.loads(args.fixture.read_text())--- events, _ = prepare_events(fixture["events"])+++ # Archived CPA fixtures contain either original discovery records or their+++ # already prepared association settings. Preserve both observed formats.+++ recorded_events = fixture["events"]+++ if recorded_events and "ridge" in recorded_events[0]:+++ events, _ = prepare_events(recorded_events)+++ else:+++ events = recorded_eventstable = pd.DataFrame(fixture["cpa_rows"])reference, counts = baseline.association_candidates(table, events)if not reference:@@@@ -92,6 -92,6 -92,6 +99,11 @@@@for repeat in range(args.repetitions):order = ("baseline", "optimized") if repeat % 2 == 0 else ("optimized", "baseline")for name in order:+++ # Release the previous result and begin each measured call at the+++ # same garbage-collector state. Otherwise a prior run can choose+++ # which arm pays for collecting thousands of candidate objects.+++ actual = None+++ gc.collect()wall, cpu = time.perf_counter(), time.process_time()actual = functions[name](table, events)run = {"implementation": name, "cpu_s": time.process_time()-cpu,@@@@ -103,6 -103,6 -103,6 +115,7 @@@@medians = {name: statistics.median(r["cpu_s"] for r in runs if r["implementation"] == name)for name in functions}report = {"fixture": str(args.fixture), "baseline_ref": args.baseline_ref,+++ "garbage_collection_reset_before_each_run": True,"events": len(events), "cpa_rows": len(table), "candidates": len(reference),"exact_candidate_order_and_work_accounting": True, "checks": checks, "runs": runs,"median_cpu_s": medians, "cpu_speedup": medians["baseline"]/medians["optimized"]}diff --cc scripts/compare_real_upload_performance.pyindex 38075ae,38075ae,38075ae..0dfe5ac--- a/scripts/compare_real_upload_performance.py+++ b/scripts/compare_real_upload_performance.py@@@@ -146,7 -146,7 -146,7 +146,8 @@@@ class Runreason = "search implementation files"elif not root and relative == "search/manifest.json" and here in {("configuration", "runtime", "ridge_fit_backend"),--- ("configuration", "runtime", "acoustic_backend")}:+++ ("configuration", "runtime", "acoustic_backend"),+++ ("configuration", "runtime", "cpa_backend")}:reason = "search native implementation metadata:" + keyelif not root and relative == "provenance.json" and here == ("resumed_from",):reason = "explicit resume metadata: resumed_from"@@@@ -330,7 -330,7 -330,7 +331,8 @@@@ def compare_runs(baseline, current, maxif passed and left.continuation is not None else None),"normalization": {"implementation_fields": sorted(IMPLEMENTATION_FIELDS),"native_backend_metadata": ["search/manifest.json/configuration/runtime/ridge_fit_backend",--- "search/manifest.json/configuration/runtime/acoustic_backend"],+++ "search/manifest.json/configuration/runtime/acoustic_backend",+++ "search/manifest.json/configuration/runtime/cpa_backend"],"performance_fields": sorted(PERFORMANCE_FIELDS),"path_rule": "Only each run directory, job directory and exact job identifier are replaced.","sqlite_rule": "Compare all logical tables, columns and rows; ignore only run.identity and physical SQLite/index layout.",diff --cc src/dopplergeist/matching.pyindex a36d049,869c9a7,2e38359..246c02e--- a/src/dopplergeist/matching.py+++ b/src/dopplergeist/matching.py@@@@ -332,7 -344,7 -332,7 +344,10 @@@@ def association_candidates(table, eventselected = max(0, right - left)if selected:reserve(selected * selected.bit_length())--- return sorted(indices[left:right])+++ return sorted(indices[left:right])+++ # Most skipped events have no overlapping row. Avoid allocating and+++ # sorting an empty slice while preserving the same branch accounting.+++ return ()def row_options(indices):return chain((records[i] for i in indices), [None])@@@@ -707,8 -713,8 -692,22 +731,22 @@@@ def refine_candidate(candidate, tracksReturned radius describes the explored alternatives only. A boundary optimumwarns callers that this local search has not enclosed the possible solution."""-- forward = Transformer.from_crs("EPSG:4326", crs, always_xy=True)-- inverse = Transformer.from_crs(crs, "EPSG:4326", always_xy=True)++ return _refine_candidate(candidate, tracks, events, measurements, terrain,++ radius_m=radius_m, clock_radius_s=clock_radius_s,++ altitude_offsets_m=altitude_offsets_m, crs=crs, c=c,- recording_start_window_s=recording_start_window_s)+++ recording_start_window_s=recording_start_window_s, bounds_wsen=bounds_wsen)++++++ def _refine_candidate(candidate, tracks, events, measurements, terrain, radius_m=500.0,++ clock_radius_s=5.0, altitude_offsets_m=(-150.0, 0.0, 150.0),- crs="EPSG:5070", c=343.0, recording_start_window_s=None, *,- _transformers=None):+++ crs="EPSG:5070", c=343.0, recording_start_window_s=None,+++ bounds_wsen=None, *, _transformers=None):++ """Share coordinate operations only within one declared sensitivity profile."""++ if _transformers is None:++ forward = Transformer.from_crs("EPSG:4326", crs, always_xy=True)++ inverse = Transformer.from_crs(crs, "EPSG:4326", always_xy=True)++ else:++ forward, inverse = _transformersx0, y0 = forward.transform(candidate["lon"], candidate["lat"])start = candidate["recording_start_time_s"]selected = [tracks[f] for f in candidate["flight_ids"]]@@@@ -802,8 -796,8 -792,22 +843,22 @@@@ def profile_candidate(candidate, tracksraise ValueError("Sensitivity ranges must be nonnegative and sound speed must remain positive")kwargs = dict(radius_m=radius_m, clock_radius_s=clock_radius_s,altitude_offsets_m=(0.0,), crs=crs, c=c,-- recording_start_window_s=recording_start_window_s)- nominal = refine_candidate(candidate, tracks, events, measurements, terrain, **kwargs)++ recording_start_window_s=recording_start_window_s, bounds_wsen=bounds_wsen)- nominal = refine_candidate(candidate, tracks, events, measurements, terrain, **kwargs)++ transformers = None++++ def refine(*args, **options):++ nonlocal transformers++ # Custom refiners keep the original dispatch and receive no private++ # arguments. The standard path owns exactly one pair for this profile.++ current_refiner = refine_candidate++ if current_refiner is not _CANONICAL_REFINE_CANDIDATE:++ return current_refiner(*args, **options)++ if transformers is None:++ transformers = (Transformer.from_crs("EPSG:4326", crs, always_xy=True),++ Transformer.from_crs(crs, "EPSG:4326", always_xy=True))++ return _refine_candidate(*args, _transformers=transformers, **options)++++ nominal = refine(candidate, tracks, events, measurements, terrain, **kwargs)scenarios = [("nominal", "nominal", events, measurements, {})]for sign in (-1, 1):scenarios.append(("altitude", f"height_{sign:+d}", events, measurements,diff --cc src/dopplergeist/terrain.pyindex 608a803,6703db0,934a435..dac01d3--- a/src/dopplergeist/terrain.py+++ b/src/dopplergeist/terrain.py@@@@ -1,6 -1,7 -1,10 +1,10 @@@@"""Terrain sampling and common projected grids."""+ from collections import OrderedDictfrom functools import lru_cachefrom pathlib import Path++ import stat++ from threading import RLock++ from weakref import refimport numpy as npfrom pyproj import Transformer@@@@ -16,11 -17,93 -54,58 +54,58 @@@@ class Terraindef __init__(self, paths=None, constant_m=None, geoid_path=None):import rasterioself.rasters = [rasterio.open(p) for p in (paths or [])]++ self._pixel_cache = OrderedDict()++ self._pixel_cache_lock = RLock()++ self._pixel_sources = {}++ for raster in self.rasters:++ source = _raster_pixel_source(raster)++ if source is not None:++ self._pixel_sources[id(raster)] = sourceself.constant = constant_mself.geoid = Terrain([geoid_path]) if geoid_path else None- # Each sampler, including the optional geoid, retains at most 4 MiB- # of exact pixels and masks for scalar optimizer evaluations.- self._scalar_blocks = OrderedDict()- self._scalar_block_bytes = 0- self._scalar_cache_limit = 4 * 1024 * 1024- self._scalar_sources = {}if not self.rasters and constant_m is None:raise ValueError("Supply DEM raster paths or an explicit constant ground elevation")- @staticmethod- def _source_signature(paths):- result = []- for path in paths:- stat = path.stat()- result.append((stat.st_dev, stat.st_ino, stat.st_size,- stat.st_mtime_ns, stat.st_ctime_ns))- return tuple(result)-- def _scalar_pixel(self, raster, row, col):- """Return an unchanged pixel and mask, with bounded local-file reuse."""- from rasterio.io import DatasetReader- from rasterio.windows import Window-- # Custom readers, writable/closed datasets and virtual sources retain- # their normal read behavior. A closed reader must not return old pixels.- cacheable = (self._scalar_cache_limit > 0 and- type(raster) is DatasetReader and not raster.closed and- raster.mode == "r" and raster.driver == "GTiff")- if cacheable and raster not in self._scalar_sources:- paths = tuple(Path(path) for path in raster.files)- try:- signature = self._source_signature(paths) if paths else None- except OSError:- signature = None- self._scalar_sources[raster] = ((paths, signature)- if signature is not None else None)- source = self._scalar_sources.get(raster) if cacheable else None- if source is None:- data = raster.read(1, window=Window(col, row, 1, 1), masked=True)- return np.asarray(data)[0, 0], np.ma.getmaskarray(data)[0, 0]-- paths, signature = source- try:- unchanged = self._source_signature(paths) == signature- except OSError:- unchanged = False- if not unchanged:- raise ValueError(f"Terrain raster changed after caching: {raster.name}")-- # Keep windows inside the same underlying storage block as the requested- # pixel, including striped TIFFs and blocks not divisible by 64.- size = 64- block_h, block_w = raster.block_shapes[0]- block_r, block_c = row // block_h * block_h, col // block_w * block_w- r0 = block_r + (row-block_r) // size * size- c0 = block_c + (col-block_c) // size * size- key = (raster, r0, c0)- entry = self._scalar_blocks.get(key)- if entry is None:- height = min(size, block_r+block_h-r0, raster.height-r0)- width = min(size, block_c+block_w-c0, raster.width-c0)- data = raster.read(1, window=Window(c0, r0, width, height), masked=True)- pixels, mask = np.asarray(data), np.ma.getmaskarray(data)- try:- unchanged = self._source_signature(paths) == signature- except OSError:- unchanged = False- if not unchanged:- raise ValueError(f"Terrain raster changed while reading: {raster.name}")- byte_count = pixels.nbytes + mask.nbytes- if byte_count <= self._scalar_cache_limit:- while self._scalar_blocks and (- self._scalar_block_bytes + byte_count > self._scalar_cache_limit or- len(self._scalar_blocks) >= 256):- _, old = self._scalar_blocks.popitem(last=False)- self._scalar_block_bytes -= old[0].nbytes + old[1].nbytes- pixels.flags.writeable = False- mask.flags.writeable = False- self._scalar_blocks[key] = (pixels, mask)- self._scalar_block_bytes += byte_count- else:- self._scalar_blocks.move_to_end(key)- pixels, mask = entry- return pixels[row-r0, col-c0], mask[row-r0, col-c0]++ def _read_window(self, raster, window):++ # Only reuse the exact 1x1 reads already requested by the original path.++ # Array coordinates, floor arithmetic, masks and raster order stay there.++ sources = getattr(self, "_pixel_sources", None)++ source = sources.get(id(raster)) if sources is not None else None++ if source is None or window.width != 1 or window.height != 1:++ return raster.read(1, window=window, masked=True)++ with self._pixel_cache_lock:++ signature = _raster_file_signature(source[2])++ if (raster is not source[0]() or raster.closed or raster.mode != "r"++ or type(raster).read is not source[1]++ or "read" in getattr(raster, "__dict__", {}) or signature != source[3]):++ # Do not re-enable a changed reader: an open handle can still++ # refer to the old inode after its path is replaced.++ self._pixel_sources.pop(id(raster), None)++ for key in list(self._pixel_cache):++ if key[0] == id(raster):++ del self._pixel_cache[key]++ return raster.read(1, window=window, masked=True)++ key = (id(raster), window.row_off, window.col_off)++ if key in self._pixel_cache:++ self._pixel_cache.move_to_end(key)++ return self._pixel_cache[key]++ data = raster.read(1, window=window, masked=True)++ if _raster_file_signature(source[2]) != signature:++ self._pixel_sources.pop(id(raster), None)++ for old in list(self._pixel_cache):++ if old[0] == id(raster):++ del self._pixel_cache[old]++ return raster.read(1, window=window, masked=True)++ # The cache never escapes sample(). Returned elevations stay fresh++ # and writable. Both numeric values and masks belong to this entry.++ data.flags.writeable = False++ if isinstance(data.mask, np.ndarray):++ data.mask.flags.writeable = False++ self._pixel_cache[key] = data++ if len(self._pixel_cache) > _PIXEL_CACHE_MAX_ENTRIES:++ self._pixel_cache.popitem(last=False)++ return data+def sample(self, lat, lon):from rasterio.windows import Window@@@@ -31,6 -114,13 -116,6 +116,15 @@@@x, y = transform.transform(lon.ravel(), lat.ravel())col,row=(~raster.transform) @ (np.asarray(x),np.asarray(y))col,row=np.floor(col).astype(int),np.floor(row).astype(int)+ + if lat.size == 1:+ + c, r = int(col[0]), int(row[0])+ + if 0 <= c < raster.width and 0 <= r < raster.height:- value, masked = self._scalar_pixel(raster, r, c)- if not masked:- elevation.ravel()[0] = value+++ # Preserve the exact read window and all reader/source+++ # guards while avoiding vector mask assembly for one pixel.+++ data = self._read_window(raster, Window(c, r, 1, 1))+++ if not np.ma.getmaskarray(data)[0, 0]:+++ elevation.ravel()[0] = np.asarray(data)[0, 0]+ + continueinside=(col>=0)&(col<raster.width)&(row>=0)&(row<raster.height)values=np.ma.masked_all(len(col))if np.any(inside):diff --cc tests/test_real_refinement_reuse.pyindex 0000000,0000000,3c6b041..f075dc1mode 000000,000000,100644..100644--- a/tests/test_real_refinement_reuse.py+++ b/tests/test_real_refinement_reuse.py@@@@ -1,0 -1,0 -1,242 +1,248 @@@@++ """Exact refinement and terrain.Terrain pixel reuse on archived real inputs."""++ from contextlib import contextmanager++ from copy import deepcopy++ import json++ import os++ from pathlib import Path++ import shutil++++ import numpy as np++ import pytest++++ from dopplergeist import matching, terrain++ from dopplergeist.data_access import FlightIndex++++++ ROOT = Path(__file__).resolve().parents[1]++++++ @pytest.fixture(scope="module")++ def recorded_profile():++ return json.loads((ROOT / "tests/data/real-refinement-profile.json").read_text())++++++ @pytest.fixture(scope="module")++ def real_rasters(recorded_profile):++ root = Path(os.environ.get("DOPPLERGEIST_REAL_REFERENCE_ROOT", ROOT))++ paths = tuple(root / name for name in recorded_profile["source"]["terrain_data"])++ if not all(path.is_file() for path in paths):++ pytest.skip("Restore the real data-terrain bundle or set DOPPLERGEIST_REAL_REFERENCE_ROOT")++ return paths++++++ def close(sampler):++ for raster in sampler.rasters:++ raster.close()++ if sampler.geoid is not None:++ close(sampler.geoid)++++++ def uncached(sampler):++ sampler._pixel_sources.clear()++ sampler._pixel_cache.clear()++ if sampler.geoid is not None:++ uncached(sampler.geoid)++++++ @contextmanager++ def samplers(paths, **options):++ fast, original = terrain.Terrain(paths, **options), terrain.Terrain(paths, **options)++ uncached(original)++ try:++ yield fast, original++ finally:++ close(fast)++ close(original)++++++ def exact_height(actual, expected):++ assert actual.shape == expected.shape and actual.dtype == expected.dtype++ assert actual.tobytes() == expected.tobytes()++ assert actual.flags.writeable and expected.flags.writeable++++++ @pytest.mark.parametrize("configuration", ["dem", "ellipsoidal", "geoid_last", "dem_last"])++ def test_recorded_positions_keep_exact_heights(real_rasters, recorded_profile, configuration):++ dem, geoid = real_rasters++ paths = {"dem": [dem], "ellipsoidal": [dem],++ "geoid_last": [dem, geoid], "dem_last": [geoid, dem]}[configuration]++ options = {"geoid_path": geoid} if configuration == "ellipsoidal" else {}++ lat, lon = np.asarray(recorded_profile["positions"]).T++ with samplers(paths, **options) as (fast, original):++ assert fast._pixel_sources, "The actual GeoTIFF must exercise pixel reuse"++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))++ for a, b in zip(lat, lon):++ exact_height(fast.sample(a, b), original.sample(a, b))++ exact_height(fast.sample(a, b), original.sample(a, b))++ assert fast._pixel_cache++ changed = fast.sample(lat[0], lon[0])++ changed[...] += 1.++ exact_height(fast.sample(lat[0], lon[0]), original.sample(lat[0], lon[0]))++++++ def test_pixel_cache_bound_uses_original_heights(real_rasters, recorded_profile, monkeypatch):++ monkeypatch.setattr(terrain, "_PIXEL_CACHE_MAX_ENTRIES", 2)++ with samplers([real_rasters[0]]) as (fast, original):++ for lat, lon in recorded_profile["positions"]:++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))++ assert len(fast._pixel_cache) <= 2++ assert len(fast._pixel_cache) == 2++ lat, lon = recorded_profile["positions"][0]++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))++++++ def test_closed_real_reader_keeps_original_error(real_rasters, recorded_profile):++ lat, lon = recorded_profile["positions"][0]++ with samplers([real_rasters[0]]) as (fast, original):++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))++ close(fast)++ close(original)++ with pytest.raises(Exception) as expected:++ original.sample(lat, lon)++ with pytest.raises(type(expected.value)) as actual:++ fast.sample(lat, lon)++ assert str(actual.value) == str(expected.value)++++++ @pytest.mark.parametrize("change", ["mtime", "replacement", "sidecar"])++ def test_changed_real_raster_discards_cached_pixels(real_rasters, recorded_profile, tmp_path, change):++ copied = tmp_path / "observed.tif"++ shutil.copyfile(real_rasters[0], copied)++ lat, lon = recorded_profile["positions"][0]++ with samplers([copied]) as (fast, original):++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))++ assert fast._pixel_cache++ if change == "mtime":++ info = copied.stat()++ os.utime(copied, ns=(info.st_atime_ns, info.st_mtime_ns + 1_000_000))++ elif change == "replacement":++ replacement = tmp_path / "replacement.tif"++ shutil.copyfile(real_rasters[0], replacement)++ replacement.replace(copied)++ else:++ copied.with_name(copied.name + ".aux.xml").write_text("<PAMDataset/>\n")++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))++ assert not fast._pixel_cache and not fast._pixel_sources++++++ def test_custom_reader_keeps_each_real_read(real_rasters, recorded_profile):++ class ReaderProxy:++ def __init__(self, reader):++ self.reader, self.calls = reader, 0++++ def __getattr__(self, name):++ return getattr(self.reader, name)++++ def read(self, *args, **kwargs):++ self.calls += 1++ return self.reader.read(*args, **kwargs)++++ lat, lon = recorded_profile["positions"][0]++ with samplers([real_rasters[0]]) as (fast, original):++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))++ proxy = ReaderProxy(fast.rasters[0])++ fast.rasters = [proxy]++ for _ in range(2):++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))++ assert proxy.calls == 2++++++ def test_replaced_dataset_keeps_new_real_values(real_rasters, recorded_profile):++ import rasterio++++ lat, lon = recorded_profile["positions"][0]++ with samplers([real_rasters[0]]) as (fast, original):++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))++ close(fast)++ close(original)++ fast.rasters = [rasterio.open(real_rasters[1])]++ original.rasters = [rasterio.open(real_rasters[1])]++ for a, b in recorded_profile["positions"][:8]:++ exact_height(fast.sample(a, b), original.sample(a, b))++++++ def test_real_pixel_nodata_metadata_and_constant(real_rasters, recorded_profile, tmp_path):++ import rasterio++++ # Change only nodata metadata on a disposable copy of the real raster.++ # Every observed pixel value stays unchanged.++ copied = tmp_path / "nodata-metadata.tif"++ shutil.copyfile(real_rasters[0], copied)++ lat, lon = recorded_profile["positions"][0]++ reference = terrain.Terrain([real_rasters[0]])++ try:++ observed_height = reference.sample(lat, lon).item()++ finally:++ close(reference)++ with rasterio.open(copied, "r+", IGNORE_COG_LAYOUT_BREAK="YES") as writer:++ writer.nodata = observed_height++ with samplers([copied]) as (fast, original):++ for _ in range(2):++ with pytest.raises(ValueError) as expected:++ original.sample(lat, lon)++ with pytest.raises(ValueError) as actual:++ fast.sample(lat, lon)++ assert str(actual.value) == str(expected.value)++ assert fast._pixel_cache++ assert all(np.ma.getmaskarray(value).all() for value in fast._pixel_cache.values())++ with samplers([copied], constant_m=observed_height) as (fast, original):++ for _ in range(2):++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))++ assert fast.sample(lat, lon).item() == observed_height++ with samplers([real_rasters[1], copied]) as (fast, original):++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))++++++ def encoded(value):++ def scalar(item):++ return item.tolist() if isinstance(item, np.ndarray) else item.item()++ return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=True, default=scalar)++++[email protected]("geographic_bounds", [False, True])++ def test_real_complete_profile_keeps_all_fields_and_custom_dispatch(- real_rasters, recorded_profile, monkeypatch):+++ real_rasters, recorded_profile, monkeypatch, geographic_bounds):++ source = os.environ.get("DOPPLERGEIST_REAL_CPA_FIXTURE")++ if not source:++ pytest.skip("Set DOPPLERGEIST_REAL_CPA_FIXTURE to the restored real-index fixture")++ diagnostic = json.loads(Path(source).read_text())++ candidate = recorded_profile["candidate"]++ index = FlightIndex(diagnostic["real_index"])++ assert index.report["input_hash"] == diagnostic["index_input_hash"]++ tracks = {track.id: track for track in index.lookup(candidate["flight_ids"])}++ assert set(tracks) == set(candidate["flight_ids"])++ events, measurements = matching.prepare_events(recorded_profile["events"], c=343., max_distance_m=20000.)++ options = {"radius_m": 500., "recording_start_window_s": recorded_profile["recording_start_window_s"], "c": 343.}+++ if geographic_bounds:+++ request = json.loads((ROOT / "benchmarks/reference-regional-upload-request.json").read_text())+++ options["bounds_wsen"] = request["search"]["bounds_wsen"]++ refiner = matching.refine_candidate++ calls = []++++ def later(*args, **kwargs):++ assert len(args) == 5- assert set(kwargs) == {"radius_m", "clock_radius_s", "altitude_offsets_m", "crs", "c", "recording_start_window_s"}+++ assert set(kwargs) == {"radius_m", "clock_radius_s", "altitude_offsets_m", "crs", "c",+++ "recording_start_window_s", "bounds_wsen"}+++ assert kwargs["bounds_wsen"] == options.get("bounds_wsen")++ calls.append("later")++ return refiner(*args, **kwargs)++++ def first(*args, **kwargs):++ calls.append("first")++ assert len(args) == 5++ assert not any(key.startswith("_") for key in kwargs)++ monkeypatch.setattr(matching, "refine_candidate", later)++ return refiner(*args, **kwargs)++++ with samplers([real_rasters[0]], geoid_path=real_rasters[1]) as (fast, original):++ actual = matching._cache_ridge_fits(matching.profile_candidate)(++ deepcopy(candidate), tracks, events, measurements, fast, **options)++ assert len(actual["sensitivity"]["profiles"]) == recorded_profile["expected_profile_count"]++ assert fast._pixel_cache and fast.geoid._pixel_cache++ # This public wrapper forces the existing unshared-transform path and++ # changes global dispatch after the nominal call, using real solves.++ monkeypatch.setattr(matching, "refine_candidate", first)++ expected = matching._cache_ridge_fits(matching.profile_candidate)(++ deepcopy(candidate), tracks, events, measurements, original, **options)++ assert encoded(actual) == encoded(expected)++ assert calls == ["first"] + ["later"] * 43diff --cc tests/test_real_scalar_terrain.pyindex 0000000,5dcb65c,0000000..dc2548dmode 000000,100644,000000..100644--- a/tests/test_real_scalar_terrain.py+++ b/tests/test_real_scalar_terrain.py@@@@ -1,0 -1,139 -1,0 +1,143 @@@@+ +"""Exact scalar terrain reuse on restored rasters and recorded candidate points."""+ +import json+ +import os+ +from pathlib import Path+ +import shutil+ ++ +import numpy as np+ +import pytest+ +import rasterio+ ++++from dopplergeist import terrain+ +from dopplergeist.terrain import Terrain+ ++ ++ [email protected](scope="module")+ +def observed_terrain():+ + root = Path(os.environ.get("DOPPLERGEIST_REAL_REFERENCE_ROOT",+ + Path(__file__).resolve().parents[1]))+ + dem = root / "data/terrain/Copernicus_DSM_COG_10_N40_00_W074_00_DEM.tif"+ + geoid = root / "data/terrain/us_nga_egm96_15.tif"+ + if not dem.exists() or not geoid.exists():+ + pytest.skip("Restore the real reference DEM and geoid rasters")+ + trace = os.environ.get("DOPPLERGEIST_REAL_TERRAIN_TRACE")+ + if trace:+ + rows = json.loads(Path(trace).read_text())["rows"]+ + points = np.asarray(rows, dtype=float)[:, :2]+ + # Retain actual requests throughout the optimizer trace, without+ + # constructing new coordinates or a simulated search workload.+ + points = points[np.linspace(0, len(points)-1, min(256, len(points)), dtype=int)]+ + else:+ + choices = [root / "results/reference-local-fast/ranked.json",+ + root / "evidence/results/reference-local-fast/ranked.json"]+ + source = next((path for path in choices if path.exists()), None)+ + if source is None:+ + pytest.skip("Restore the archived real reference candidate results")+ + rows = json.loads(source.read_text())+ + points = np.asarray([(row["lat"], row["lon"]) for row in rows])+ + assert len(points) > 1+ + return dem, geoid, points+ ++ ++ +def _close(sampler):+ + for raster in sampler.rasters:+ + raster.close()+ + if sampler.geoid is not None:+ + _close(sampler.geoid)+ ++ ++ [email protected]("with_geoid", [False, True])+ +def test_scalar_cache_preserves_real_vector_samples_and_result_ownership(observed_terrain, with_geoid):+ + dem, geoid, points = observed_terrain+ + sampler = Terrain([dem], geoid_path=geoid if with_geoid else None)+ + try:+ + expected = sampler.sample(points[:, 0], points[:, 1])+ + for _ in range(2):+ + actual = np.asarray([sampler.sample(lat, lon).item() for lat, lon in points])+ + np.testing.assert_array_equal(actual, expected)- assert sampler._scalar_blocks+++ assert sampler._pixel_cache+ + lat, lon = points[0]+ + result = sampler.sample([[lat]], [[lon]])+ + assert result.shape == (1, 1)+ + assert result.item() == expected[0]+ + result[...] = np.nan+ + assert sampler.sample(lat, lon).item() == expected[0]- for pixels, mask in sampler._scalar_blocks.values():- assert not pixels.flags.writeable and not mask.flags.writeable+++ for pixels in sampler._pixel_cache.values():+++ assert not pixels.flags.writeable+++ if isinstance(pixels.mask, np.ndarray):+++ assert not pixels.mask.flags.writeable+ + finally:+ + _close(sampler)+ ++ +- def test_scalar_cache_stays_bounded_on_actual_points(observed_terrain):+++def test_scalar_cache_stays_bounded_on_actual_points(observed_terrain, monkeypatch):+ + dem, _, points = observed_terrain+ + sampler = Terrain([dem])- sampler._scalar_cache_limit = 21_000+++ monkeypatch.setattr(terrain, "_PIXEL_CACHE_MAX_ENTRIES", 2)+ + try:+ + expected = sampler.sample(points[:, 0], points[:, 1])+ + for point, height in zip(points, expected):+ + assert sampler.sample(*point).item() == height- assert sampler._scalar_block_bytes <= sampler._scalar_cache_limit- assert sampler._scalar_block_bytes == sum(- pixels.nbytes + mask.nbytes for pixels, mask in sampler._scalar_blocks.values())+++ assert len(sampler._pixel_cache) <= 2+++ assert all(pixels.shape == (1, 1) for pixels in sampler._pixel_cache.values())+ + # The same observed points also take the explicit uncached read path.- sampler._scalar_cache_limit = 0+++ sampler._pixel_sources.clear()+++ sampler._pixel_cache.clear()+ + np.testing.assert_array_equal(+ + [sampler.sample(*point).item() for point in points], expected)+ + finally:+ + _close(sampler)+ ++ ++ +def test_changed_real_raster_is_not_served_from_cache(observed_terrain, tmp_path):+ + dem, _, points = observed_terrain+ + copy = tmp_path / dem.name+ + shutil.copy2(dem, copy)+ + sampler = Terrain([copy])+ + try:- sampler.sample(*points[0])+++ expected = sampler.sample(*points[0])+++ assert sampler._pixel_cache+ + stat = copy.stat()+ + # Change file metadata, keeping the real measured terrain untouched.+ + os.utime(copy, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000))- with pytest.raises(ValueError, match="Terrain raster changed after caching"):- sampler.sample(*points[0])+++ np.testing.assert_array_equal(sampler.sample(*points[0]), expected)+++ assert not sampler._pixel_cache and not sampler._pixel_sources+ + finally:+ + _close(sampler)+ ++ ++ +def test_closed_real_reader_does_not_serve_cached_pixels(observed_terrain):+ + dem, _, points = observed_terrain+ + sampler = Terrain([dem])+ + sampler.sample(*points[0])+ + sampler.rasters[0].close()+ + with pytest.raises(rasterio.errors.RasterioIOError):+ + sampler.sample(*points[0])+ ++ ++ +def test_custom_reader_keeps_its_read_behavior(observed_terrain):+ + dem, _, points = observed_terrain+ ++ + class ObservedReader:+ + def __init__(self, reader):+ + self.reader, self.windows = reader, []+ ++ + def __getattr__(self, name):+ + return getattr(self.reader, name)+ ++ + def read(self, *args, **kwargs):+ + self.windows.append(kwargs["window"])+ + return self.reader.read(*args, **kwargs)+ ++ + sampler = Terrain([dem])+ + reader = ObservedReader(sampler.rasters[0])+ + sampler.rasters = [reader]+ + try:+ + first = sampler.sample(*points[0])+ + second = sampler.sample(*points[0])+ + np.testing.assert_array_equal(first, second)+ + assert len(reader.windows) == 2+ + assert all(window.width == window.height == 1 for window in reader.windows)- assert not sampler._scalar_blocks+++ assert not sampler._pixel_cache+ + finally:+ + _close(sampler)