Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Reuse complete source-fit batches and integrate staged scoring

Cinna Davis01775bb127ea
diff --cc benchmarks/results/pipeline-components-20261103.json
index 298ab89,0000000..35f3e06
mode 100644,000000..100644
--- a/benchmarks/results/pipeline-components-20261103.json
+++ b/benchmarks/results/pipeline-components-20261103.json
@@@ -1,951 -1,0 +1,987 @@@
+{
+ "date_utc": "2026-11-03",
+ "scope": "Runtime and result consistency on recorded audio and historical trajectories; no independent location accuracy claim.",
+ "sources": {
+ "original_baseline": "7a8dd27979bd9a984c8675c38a00dfbf98a5b01e",
+ "core_changes": "a2236ac",
+ "core_with_main": "fa19b59492a15ca3f3dd526bed5706b770793a58",
+ "combined_http": "00727d4",
+ "combined_scoring_dependency": "3725e9d2027d979e579ba9c347fb688ebf0895c3"
+ },
+ "runtime": {
+ "numerical_packages": {
+ "numpy": "2.3.5",
+ "scipy": "1.17.0",
+ "pandas": "2.2.3",
+ "pyarrow": "26.0.1",
+ "pyproj": "3.8.0",
+ "rasterio": "1.5.2",
+ "soundfile": "0.14.0"
+ },
+ "numerical_threads": 1,
+ "audio_helpers_search_python": "3.12.14",
+ "cache_empty_tiles_configured_http_python": "3.12.3",
+ "cache_empty_tiles_configured_http_sqlite": "3.45.1"
+ },
+ "inputs": {
+ "wav_sha256": "262d9c4eba1c80657813c604140ba2afbbaabed2b3aebc833cba7cafdc005e9f",
+ "wav_duration_s": 3600.1845,
+ "wav_ridges": 18,
+ "wav_independent_groups": 16,
+ "mp3_sha256": "707083f0344823eeea7f130faec6c22b6628b8b9fea04d026915d93df9a07b15",
+ "mp3_ridges": 17,
+ "mp3_independent_groups": 15,
+ "clock_note": "Historical fitted clock, not independent timing evidence",
+ "coverage_note": "Regional observed archive excludes known jets; complete coverage is not established"
+ },
+ "audio": {
+ "comparison": "Original/new implementation, three alternating pairs; intake and plots excluded",
+ "discovery": {
+ "baseline": {
+ "trials": [
+ {
+ "wall_s": 9.284779882999942,
+ "cpu_s": 9.239130253999999
+ },
+ {
+ "wall_s": 7.356600091000018,
+ "cpu_s": 7.355370614000002
+ },
+ {
+ "wall_s": 6.731685112999912,
+ "cpu_s": 6.730280811
+ }
+ ],
+ "median_wall_s": 7.356600091000018,
+ "median_cpu_s": 7.355370614000002
+ },
+ "current": {
+ "trials": [
+ {
+ "wall_s": 6.414081021000015,
+ "cpu_s": 6.4099143330000015
+ },
+ {
+ "wall_s": 5.961681690000091,
+ "cpu_s": 5.953134779000003
+ },
+ {
+ "wall_s": 6.04402048999998,
+ "cpu_s": 6.043257627000003
+ }
+ ],
+ "median_wall_s": 6.04402048999998,
+ "median_cpu_s": 6.043257627000003
+ },
+ "wall_speedup": 1.217169945596932,
+ "cpu_speedup": 1.2171201474412996,
+ "exact_event_payload_match": true,
+ "ridge_count": 18,
+ "independent_group_count": 16
+ },
+ "configured_ridges": [
+ {
+ "event": 1,
+ "spectrum_shape": [
+ 921,
+ 713
+ ],
+ "exact_frequency_and_weight_match": true,
+ "baseline": {
+ "trials": [
+ {
+ "wall_s": 0.18297023399998125,
+ "cpu_s": 0.1829722699999934
+ },
+ {
+ "wall_s": 0.17685444400001415,
+ "cpu_s": 0.17680868999999433
+ },
+ {
+ "wall_s": 0.18520073299998785,
+ "cpu_s": 0.18520574200000084
+ }
+ ],
+ "median_wall_s": 0.18297023399998125,
+ "median_cpu_s": 0.1829722699999934
+ },
+ "current": {
+ "trials": [
+ {
+ "wall_s": 0.058338671999990765,
+ "cpu_s": 0.05833744499999938
+ },
+ {
+ "wall_s": 0.06065663100002894,
+ "cpu_s": 0.0606612240000004
+ },
+ {
+ "wall_s": 0.06332199999997101,
+ "cpu_s": 0.06331132200000411
+ }
+ ],
+ "median_wall_s": 0.06065663100002894,
+ "median_cpu_s": 0.0606612240000004
+ },
+ "wall_speedup": 3.016491865496024,
+ "cpu_speedup": 3.0162970335051624
+ },
+ {
+ "event": 2,
+ "spectrum_shape": [
+ 721,
+ 333
+ ],
+ "exact_frequency_and_weight_match": true,
+ "baseline": {
+ "trials": [
+ {
+ "wall_s": 0.07580405500004872,
+ "cpu_s": 0.07584756099999623
+ },
+ {
+ "wall_s": 0.0735758349999287,
+ "cpu_s": 0.07330923200000683
+ },
+ {
+ "wall_s": 0.0675049259999696,
+ "cpu_s": 0.06750940799999938
+ }
+ ],
+ "median_wall_s": 0.0735758349999287,
+ "median_cpu_s": 0.07330923200000683
+ },
+ "current": {
+ "trials": [
+ {
+ "wall_s": 0.029976841999996395,
+ "cpu_s": 0.02998151999999976
+ },
+ {
+ "wall_s": 0.025200435999977344,
+ "cpu_s": 0.02520547499999992
+ },
+ {
+ "wall_s": 0.025601695999966978,
+ "cpu_s": 0.025605783999999687
+ }
+ ],
+ "median_wall_s": 0.025601695999966978,
+ "median_cpu_s": 0.025605783999999687
+ },
+ "wall_speedup": 2.873865661088375,
+ "cpu_speedup": 2.862995017063634
+ }
+ ]
+ },
+ "helpers": {
+ "scope": "Real observations and archived candidate coordinates; component timings only",
+ "cpu_affinity": [
+ 6
+ ],
+ "tracks": 214,
+ "samples": 71571,
+ "independent_decodes_exact": true,
+ "old_new_identity_relation_exact": true,
+ "duplicate_track_geometry_exact": true,
+ "terrain_arrays_exact": true,
+ "candidate_rows": 283,
+ "unique_candidate_coordinates": 274,
+ "identity_digest": {
+ "trials": {
+ "baseline": [
+ {
+ "wall_s": 0.7115346380001029,
+ "cpu_s": 0.7115334740000003
+ },
+ {
+ "wall_s": 0.6816926090000379,
+ "cpu_s": 0.6816836399999993
+ },
+ {
+ "wall_s": 0.7168531109998639,
+ "cpu_s": 0.7168470490000001
+ }
+ ],
+ "current": [
+ {
+ "wall_s": 0.011288004000107321,
+ "cpu_s": 0.011292508000000367
+ },
+ {
+ "wall_s": 0.009811315999968429,
+ "cpu_s": 0.009814100999999908
+ },
+ {
+ "wall_s": 0.010700293999889254,
+ "cpu_s": 0.010702902000000236
+ }
+ ]
+ },
+ "median": {
+ "baseline": {
+ "wall_s": 0.7115346380001029,
+ "cpu_s": 0.7115334740000003
+ },
+ "current": {
+ "wall_s": 0.010700293999889254,
+ "cpu_s": 0.010702902000000236
+ }
+ },
+ "wall_speedup": 66.49673719315255,
+ "cpu_speedup": 66.48042502864968
+ },
+ "duplicate_index_query": {
+ "trials": {
+ "baseline": [
+ {
+ "wall_s": 2.97018349699988,
+ "cpu_s": 2.9699184150000004
+ },
+ {
+ "wall_s": 2.807910829000093,
+ "cpu_s": 2.8075487179999996
+ },
+ {
+ "wall_s": 3.2889418439999645,
+ "cpu_s": 3.288740260999999
+ }
+ ],
+ "current": [
+ {
+ "wall_s": 1.3139952129999983,
+ "cpu_s": 1.3139138050000003
+ },
+ {
+ "wall_s": 1.490499765000095,
+ "cpu_s": 1.490374632
+ },
+ {
+ "wall_s": 1.762300514000117,
+ "cpu_s": 1.7621483060000003
+ }
+ ]
+ },
+ "median": {
+ "baseline": {
+ "wall_s": 2.97018349699988,
+ "cpu_s": 2.9699184150000004
+ },
+ "current": {
+ "wall_s": 1.490499765000095,
+ "cpu_s": 1.490374632
+ }
+ },
+ "wall_speedup": 1.9927433514219242,
+ "cpu_speedup": 1.9927328010236829
+ },
+ "terrain_scalar_queries": {
+ "trials": {
+ "baseline": [
+ {
+ "wall_s": 0.4443265560000782,
+ "cpu_s": 0.44422315499999954
+ },
+ {
+ "wall_s": 0.48182881100001396,
+ "cpu_s": 0.4817972530000034
+ },
+ {
+ "wall_s": 0.49004781799999364,
+ "cpu_s": 0.4900461589999985
+ }
+ ],
+ "current": [
+ {
+ "wall_s": 0.1664448099998026,
+ "cpu_s": 0.1664502170000013
+ },
+ {
+ "wall_s": 0.1315762229999109,
+ "cpu_s": 0.13158169000000086
+ },
+ {
+ "wall_s": 0.1506844049999927,
+ "cpu_s": 0.15069019499999925
+ }
+ ]
+ },
+ "median": {
+ "baseline": {
+ "wall_s": 0.48182881100001396,
+ "cpu_s": 0.4817972530000034
+ },
+ "current": {
+ "wall_s": 0.1506844049999927,
+ "cpu_s": 0.15069019499999925
+ }
+ },
+ "wall_speedup": 3.197602372986357,
+ "cpu_speedup": 3.1972700878116576
+ },
+ "terrain_vector_query": {
+ "trials": {
+ "baseline": [
+ {
+ "wall_s": 0.0019550570000319567,
+ "cpu_s": 0.0019570879999974977
+ },
+ {
+ "wall_s": 0.0018791140000757878,
+ "cpu_s": 0.0018804039999977817
+ },
+ {
+ "wall_s": 0.0031173479999324627,
+ "cpu_s": 0.003119555000001384
+ }
+ ],
+ "current": [
+ {
+ "wall_s": 0.000638515999980882,
+ "cpu_s": 0.0006401309999972682
+ },
+ {
+ "wall_s": 0.00037645799989149964,
+ "cpu_s": 0.0003774209999996003
+ },
+ {
+ "wall_s": 0.0004980270000487508,
+ "cpu_s": 0.0004988109999999324
+ }
+ ]
+ },
+ "median": {
+ "baseline": {
+ "wall_s": 0.0019550570000319567,
+ "cpu_s": 0.0019570879999974977
+ },
+ "current": {
+ "wall_s": 0.0004980270000487508,
+ "cpu_s": 0.0004988109999999324
+ }
+ },
+ "wall_speedup": 3.925604434780806,
+ "cpu_speedup": 3.92350609749537
+ },
+ "terrain_transform_cache": {
+ "hits": 1704,
+ "misses": 2,
+ "maxsize": 32,
+ "currsize": 2
+ }
+ },
+ "empty_tiles": {
+ "scope": "Recorded empty coarse tiles with receiver cells; includes geometry, real index queries, coarse/global stages, checksums and checkpoint writes; excludes progress/resume reads",
+ "tiles": 256,
+ "batch_size": 32,
+ "passes": 3,
+ "runtime": {
+ "python": "3.12.3",
+ "sqlite": "3.45.1"
+ },
+ "timings": {
+ "ordinary": [
+ {
+ "wall_s": 2.1692930350000097,
+ "cpu_s": 1.09747029
+ },
+ {
+ "wall_s": 2.009673713999746,
+ "cpu_s": 1.0613797029999992
+ },
+ {
+ "wall_s": 2.317755713999759,
+ "cpu_s": 1.1247887930000005
+ }
+ ],
+ "batched": [
+ {
+ "wall_s": 0.7647061020002184,
+ "cpu_s": 0.741824603
+ },
+ {
+ "wall_s": 0.7330908949998047,
+ "cpu_s": 0.7171870200000003
+ },
+ {
+ "wall_s": 0.928092106999884,
+ "cpu_s": 0.8675067000000007
+ }
+ ]
+ },
+ "median_wall_s": {
+ "ordinary": 2.1692930350000097,
+ "batched": 0.7647061020002184
+ },
+ "speedup": 2.836767000192435,
+ "all_checkpoints_match_recorded_tiles": true,
+ "all_ledger_tables_match": true,
+ "all_committed_stages_valid": true,
+ "stage_count": 512
+ },
+ "track_cache": {
+ "regional": {
+ "scope": "Repeated real trajectory query; excludes acquisition, audio and scoring",
+ "bounds": [
+ -74.299988,
+ 40.100005,
+ -72.700011,
+ 41.599964
+ ],
+ "time_range_s": [
+ 1788898500.0,
+ 1788902400.0
+ ],
+ "passes": {
+ "uncached": [
+ {
+ "wall_s": 1.5943461480001133,
+ "cpu_s": 1.5940993289999998,
+ "tracks": 441,
+ "samples": 89504,
+ "geometry_and_labels_sha256": "e82c50a6054bcb7f5646ab11d55ff631069ae5cb6127baaf4f586c019ef80ecc",
+ "cached_tracks": 0,
+ "serialized_cache_bytes": 0
+ },
+ {
+ "wall_s": 1.5704857300002004,
+ "cpu_s": 1.5702036489999998,
+ "tracks": 441,
+ "samples": 89504,
+ "geometry_and_labels_sha256": "e82c50a6054bcb7f5646ab11d55ff631069ae5cb6127baaf4f586c019ef80ecc",
+ "cached_tracks": 0,
+ "serialized_cache_bytes": 0
+ },
+ {
+ "wall_s": 1.531997720000163,
+ "cpu_s": 1.531999172,
+ "tracks": 441,
+ "samples": 89504,
+ "geometry_and_labels_sha256": "e82c50a6054bcb7f5646ab11d55ff631069ae5cb6127baaf4f586c019ef80ecc",
+ "cached_tracks": 0,
+ "serialized_cache_bytes": 0
+ },
+ {
+ "wall_s": 1.567756417000055,
+ "cpu_s": 1.5673321649999998,
+ "tracks": 441,
+ "samples": 89504,
+ "geometry_and_labels_sha256": "e82c50a6054bcb7f5646ab11d55ff631069ae5cb6127baaf4f586c019ef80ecc",
+ "cached_tracks": 0,
+ "serialized_cache_bytes": 0
+ }
+ ],
+ "cached": [
+ {
+ "wall_s": 1.5197348029996647,
+ "cpu_s": 1.519719695,
+ "tracks": 441,
+ "samples": 89504,
+ "geometry_and_labels_sha256": "e82c50a6054bcb7f5646ab11d55ff631069ae5cb6127baaf4f586c019ef80ecc",
+ "cached_tracks": 441,
+ "serialized_cache_bytes": 26737306
+ },
+ {
+ "wall_s": 0.104930924999735,
+ "cpu_s": 0.10493462900000061,
+ "tracks": 441,
+ "samples": 89504,
+ "geometry_and_labels_sha256": "e82c50a6054bcb7f5646ab11d55ff631069ae5cb6127baaf4f586c019ef80ecc",
+ "cached_tracks": 441,
+ "serialized_cache_bytes": 26737306
+ },
+ {
+ "wall_s": 0.10096911500022543,
+ "cpu_s": 0.10097258199999981,
+ "tracks": 441,
+ "samples": 89504,
+ "geometry_and_labels_sha256": "e82c50a6054bcb7f5646ab11d55ff631069ae5cb6127baaf4f586c019ef80ecc",
+ "cached_tracks": 441,
+ "serialized_cache_bytes": 26737306
+ },
+ {
+ "wall_s": 0.10028485099974205,
+ "cpu_s": 0.1002887260000005,
+ "tracks": 441,
+ "samples": 89504,
+ "geometry_and_labels_sha256": "e82c50a6054bcb7f5646ab11d55ff631069ae5cb6127baaf4f586c019ef80ecc",
+ "cached_tracks": 441,
+ "serialized_cache_bytes": 26737306
+ }
+ ]
+ },
+ "same_geometry_and_labels": true,
+ "median_warm_query_s": {
+ "uncached": 1.567756417000055,
+ "cached": 0.10096911500022543
+ },
+ "warm_query_speedup": 15.527088823117394,
+ "first_query_s": {
+ "uncached": 1.5943461480001133,
+ "cached": 1.5197348029996647
+ },
+ "all_query_passes_s": {
+ "uncached": 6.264586015000532,
+ "cached": 1.8259196939993672
+ },
+ "all_passes_speedup_including_cold_cache": 3.4309208863830256,
+ "comparison": "Cache off/on in the same final source, one cold decoded-cache query and three warm queries; operating-system cache is uncontrolled."
+ },
+ "full_day": {
+ "scope": "Repeated real trajectory query; excludes acquisition, audio and scoring",
+ "bounds": [
+ -74.05,
+ 40.55,
+ -73.55,
+ 41.05
+ ],
+ "time_range_s": [
+ 1788825600.0,
+ 1788912000.0
+ ],
+ "passes": {
+ "uncached": [
+ {
+ "wall_s": 4.041226284999993,
+ "cpu_s": 4.041035789,
+ "tracks": 1183,
+ "samples": 287491,
+ "geometry_and_labels_sha256": "b1ec19df74ed40615dfc500abde45420a005d75d7224887ff9032c9f7f916f44",
+ "cached_tracks": 0,
+ "serialized_cache_bytes": 0
+ },
+ {
+ "wall_s": 4.72467777699967,
+ "cpu_s": 4.642505895999998,
+ "tracks": 1183,
+ "samples": 287491,
+ "geometry_and_labels_sha256": "b1ec19df74ed40615dfc500abde45420a005d75d7224887ff9032c9f7f916f44",
+ "cached_tracks": 0,
+ "serialized_cache_bytes": 0
+ },
+ {
+ "wall_s": 4.651927691999845,
+ "cpu_s": 4.651892911000001,
+ "tracks": 1183,
+ "samples": 287491,
+ "geometry_and_labels_sha256": "b1ec19df74ed40615dfc500abde45420a005d75d7224887ff9032c9f7f916f44",
+ "cached_tracks": 0,
+ "serialized_cache_bytes": 0
+ },
+ {
+ "wall_s": 4.753316680000353,
+ "cpu_s": 4.751010804,
+ "tracks": 1183,
+ "samples": 287491,
+ "geometry_and_labels_sha256": "b1ec19df74ed40615dfc500abde45420a005d75d7224887ff9032c9f7f916f44",
+ "cached_tracks": 0,
+ "serialized_cache_bytes": 0
+ }
+ ],
+ "cached": [
+ {
+ "wall_s": 4.936198077999961,
+ "cpu_s": 4.9358338879999994,
+ "tracks": 1183,
+ "samples": 287491,
+ "geometry_and_labels_sha256": "b1ec19df74ed40615dfc500abde45420a005d75d7224887ff9032c9f7f916f44",
+ "cached_tracks": 441,
+ "serialized_cache_bytes": 33430432
+ },
+ {
+ "wall_s": 2.6574560640001437,
+ "cpu_s": 2.65744765,
+ "tracks": 1183,
+ "samples": 287491,
+ "geometry_and_labels_sha256": "b1ec19df74ed40615dfc500abde45420a005d75d7224887ff9032c9f7f916f44",
+ "cached_tracks": 441,
+ "serialized_cache_bytes": 33430432
+ },
+ {
+ "wall_s": 2.744923665999977,
+ "cpu_s": 2.7434365110000023,
+ "tracks": 1183,
+ "samples": 287491,
+ "geometry_and_labels_sha256": "b1ec19df74ed40615dfc500abde45420a005d75d7224887ff9032c9f7f916f44",
+ "cached_tracks": 441,
+ "serialized_cache_bytes": 33430432
+ },
+ {
+ "wall_s": 3.078974736999953,
+ "cpu_s": 3.0784054989999987,
+ "tracks": 1183,
+ "samples": 287491,
+ "geometry_and_labels_sha256": "b1ec19df74ed40615dfc500abde45420a005d75d7224887ff9032c9f7f916f44",
+ "cached_tracks": 441,
+ "serialized_cache_bytes": 33430432
+ }
+ ]
+ },
+ "same_geometry_and_labels": true,
+ "median_warm_query_s": {
+ "uncached": 4.72467777699967,
+ "cached": 2.744923665999977
+ },
+ "warm_query_speedup": 1.72124195493009,
+ "first_query_s": {
+ "uncached": 4.041226284999993,
+ "cached": 4.936198077999961
+ },
+ "all_query_passes_s": {
+ "uncached": 18.17114843399986,
+ "cached": 13.417552545000035
+ },
+ "all_passes_speedup_including_cold_cache": 1.3542818910570409,
+ "comparison": "Cache off/on in the same final source, one cold decoded-cache query and three warm queries; operating-system cache is uncontrolled."
+ }
+ },
+ "configured_workers": {
+ "scope": "Full configured reference replay using original audio, observations and source-informed event/grid settings; not automatic or independent localization validation",
+ "upstream_parallelism": "PR #210, merged in adbfb3e; final local performance changes active in both controls",
+ "runtime": {
+ "python": "3.12.3",
+ "sqlite": "3.45.1",
+ "blas_threads": 1,
+ "cpu_affinity": {
+ "workers_1": [
+ 4
+ ],
+ "workers_4": [
+ 4,
+ 5,
+ 7,
+ 8
+ ]
+ }
+ },
+ "measurement_design": "One sequential pair with fresh process/output directories; all scientific settings identical; worker count is the only setting changed",
+ "workers": [
+ 1,
+ 4
+ ],
+ "process_wall_s": [
+ 63.080272204000266,
+ 50.85959439999988
+ ],
+ "report_elapsed_s": [
+ 60.038204577000215,
+ 47.425417169999946
+ ],
+ "process_wall_speedup": 1.240282643779802,
+ "process_wall_reduction_fraction": 0.19373216660319692,
+ "same_scientific_json": true,
+ "same_csv_bytes": true,
+ "excluded_report_metadata": [
+ "elapsed_s",
+ "provenance",
+ "implementation_sha256",
+ "implementation_sources",
+ "config.scoring_workers",
+ "sources entry with roles=[configuration]"
+ ],
+ "ranked_rows": 283,
+ "coarse_points": 2525,
+ "trajectory_segments": 3587,
+ "accepted_location": false,
+ "scientific_json_artifact_count": 16,
+ "csv_artifact_count": 3
+ },
+ "search_architecture_comparison": {
+ "dense": {
+ "case": "previous-fit",
+ "scope": "One separately loaded process per frozen variant; same recorded arrays, query tracks, point, clocks and budgets. No location accuracy claim.",
+ "timings": {
+ "pr212": {
+ "wall_seconds": 16.130660911000177,
+ "cpu_seconds": 16.091403776,
+ "load_seconds_excluded": 1.0451894960001482,
+ "operations": {
+ "scalar_fit_calls": 68,
+ "acoustic_calls": 397,
+ "scored_candidates": 1927
+ },
+ "peak_rss_mib": 230.953125,
+ "peak_rss_scope": "Worker lifetime through scoring, including imports and input loading",
+ "retained": 100,
+ "independent_groups": 16,
+ "scope": "Real measured audio and observed regional ADS-B; development receiver hypotheses; no independent location-accuracy claim. The restored historical region excludes known jets and does not establish complete flight coverage."
+ },
+ "exact": {
+ "wall_seconds": 19.287786888000028,
+ "cpu_seconds": 19.256217652,
+ "load_seconds_excluded": 0.9172454069998821,
+ "operations": {
+ "scalar_fit_calls": 6493,
+ "acoustic_calls": 397,
+ "scored_candidates": 1927
+ },
+ "peak_rss_mib": 255.4375,
+ "peak_rss_scope": "Worker lifetime through scoring, including imports and input loading",
+ "retained": 100,
+ "independent_groups": 16,
+ "scope": "Real measured audio and observed regional ADS-B; development receiver hypotheses; no independent location-accuracy claim. The restored historical region excludes known jets and does not establish complete flight coverage."
+ },
+ "original": {
+ "wall_seconds": 35.92530473200031,
+ "cpu_seconds": 35.877964218,
+ "load_seconds_excluded": 0.9022394250000616,
+ "operations": {
+ "scalar_fit_calls": 32255,
+ "acoustic_calls": 397,
+ "scored_candidates": 1927
+ },
+ "peak_rss_mib": 241.79296875,
+ "peak_rss_scope": "Worker lifetime through scoring, including imports and input loading",
+ "retained": 100,
+ "independent_groups": 16,
+ "scope": "Real measured audio and observed regional ADS-B; development receiver hypotheses; no independent location-accuracy claim. The restored historical region excludes known jets and does not establish complete flight coverage."
+ },
+ "corrected": {
+ "wall_seconds": 36.8815337570004,
+ "cpu_seconds": 36.826490442,
+ "load_seconds_excluded": 0.840361995999956,
+ "operations": {
+ "scalar_fit_calls": 32255,
+ "acoustic_calls": 397,
+ "scored_candidates": 1927
+ },
+ "peak_rss_mib": 241.58984375,
+ "peak_rss_scope": "Worker lifetime through scoring, including imports and input loading",
+ "retained": 100,
+ "independent_groups": 16,
+ "scope": "Real measured audio and observed regional ADS-B; development receiver hypotheses; no independent location-accuracy claim. The restored historical region excludes known jets and does not establish complete flight coverage."
+ }
+ },
+ "exact_variant_equal_original": true,
+ "pr212_equal_corrected_baseline": true,
+ "pr212_retained_equal_original": true,
+ "pr212_accounting_equal_original": true,
+ "original_to_corrected": {
+ "difference_count": 11,
+ "first_differences": [
+ {
+ "path": "$.all_scored_candidates[1079].event_diagnostics[0].f0_hz",
+ "before": 113.02531383265242,
+ "after": 113.02531383265243,
+ "absolute_difference": 1.4210854715202004e-14,
+ "difference_in_ulp_at_before": 1.0
+ },
+ {
+ "path": "$.all_scored_candidates[1079].event_diagnostics[0].rmse_hz",
+ "before": 1.6618731136088518,
+ "after": 1.6618731136088511,
+ "absolute_difference": 6.661338147750939e-16,
+ "difference_in_ulp_at_before": 3.0
+ },
+ {
+ "path": "$.all_scored_candidates[1079].event_diagnostics[0].score",
+ "before": -1.2918469704343516,
+ "after": -1.2918469704343505,
+ "absolute_difference": 1.1102230246251565e-15,
+ "difference_in_ulp_at_before": 5.0
+ },
+ {
+ "path": "$.all_scored_candidates[1079].event_diagnostics[0].source_drift_hz",
+ "before": -1.6022669802499345,
+ "after": -1.6022669802499139,
+ "absolute_difference": 2.0650148258027912e-14,
+ "difference_in_ulp_at_before": 93.0
+ },
+ {
+ "path": "$.all_scored_candidates[1079].event_diagnostics[0].source_drift_hz_s",
+ "before": -0.08475750862738778,
+ "after": -0.0847575086273867,
+ "absolute_difference": 1.0824674490095276e-15,
+ "difference_in_ulp_at_before": 78.0
+ },
+ {
+ "path": "$.all_scored_candidates[1079].event_scores[0]",
+ "before": -1.2918469704343516,
+ "after": -1.2918469704343505,
+ "absolute_difference": 1.1102230246251565e-15,
+ "difference_in_ulp_at_before": 5.0
+ },
+ {
+ "path": "$.all_scored_candidates[1079].f0_hz[0]",
+ "before": 113.02531383265242,
+ "after": 113.02531383265243,
+ "absolute_difference": 1.4210854715202004e-14,
+ "difference_in_ulp_at_before": 1.0
+ },
+ {
+ "path": "$.all_scored_candidates[1079].residual_score",
+ "before": -7.66681274669189,
+ "after": -7.666812746691889,
+ "absolute_difference": 8.881784197001252e-16,
+ "difference_in_ulp_at_before": 1.0
+ },
+ {
+ "path": "$.all_scored_candidates[1079].score",
+ "before": 1.4443957511027032,
+ "after": 1.4443957511027037,
+ "absolute_difference": 4.440892098500626e-16,
+ "difference_in_ulp_at_before": 2.0
+ },
+ {
+ "path": "$.retention.records[42][2]",
+ "before": 1.4443957511027032,
+ "after": 1.4443957511027037,
+ "absolute_difference": 4.440892098500626e-16,
+ "difference_in_ulp_at_before": 2.0
+ },
+ {
+ "path": "$.retention.stages[0][1]",
+ "before": "853b303924e0d892b94e0254837f004aa0c937876326d4118deca573394ef45d",
+ "after": "4099f0a939775b98a2fb0c6f1ce807c124d9e36c1ca59238cab8f701233b433b"
+ }
+ ]
+ },
+ "corrected_to_pr212": {
+ "difference_count": 0,
+ "first_differences": []
+ },
+ "original_to_exact": {
+ "difference_count": 0,
+ "first_differences": []
+ },
+ "pr212_cpu_speedup_over_exact": 1.1966773017479218,
+ "pr212_cpu_speedup_over_original": 2.2296354449517484
+ },
+ "sparse": {
+ "case": "offset",
+ "scope": "One separately loaded process per frozen variant; same recorded arrays, query tracks, point, clocks and budgets. No location accuracy claim.",
+ "timings": {
+ "corrected": {
+ "wall_seconds": 3.6900103289999606,
+ "cpu_seconds": 3.683335948,
+ "load_seconds_excluded": 0.9317238620001262,
+ "operations": {
+ "scalar_fit_calls": 2191,
+ "acoustic_calls": 297,
+ "scored_candidates": 206
+ },
+ "peak_rss_mib": 205.8359375,
+ "peak_rss_scope": "Worker lifetime through scoring, including imports and input loading",
+ "retained": 28,
+ "independent_groups": 16,
+ "scope": "Real measured audio and observed regional ADS-B; development receiver hypotheses; no independent location-accuracy claim. The restored historical region excludes known jets and does not establish complete flight coverage."
+ },
+ "original": {
+ "wall_seconds": 3.1404853110002477,
+ "cpu_seconds": 3.130182064,
+ "load_seconds_excluded": 0.7733666260000973,
+ "operations": {
+ "scalar_fit_calls": 2191,
+ "acoustic_calls": 297,
+ "scored_candidates": 206
+ },
+ "peak_rss_mib": 206.23046875,
+ "peak_rss_scope": "Worker lifetime through scoring, including imports and input loading",
+ "retained": 28,
+ "independent_groups": 16,
+ "scope": "Real measured audio and observed regional ADS-B; development receiver hypotheses; no independent location-accuracy claim. The restored historical region excludes known jets and does not establish complete flight coverage."
+ },
+ "exact": {
+ "wall_seconds": 2.685179932999745,
+ "cpu_seconds": 2.6791283659999996,
+ "load_seconds_excluded": 0.8818394079999052,
+ "operations": {
+ "scalar_fit_calls": 1161,
+ "acoustic_calls": 297,
+ "scored_candidates": 206
+ },
+ "peak_rss_mib": 225.56640625,
+ "peak_rss_scope": "Worker lifetime through scoring, including imports and input loading",
+ "retained": 28,
+ "independent_groups": 16,
+ "scope": "Real measured audio and observed regional ADS-B; development receiver hypotheses; no independent location-accuracy claim. The restored historical region excludes known jets and does not establish complete flight coverage."
+ },
+ "pr212": {
+ "wall_seconds": 2.034704806999798,
+ "cpu_seconds": 2.030037885,
+ "load_seconds_excluded": 0.7205967680001777,
+ "operations": {
+ "scalar_fit_calls": 437,
+ "acoustic_calls": 297,
+ "scored_candidates": 206
+ },
+ "peak_rss_mib": 203.91015625,
+ "peak_rss_scope": "Worker lifetime through scoring, including imports and input loading",
+ "retained": 28,
+ "independent_groups": 16,
+ "scope": "Real measured audio and observed regional ADS-B; development receiver hypotheses; no independent location-accuracy claim. The restored historical region excludes known jets and does not establish complete flight coverage."
+ }
+ },
+ "exact_variant_equal_original": true,
+ "pr212_equal_corrected_baseline": true,
+ "pr212_retained_equal_original": true,
+ "pr212_accounting_equal_original": true,
+ "original_to_corrected": {
+ "difference_count": 0,
+ "first_differences": []
+ },
+ "corrected_to_pr212": {
+ "difference_count": 0,
+ "first_differences": []
+ },
+ "original_to_exact": {
+ "difference_count": 0,
+ "first_differences": []
+ },
+ "pr212_cpu_speedup_over_exact": 1.31974303819458,
+ "pr212_cpu_speedup_over_original": 1.5419328314653595
+ }
+ },
+ "verification": {
+ "focused_real_tests": 23,
+ "result": "passed",
+ "elapsed_s": 2.47,
+ "source": "fa19b594",
+ "cli_help": "passed",
- "synthetic_suites_run": false
++ "synthetic_suites_run": false,
++ "upstream_discovery_integration": {
++ "source": "dcddc56",
++ "upstream_discovery_source": "9691807",
++ "full_wav_event_payload_exact": true,
++ "ridges": 18,
++ "independent_groups": 16,
++ "trials": 1,
++ "scope": "Integration parity only; repeated headline timings remain unchanged"
++ },
++ "hosted_installation_and_static_checks": {
++ "source": "f805f6f",
++ "python": [
++ "3.11",
++ "3.14"
++ ],
++ "result": "success",
++ "run_url": "https://github.com/1337isnot1337/DOPPLERGEIST/actions/runs/41662314741"
++ }
+ },
+ "full_workflow": {
+ "status": "in_progress",
+ "fresh_run_speedup": null
+ },
+ "preserved_failures": [
+ "Two original-baseline runs stopped after SQLite retention-database errors under Python 3.12.14 / SQLite 3.53.1. Root cause is unestablished.",
+ "The system-runtime baseline reached the 900-second HTTP timeout. A separate continuation retains its original source and inputs; its cumulative active cost is not a fresh-run timing."
- ]
++ ],
++ "published_sources": {
++ "core": {
++ "commit": "f805f6fc6bb72195bdf3c982f359b3201259a5c2",
++ "local_equivalent": "dcddc56fd8ca76d326cdf0fa008299c4bedd092c",
++ "branch": "perf/pipeline-core-20261103",
++ "pull_request": 222
++ },
++ "exact_alternative": {
++ "commit": "c1b7e93ed47ae26beffcf07fdf3ee9ebe639b955",
++ "local_equivalent": "931b1bdb34fa759e4ae0a99ca5560a7ae146ec18",
++ "branch": "perf/pipeline-exact-20261103"
++ },
++ "combined_http": {
++ "commit": "bee7793cf539d4d48fe2fb888f223c1e403a71ef",
++ "local_equivalent": "00727d46aa51691389b453573b6c6644538a9fd5",
++ "branch": "perf/pipeline-combined-20261103"
++ }
++ }
+}
diff --cc docs/PIPELINE_PERFORMANCE_2026-11-03.md
index 26fbf40,0000000..55ea242
mode 100644,000000..100644
--- a/docs/PIPELINE_PERFORMANCE_2026-11-03.md
+++ b/docs/PIPELINE_PERFORMANCE_2026-11-03.md
@@@ -1,130 -1,0 +1,142 @@@
+# Pipeline performance: 2026-11-03
+
+The core changes in [PR #222](https://github.com/1337isnot1337/DOPPLERGEIST/pull/222) reduce repeated audio work, trajectory construction, coordinate transforms, and database commits. They preserve the scientific search rules. The committed [component evidence](../benchmarks/results/pipeline-components-20261103.json) contains individual timings, input hashes, runtime versions, and comparison results.
+
+## Source and measurement scope
+
+| Revision | Role |
+| --- | --- |
+| `7a8dd27979bd9a984c8675c38a00dfbf98a5b01e` | Unmodified source baseline |
- | Core `a2236ac`, tested with main as `fa19b594` | Core changes described here; public commit `1d045b3664648060071fa6e0799738a3fe74b19a` has the same source tree |
- | `931b1bd` | Separate scoring alternative that preserves the original numerical results exactly |
- | `3725e9d` from [PR #212](https://github.com/1337isnot1337/DOPPLERGEIST/pull/212), combined as `00727d4` | Separate scoring implementation used in the combined HTTP evaluation |
++| Core `a2236ac`, tested with main as `fa19b594` | Initial public commit `1d045b3` has the same tree. Current public [core branch](https://github.com/1337isnot1337/DOPPLERGEIST/tree/perf/pipeline-core-20261103) at `f805f6f` also incorporates upstream PR #221 and portable controls. |
++| Local `931b1bd`, public `c1b7e93` | [Separate scoring alternative](https://github.com/1337isnot1337/DOPPLERGEIST/tree/perf/pipeline-exact-20261103) that preserves the original numerical results exactly |
++| `3725e9d` from [PR #212](https://github.com/1337isnot1337/DOPPLERGEIST/pull/212), combined locally as `00727d4` and publicly as `bee7793` | [Combined implementation](https://github.com/1337isnot1337/DOPPLERGEIST/tree/perf/pipeline-combined-20261103) used in the full HTTP evaluation |
+
+All workloads use actual recordings and observed trajectories. The full WAV produces 18 ridges in 16 independent groups. The common HTTP MP3 produces 17 ridges in 15 groups. Each comparison uses identical input bytes within its pair. These development workloads measure runtime and result consistency. They do not establish location accuracy.
+
+## Core changes
+
+- **Audio:** discovery uses scalar medians from upstream [PR #221](https://github.com/1337isnot1337/DOPPLERGEIST/pull/221). PR #222 adds reusable transition slices, penalties, and predecessor indices for ridge tracing. Window lengths and strict tie comparisons stay unchanged.
+- **Flight decoding:** an index reuses decoded-track snapshots. Every read still checks the stored payload hash. Each return contains independent objects. Admission protection stops large repeated queries from evicting entries they just used.
+- **Duplicate tracks:** canonical binary geometry replaces float-to-JSON conversion for private duplicate detection. Persisted evidence hashes retain their original representation.
+- **Terrain:** a bounded cache reuses up to 32 coordinate transforms. DEM and geoid samples remain exactly equal.
+- **Retention:** serial search commits at most 32 empty tiles together. Coarse stages, global stages, checksums, bounds, and checkpoint files remain present. Normal stage validation rejects checkpoints from an interrupted, uncommitted transaction.
+
+The track cache holds at most 32 MiB of serialized, uncompressed snapshots and 2,048 entries per index instance. These limits exclude returned objects and temporary allocations. `track_cache_bytes=0` disables reuse. The transaction benchmark compares ordinary commits with `--batch-size 32`. It measures the same implementation in both arms.
+
+The discovery measurement below used the equivalent local scalar-median implementation before PR #221 merged. The final integration adopts that upstream implementation.
+
+## Component timings
+
+Times below are wall seconds. Audio, helper, and empty-tile rows use medians from three alternating pairs. Warm-cache rows use three queries after an initial query. The [evidence file](../benchmarks/results/pipeline-components-20261103.json) preserves CPU timings where measured.
+
+| Workload | Control | Before | After | Speedup |
+| --- | --- | ---: | ---: | ---: |
+| Full WAV discovery, intake and plots excluded | Old/new source | 7.357 | 6.044 | 1.22× |
+| Two recorded ridge windows | Old/new source | 0.183 / 0.0736 | 0.0607 / 0.0256 | 3.02× / 2.87× |
+| 441 tracks, 89,504 samples, warm query | Cache disabled/enabled | 1.568 | 0.101 | 15.53× |
+| 1,183 tracks, 287,491 samples, warm query | Cache disabled/enabled | 4.725 | 2.745 | 1.72× |
+| Geometry identities for 214 tracks | Old/new helper | 0.712 | 0.0107 | 66.50× |
+| Duplicate-index query, track cache disabled | Old/new helper | 2.970 | 1.490 | 1.99× |
+| 283 scalar DEM and geoid queries | Old/new helper | 0.482 | 0.151 | 3.20× |
+| 256 recorded empty tiles | Ordinary/batched commits | 2.169 | 0.765 | 2.84× |
+
+The helper query contains 71,571 samples. Terrain sampling covers 274 distinct candidate coordinates. Empty-tile timing includes real index queries, geometry, checkpoint writes, and all 512 ledger stages. It excludes progress updates, resume reads, and nonempty scoring.
+
+Audio, helper, and scoring measurements use Python 3.12.14. Cache, transaction, configured-worker, and HTTP controls use Python 3.12.3 with SQLite 3.45.1. Each pair uses the same interpreter and numerical packages. Cross-runtime ratios are not controlled comparisons.
+
+### Cold and warm cache behavior
+
+Cold means an empty decoded-track cache. Warm means reuse after an earlier query. Neither term establishes a cold operating-system file cache. The dense query costs 4.041 seconds without caching and 4.936 seconds with caching on its first pass: approximately **22% slower**. Across one cold and three warm passes, it improves by 1.35×. The regional workload improves by 3.43× across all four passes. Its 15.53× result applies only to warm queries.
+
+## Separate scoring implementations
+
+The four-arm comparison loads each frozen variant in a separate process. Each cell below is **one process CPU measurement**, not a median. Input loading is excluded. Both workloads use the same 16 observed groups, receiver hypothesis, clock range, and budgets across variants.
+
+| Workload | Original baseline | Arithmetic-corrected baseline | Exact alternative `931b1bd` | PR #212 `3725e9d` |
+| --- | ---: | ---: | ---: | ---: |
+| Dense: 1,927 associations | 35.878 s | 36.826 s | 19.256 s | 16.091 s |
+| Offset: 206 associations | 3.130 s | 3.683 s | 2.679 s | 2.030 s |
+
+PR #212 is 2.23× and 1.54× faster than the original on these cases. It exactly matches the corrected baseline, including discarded candidates and ledger contents. Retained candidates and accounting also match the original baseline. A separate power-arithmetic correction changes one discarded dense candidate's total score by two units in the last place. Related diagnostics and its ledger checksum also change. The comparison preserves those differences without treating them as exact equivalence.
+
- The exact alternative matches the original results and remains available separately. Its portable scoring workload and paired benchmark are preserved at `931b1bd`, in `docs/verification/REAL_SEARCH_BATCH_PERFORMANCE.md`. Those scoring changes are separate from the core changes here.
++The exact alternative matches the original results and remains available separately. Its portable workload and paired benchmark are in the [scoring methodology](https://github.com/1337isnot1337/DOPPLERGEIST/blob/c1b7e93ed47ae26beffcf07fdf3ee9ebe639b955/docs/verification/REAL_SEARCH_BATCH_PERFORMANCE.md). Those scoring changes are separate from the core changes here.
+
+## Reproduce the core comparisons
+
+Follow [installation instructions](SETUP.md), then run from the repository root. Select the same interpreter and package versions for both arms. Limit numerical threads. Optional Linux CPU affinity can reduce interference. Use a new output path for every run.
+
+```bash
+. .venv/bin/activate
+export PYTHONPATH=src
+export OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1
+python scripts/restore_artifacts.py \
+ --bundle data-source --bundle data-adsb-derived --bundle data-terrain
+git worktree add --detach ../DOPPLERGEIST-perf-baseline \
+ 7a8dd27979bd9a984c8675c38a00dfbf98a5b01e
+PERF_BASELINE=../DOPPLERGEIST-perf-baseline
+PERF_INDEX=results/perf-input/reference-region.sqlite
+
+python - <<'PY'
+import json
+from pathlib import Path
+from dopplergeist.data_access import FlightIndex
+fixture = json.loads(Path('tests/data/real-empty-tile-workload.json').read_text())
+FlightIndex.from_parquet(
+ 'results/perf-input/reference-region.sqlite',
+ 'data/adsb/2026-09-08/reference-region.parquet',
+ coverage=fixture['index_inputs'][0]['coverage'],
+)
+PY
+
+python scripts/benchmark_real_audio_performance.py \
+ data/source/reference-recording.wav --baseline "$PERF_BASELINE" \
+ --ridge-config data/source/reproduce-local.json \
+ --repeats 3 --output results/new-audio-comparison
+
+python scripts/benchmark_real_performance_helpers.py \
+ --index "$PERF_INDEX" \
+ --bounds -73.87498229500986 40.62693175890468 -73.17498229500987 41.12693175890468 \
+ --time-range 1788897749.6620934 1788903449.6620934 \
+ --baseline "$PERF_BASELINE" --repeats 3 --output results/new-helper-comparison
+
+python scripts/benchmark_real_track_cache.py --index "$PERF_INDEX" \
+ --bounds -74.299988 40.100005 -72.700011 41.599964 \
+ --time-range 1788898500 1788902400 --passes 4 \
+ --output results/new-track-cache.json
+
+python scripts/benchmark_real_empty_tiles.py --index "$PERF_INDEX" \
+ --tiles 256 --batch-size 32 --passes 3 --output results/new-empty-tile-comparison
+```
+
+For the dense cache case, substitute bounds `-74.05 40.55 -73.55 41.05`, times `1788825600 1788912000`, and limits `--max-tracks 5000 --max-samples 750000`. The portable empty-tile fixture contains completed checkpoints from a failed baseline run. It compares all scientific index inputs and source byte hashes, allowing only local source paths to change after restoration.
+
+## Verification and complete workflows
+
- The merged core revision passed **23 focused real-input checks in 2.47 seconds**, plus CLI help. No synthetic suite ran. Component gates require exact measured arrays, diagnostics, accounting, and logical ledger contents. Numeric differences fail these gates. Event thresholds, harmonic grouping, clock domains, coverage requirements, and unknown-support rules remain unchanged. The regional archive excludes known jets and cannot establish complete coverage. Historical receiver and clock hypotheses do not provide independent accuracy evidence.
++The initial merged core revision passed **23 focused real-input checks in 2.47 seconds**, plus CLI help. No synthetic suite ran. After adopting upstream PR #221, a full-WAV comparison again produces exactly equal event payloads: 18 ridges in 16 groups. This single-pair integration check does not replace the repeated timing measurements. Hosted installation and static checks also pass on Python 3.11 and 3.14 for `f805f6f`. Component gates require exact measured arrays, diagnostics, accounting, and logical ledger contents. Numeric differences fail these gates. Event thresholds, harmonic grouping, clock domains, coverage requirements, and unknown-support rules remain unchanged. The regional archive excludes known jets and cannot establish complete coverage. Historical receiver and clock hypotheses do not provide independent accuracy evidence.
++
++After restoring the terrain bundle, reproduce the 23 cases with:
++
++```bash
++python -m pip install '.[test]'
++PYTHONPATH=src OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 \
++ python -m pytest -q tests/test_real_track_cache.py \
++ tests/test_real_retention_batch.py tests/test_real_performance_helpers.py \
++ tests/test_real_index_staging.py
++```
++
++These tests build temporary indexes from committed observations. They do not need the benchmark WAV or regional index. Without the restored terrain files, two cases skip.
+
+Configured scoring with one versus four workers took **63.08 versus 50.86 seconds**, a 1.24× improvement in one sequential pair. This uses upstream [PR #210](https://github.com/1337isnot1337/DOPPLERGEIST/pull/210) from `adbfb3e`, with the same local changes in both arms. All 16 scientific JSON artifacts, three CSV files, and 283 ranked rows match after explicit metadata exclusions. The source-informed configured replay remains unaccepted, with incomplete model support.
+
+The complete automatic HTTP comparison is pending. Two original baseline attempts stopped with retention database errors. Their cause is unestablished. The system-runtime baseline reached its 900-second HTTP timeout. Its original receipt remains failed. A separate continuation preserves the baseline source and inputs and records restart/replay costs explicitly.
+
+After both workers stop, the comparator requires full-workflow evidence and compares discovery, candidates, clocks, associations, diagnostics, decisions, and logical retention ledgers. It ignores only enumerated implementation and run metadata. Use the explicit continuation report:
+
+```bash
+python scripts/compare_real_upload_performance.py \
+ --baseline path/to/baseline --current path/to/current \
+ --baseline-continuation path/to/baseline/continuation.json \
+ --output results/new-upload-comparison.json
+```
+
+A resumed baseline can establish scientific parity. Its cumulative active-time ratio is reported separately and cannot establish an uninterrupted fresh-run speedup. Stage history has polling uncertainty. Failed or incomplete workflows do not qualify for an overall speedup claim.
diff --cc src/dopplergeist/batch_ridge.py
index 3b9d579,78995bd..71be78b
--- a/src/dopplergeist/batch_ridge.py
+++ b/src/dopplergeist/batch_ridge.py
@@@ -1,10 -1,10 +1,66 @@@
- """Batched versions of the existing robust first-event ridge fits."""
+ """Batched robust ridge fits with the scalar model's arithmetic order."""
++import hashlib
++import pickle
++
import numpy as np
- from .matching import _fit_ridge_uncached, _frequency_scale, _ridge_fit_inputs
-from .matching import _frequency_scale, _ridge_fit_inputs
++from .matching import _fit_argument_key, _frequency_scale, _ridge_fit_cache, _ridge_fit_inputs
++
++
++def _batch_fit_key(measurement, event, factors, emissions):
++ """Identify a complete batch without retaining its input arrays.
++
++ Reduction layout is part of the input: equal values with different strides
++ need not follow the same floating-point summation order.
++ """
++ if type(measurement) not in (tuple, list) or len(measurement) != 4:
++ raise TypeError("Uncached custom measurement container")
++ digest = hashlib.sha256()
++ settings = tuple(_fit_argument_key(value) for value in
++ (_frequency_scale(event), event.get("quality", {}).get("source_drift_fraction", .02),
++ event["f0_range_hz"]))
++ digest.update(pickle.dumps(settings, protocol=5))
++ for array in (measurement[0].time, measurement[1], measurement[2], factors, emissions):
++ if type(array) is not np.ndarray or array.dtype.kind not in "biuf":
++ raise TypeError("Uncached custom batch input")
++ layout = (array.dtype.str, array.shape, array.strides, array.flags.aligned)
++ digest.update(pickle.dumps(layout, protocol=5))
++ digest.update(array.tobytes(order="C"))
++ return digest.digest()
def first_event_fits(measurement, event, factors, emissions):
++ """Reuse an identical complete fit batch within the current scoring call."""
++ cache = _ridge_fit_cache.get()
++ if cache is None or cache.max_entries < 1 or cache.batch_fit_limit <= 0:
++ return _first_event_fits_uncached(measurement, event, factors, emissions)
++ try:
++ key = _batch_fit_key(measurement, event, factors, emissions)
++ except (TypeError, ValueError, KeyError, AttributeError):
++ return _first_event_fits_uncached(measurement, event, factors, emissions)
++ entries = cache.batch_fit_entries
++ if key in entries:
++ entries.move_to_end(key)
++ # Only snapshots produced below from our own scalar-valued diagnostics
++ # enter this private memory cache. No files or external pickles are read.
++ return pickle.loads(entries[key])
++ result = _first_event_fits_uncached(measurement, event, factors, emissions)
++ if result is not None:
++ snapshot = pickle.dumps(result, protocol=5)
++ size = len(key) + len(snapshot)
++ if size <= cache.batch_fit_limit:
++ while entries and (cache.batch_fit_bytes + size > cache.batch_fit_limit
++ or len(entries) >= cache.max_entries):
++ old_key, old_snapshot = entries.popitem(last=False)
++ cache.batch_fit_bytes -= len(old_key) + len(old_snapshot)
++ entries[key] = snapshot
++ cache.batch_fit_bytes += size
++ # None fallbacks and exceptions are never cached. The first caller also
++ # owns its diagnostics: later mutations cannot change the stored bytes.
++ return result
++
++
++def _first_event_fits_uncached(measurement, event, factors, emissions):
"""Fit supported clocks together, or return None for the ordinary fitter.
This uses the same six iterations, box constraints, and diagnostics as the
diff --cc src/dopplergeist/matching.py
index 3a8fb47,88a7ec9..aed8035
--- a/src/dopplergeist/matching.py
+++ b/src/dopplergeist/matching.py
@@@ -51,6 -50,26 +50,29 @@@ class _RidgeFitCache
self.acoustic_entries = OrderedDict()
self.acoustic_bytes = 0
self.acoustic_limit = 32 * 1024 * 1024
+ self.measurement_entries = OrderedDict()
+ self.measurement_bytes = 0
+ self.measurement_limit = 4 * 1024 * 1024
++ self.batch_fit_entries = OrderedDict()
++ self.batch_fit_bytes = 0
++ self.batch_fit_limit = 4 * 1024 * 1024
+
+ def prepared_measurement(self, measurement, sigma_hz, drift_fraction, key):
+ """Reuse selected samples only after checking their current content key."""
+ if key in self.measurement_entries:
+ self.measurement_entries.move_to_end(key)
+ return self.measurement_entries[key][0]
+ prepared = _prepare_ridge_fit_inputs(measurement, sigma_hz, drift_fraction)
+ # Include the lazily normalized weights in the bound before allocation.
+ size = prepared.times.nbytes + prepared.ridge.nbytes + 2 * prepared.weight.nbytes + 512
+ if size <= self.measurement_limit and self.max_entries > 0:
+ while (self.measurement_bytes + size > self.measurement_limit or
+ len(self.measurement_entries) >= self.max_entries):
+ _, (_, removed) = self.measurement_entries.popitem(last=False)
+ self.measurement_bytes -= removed
+ self.measurement_entries[key] = (prepared, size)
+ self.measurement_bytes += size
+ return prepared
def acoustic_batch(self, track, observer, starts, times, c):
"""Reuse exact batched propagation inputs within this scoring call."""
@@@ -121,7 -145,7 +148,9 @@@
def _cache_ridge_fits(function, *, max_entries=4096):
"""Reuse fits only while this batch's tracks stay unchanged.
-- Each call has at most 4,096 entries by default. Context-local state prevents
++ Each fit cache has at most 4,096 entries by default. Complete-batch snapshots
++ and keys are additionally bounded to 4 MiB; prepared measurements use their
++ own 4 MiB bound and acoustic arrays use 32 MiB. Context-local state prevents
reuse between concurrent jobs, nested batches or later calls. Cached results
are copied so candidate diagnostics remain independently mutable.
"""
diff --cc tests/data/real-ridge-batch-arithmetic.json
index 0000000,0000000..aebf94e
new file mode 100644
--- /dev/null
+++ b/tests/data/real-ridge-batch-arithmetic.json
@@@ -1,0 -1,0 +1,1 @@@
++{"scope":"Numerical regression from actual recorded ridge samples and acoustic geometry computed from an observed ADS-B trajectory. No synthesized recording or aircraft track. Not a location-accuracy benchmark.","provenance":{"recording":"Original public reference recording, automatic discovery at source 7a8dd27979bd9a984c8675c38a00dfbf98a5b01e","track":"ab50b9:11","index_source":"Restored reference-region.parquet","receiver":[40.87693175890468,-73.52498229500986,-31.25569725036621],"clock_span_s":[1788898669.322983,1788898687.9139113],"original_clock_rows":[0,16,20,38],"event_index":5},"event":{"f0_range_hz":[9.830666813254357,216.125],"quality":{"fragment_count":1.0,"frequency_drop_hz":13.766116619110107,"frequency_resolution_hz":0.5,"frequency_sigma_hz":0.5,"frequency_uncertainty_hz":0.5,"harmonic_count":1.0,"max_observed_gap_s":2.0,"median_contrast":2.942490339279175,"support_fraction":0.7560975609756098,"time_uncertainty_s":1.0,"frequency_loss_scale_hz":1.0,"cpa_tolerance_s":11.0}},"times":[731.5,732.0,733.0,733.5,735.0,735.5,736.5,737.0,737.5,738.0,738.5,739.0,739.5,740.0,740.5,741.0,741.5,742.0,742.5,744.5,745.0,745.5,746.0,746.5,747.0,748.5,749.0,749.5,750.0,750.5,751.5],"ridge":[113.75,113.69526600837708,112.95879816636443,113.3504446297884,112.09085258096457,112.22484920918941,111.12312008440495,110.55429127067327,110.32999286055565,109.80890287458897,109.37539254128933,109.01969337835908,108.8177992850542,108.25,107.94676958024502,107.61872723698616,107.4870186606422,106.25,105.99334615562111,104.66795042157173,104.25,103.58017382770777,102.9968695116695,102.56404300034046,101.91053862124681,100.75,100.75,99.69266101717949,99.58432801067829,99.42714183032513,98.30666813254356],"weight":[0.02854685239237662,0.02965872021752487,0.02778651736172654,0.02698477342300842,0.03369723265374049,0.028743482439561027,0.03264757447853784,0.03526625718622174,0.028974896791178855,0.028907138393770307,0.029010587485179342,0.03306818032046818,0.03324788424859854,0.03509938393420076,0.0343728010023015,0.03979248937514358,0.034447029942795034,0.03127746008937571,0.03246525665641521,0.03168948219101514,0.03488261047015061,0.03142208186267668,0.04298980039820184,0.0448538457478966,0.03534476900999533,0.02713655130884027,0.028379378304212034,0.02764316658796013,0.03383168899844089,0.029123100472259005,0.028709006256226886],"factors":[[1.1004948669942944,1.1027086514270337,1.0948675971327606,1.0859581456188934,1.0830173410897916,1.0753297043948171,1.0383552736585195,1.0528205994900206,1.0384009474732903,1.0358250254237504,1.04761153209947,1.0228884857309584,1.0181660237312717,1.0278199833233024,1.0116148026991694,1.0068254861476484,1.0018016574851893,0.991491667837132,0.9877144363698785,0.9878898509239141,0.9632596484746797,0.947222648472472,0.9419343367880254,0.9468162368272132,0.9624242302218193,0.9454039727961214,0.9326350680366003,0.9353115763687707,0.9233635295175077,0.9173700807672138,0.9221078194465587],[1.0176670016711606,1.0253388840284339,1.0072324154963006,1.0069843384699018,0.9886310079474123,0.9782634394730728,0.9821357827257494,0.9715686041871266,0.9514689694213602,0.9426024835408328,0.9439757276698636,0.9557521387336612,0.9583111840743384,0.9431073307934533,0.9608928685315397,0.932000824234714,0.9409344088536367,0.9267769605036743,0.9187932566646079,0.9232423709300913,0.9198603736377551,0.9172804139976666,0.9154790042959016,0.9144395737487249,0.9141520134665783,0.9107439144472869,0.9093911885660734,0.9081258662083285,0.9069440544817227,0.9058845040680077,0.9041598941029166],[1.008419193319764,0.9950986800574171,0.9883580215246665,0.9792045464121627,0.9738916379773435,0.9527342826532316,0.9434580107739216,0.9543035008440677,0.9622919780338096,0.9423047673822649,0.9658056363303824,0.932652549170973,0.9424696051970997,0.927743045407192,0.919253176971592,0.9164608805692815,0.9191968747557708,0.9276312083667423,0.9235720897563096,0.914499400660036,0.9141473311413687,0.9138651152427091,0.9123167845446678,0.9108646261548182,0.9095041913083692,0.9059655815552438,0.9050589389300411,0.904227945915207,0.9034718519042229,0.9027899713545361,0.9016464319795076],[0.9218464753595064,0.9168851928415691,0.923660853283718,0.9251328420137426,0.916236420428088,0.9148377169255751,0.9143010975549727,0.9129051462283804,0.9114162828320131,0.9100207915560425,0.9087145507306122,0.9074936779215562,0.9063541760882644,0.9054012797106908,0.9045410622258298,0.9037560264214068,0.9030454620250373,0.9024087239056283,0.9018452297725825,0.9003141886050018,0.900110302018388,0.8999774217964706,0.8999153748043853,0.8995211445228067,0.8975750523882486,0.8936095402774326,0.8928960200476092,0.892481461378123,0.8923642777611899,0.8925438358081533,0.8800764269028939]],"emissions":[[1788899349.6357303,1788899350.1867025,1788899351.2868345,1788899351.8319294,1788899353.4541078,1788899353.995684,1788899355.0601423,1788899355.5794163,1788899356.1044846,1788899356.6219132,1788899357.1443346,1788899357.661057,1788899358.170534,1788899358.6822662,1788899359.1916256,1788899359.6957762,1788899360.1990182,1788899360.6970577,1788899361.191596,1788899363.153552,1788899363.6416903,1788899364.118835,1788899364.5906973,1788899365.0624583,1788899365.5392942,1788899366.9712296,1788899367.4388676,1788899367.907924,1788899368.3723316,1788899368.8322775,1788899369.7501614],[1788899357.995239,1788899358.505196,1788899359.5223434,1788899360.0260334,1788899361.515406,1788899362.0078118,1788899362.9839156,1788899363.4750843,1788899363.9553375,1788899364.4284174,1788899364.8996406,1788899365.3741207,1788899365.8559372,1788899366.3290184,1788899366.8071504,1788899367.2783587,1788899367.7463527,1788899368.2130072,1788899368.674156,1788899370.5160086,1788899370.9767501,1788899371.4360025,1788899371.8941603,1788899372.3516085,1788899372.808725,1788899374.1780057,1788899374.6330357,1788899375.0874114,1788899375.5411754,1788899375.9943724,1788899376.8993692],[1788899359.982642,1788899360.4832225,1788899361.4728415,1788899361.9656677,1788899363.433199,1788899363.9143403,1788899364.8590043,1788899365.3329973,1788899365.8145883,1788899366.2884269,1788899366.76567,1788899367.2382138,1788899367.7058032,1788899368.1730795,1788899368.634583,1788899369.0932803,1788899369.5519633,1788899370.0134249,1788899370.4762466,1788899372.3122315,1788899372.769362,1788899373.2264535,1788899373.6829948,1788899374.1387863,1788899374.5938747,1788899375.955363,1788899376.408116,1788899376.8604348,1788899377.3123565,1788899377.7639189,1788899378.6661127],[1788899368.456221,1788899368.915669,1788899369.8340764,1788899370.297136,1788899371.6772604,1788899372.1349971,1788899373.049317,1788899373.5061576,1788899373.962234,1788899374.4175894,1788899374.8722696,1788899375.3263183,1788899375.779777,1788899376.232709,1788899376.6851914,1788899377.1372626,1788899377.58896,1788899378.0403204,1788899378.491381,1788899380.2933486,1788899380.7434516,1788899381.1934707,1788899381.643441,1788899382.093379,1788899382.54264,1788899383.8856823,1788899384.3322961,1788899384.778628,1788899385.2248273,1788899385.6710417,1788899386.5586326]],"legacy_diagnostics":[{"score":-1.1380688253736158,"f0_hz":106.36462999886578,"source_drift_hz":2.1317529588209245,"source_drift_hz_s":0.10598127004865594,"source_reference_emission_time_s":1788899359.692946,"source_span_s":20.114431142807007,"drift_bound_hz":2.1317529588209245,"drift_at_bound":true,"rmse_hz":1.3395265267977907,"support_fraction":1.0,"sample_count":31,"independent_sample_count":31,"outlier_fraction":0.0},{"score":-1.2918469704343516,"f0_hz":113.02531383265242,"source_drift_hz":-1.6022669802499345,"source_drift_hz_s":-0.08475750862738778,"source_reference_emission_time_s":1788899367.4473042,"source_span_s":18.904130220413208,"drift_bound_hz":2.2628871086643803,"drift_at_bound":false,"rmse_hz":1.6618731136088518,"support_fraction":1.0,"sample_count":31,"independent_sample_count":31,"outlier_fraction":0.05761614464999719},{"score":-1.439823911964601,"f0_hz":114.29446660959415,"source_drift_hz":-2.2890764137412907,"source_drift_hz_s":-0.12251880002970683,"source_reference_emission_time_s":1788899369.3243773,"source_span_s":18.683470726013184,"drift_bound_hz":2.2890764137412907,"drift_at_bound":true,"rmse_hz":1.8847690987551526,"support_fraction":1.0,"sample_count":31,"independent_sample_count":31,"outlier_fraction":0.028709006256226882},{"score":-1.932810318502893,"f0_hz":118.5637088956719,"source_drift_hz":-2.3815880412576145,"source_drift_hz_s":-0.1315619214672819,"source_reference_emission_time_s":1788899377.5074267,"source_span_s":18.10241150856018,"drift_bound_hz":2.3815880412576145,"drift_at_bound":true,"rmse_hz":2.958636284492512,"support_fraction":1.0,"sample_count":31,"independent_sample_count":31,"outlier_fraction":0.38496036213349544}]}
diff --cc tests/test_real_complete_batch_cache.py
index 0000000,0000000..9d13231
new file mode 100644
--- /dev/null
+++ b/tests/test_real_complete_batch_cache.py
@@@ -1,0 -1,0 +1,146 @@@
++"""Bounded whole-batch reuse on actual recorded ridge/trajectory inputs."""
++import json
++from pathlib import Path
++
++import numpy as np
++import pytest
++
++from dopplergeist import batch_ridge, matching
++
++
++def recorded():
++ fixture = json.loads((Path(__file__).parent/'data/real-ridge-batch-arithmetic.json').read_text())
++ measurement = (matching.RidgeTimes(np.asarray(fixture['times'])),
++ np.asarray(fixture['ridge']), np.asarray(fixture['weight']), {})
++ return measurement, fixture['event'], np.asarray(fixture['factors']), np.asarray(fixture['emissions'])
++
++
++def counted_kernel(monkeypatch):
++ calls = []
++ original = batch_ridge._first_event_fits_uncached
++
++ def run(*args):
++ calls.append(1)
++ return original(*args)
++
++ monkeypatch.setattr(batch_ridge, '_first_event_fits_uncached', run)
++ return calls, original
++
++
++def test_recorded_complete_batch_cache_preserves_values_and_caller_ownership(monkeypatch):
++ measurement, event, factors, emissions = recorded()
++ calls, original = counted_kernel(monkeypatch)
++ expected = original(measurement, event, factors, emissions)
++ assert expected is not None
++
++ @matching._cache_ridge_fits
++ def run():
++ before = len(calls)
++ first = batch_ridge.first_event_fits(measurement, event, factors, emissions)
++ assert first == expected and len(calls) == before+1
++ first[0]['f0_hz'] = -123.
++ first.append({'caller_only': True})
++ second = batch_ridge.first_event_fits(measurement, event, factors.copy(), emissions.copy())
++ assert second == expected and len(calls) == before+1
++ second[1].clear()
++ third = batch_ridge.first_event_fits(measurement, event, factors, emissions)
++ assert third == expected and len(calls) == before+1
++ cache = matching._ridge_fit_cache.get()
++ assert len(cache.batch_fit_entries) == 1
++ assert cache.batch_fit_bytes == sum(len(key)+len(value) for key, value in cache.batch_fit_entries.items())
++ assert 0 < cache.batch_fit_bytes <= cache.batch_fit_limit
++
++ run()
++ run()
++ assert len(calls) == 2
++
++
++def test_recorded_batch_layout_and_changed_inputs_do_not_share_results(monkeypatch):
++ measurement, event, factors, emissions = recorded()
++ calls, original = counted_kernel(monkeypatch)
++
++ @matching._cache_ridge_fits
++ def run():
++ assert batch_ridge.first_event_fits(measurement, event, factors, emissions) == original(
++ measurement, event, factors, emissions)
++ assert len(calls) == 1
++ # Fortran layout keeps every observed value but changes reduction
++ # strides. Each layout must reproduce its own unchanged kernel result.
++ column_factors, column_emissions = np.asfortranarray(factors), np.asfortranarray(emissions)
++ expected = original(measurement, event, column_factors, column_emissions)
++ assert batch_ridge.first_event_fits(measurement, event, column_factors, column_emissions) == expected
++ assert len(calls) == 2
++ assert batch_ridge.first_event_fits(measurement, event, column_factors.copy(order='F'),
++ column_emissions.copy(order='F')) == expected
++ assert len(calls) == 2
++ # Use a shorter portion of the recorded ridge, without manufacturing
++ # samples. Its normalization and acoustic arrays require another key.
++ shorter = (matching.RidgeTimes(measurement[0].time[:-1]),
++ measurement[1][:-1], measurement[2][:-1], {})
++ assert batch_ridge.first_event_fits(shorter, event, factors[:, :-1], emissions[:, :-1]) == original(
++ shorter, event, factors[:, :-1], emissions[:, :-1])
++ assert len(calls) == 3
++ quality = dict(event.get('quality', {}), frequency_loss_scale_hz=matching._frequency_scale(event)*1.01)
++ changed_model = dict(event, quality=quality)
++ assert batch_ridge.first_event_fits(measurement, changed_model, factors, emissions) == original(
++ measurement, changed_model, factors, emissions)
++ assert len(calls) == 4
++
++ run()
++
++
++def test_recorded_batch_cache_respects_byte_and_entry_limits(monkeypatch):
++ measurement, event, factors, emissions = recorded()
++ calls, original = counted_kernel(monkeypatch)
++
++ def run():
++ cache = matching._ridge_fit_cache.get()
++ expected = original(measurement, event, factors, emissions)
++ batch_ridge.first_event_fits(measurement, event, factors, emissions)
++ batch_ridge.first_event_fits(measurement, event, factors[::-1], emissions[::-1])
++ assert len(cache.batch_fit_entries) == 1
++ assert batch_ridge.first_event_fits(measurement, event, factors, emissions) == expected
++ assert len(calls) == 3
++ cache.batch_fit_entries.clear()
++ cache.batch_fit_bytes = 0
++ cache.batch_fit_limit = 1
++ for _ in range(2):
++ assert batch_ridge.first_event_fits(measurement, event, factors, emissions) == expected
++ assert not cache.batch_fit_entries and cache.batch_fit_bytes == 0
++ assert len(calls) == 5
++
++ matching._cache_ridge_fits(run, max_entries=1)()
++
++
++def test_recorded_zero_drift_fallback_is_not_cached(monkeypatch):
++ measurement, event, factors, emissions = recorded()
++ calls, _ = counted_kernel(monkeypatch)
++ no_drift = dict(event, quality=dict(event.get('quality', {}), source_drift_fraction=0.))
++
++ @matching._cache_ridge_fits
++ def run():
++ for _ in range(2):
++ assert batch_ridge.first_event_fits(measurement, no_drift, factors, emissions) is None
++ assert len(calls) == 2
++ cache = matching._ridge_fit_cache.get()
++ assert not cache.batch_fit_entries and cache.batch_fit_bytes == 0
++
++ run()
++
++
++def test_recorded_warm_cache_does_not_hide_invalid_measurement_arity():
++ measurement, event, factors, emissions = recorded()
++
++ class DifferentIteration(list):
++ def __iter__(self):
++ return iter(measurement[:3])
++
++ @matching._cache_ridge_fits
++ def run():
++ assert batch_ridge.first_event_fits(measurement, event, factors, emissions) is not None
++ for invalid in (measurement[:3], measurement+(None,), DifferentIteration(measurement)):
++ with pytest.raises(ValueError, match='unpack'):
++ batch_ridge.first_event_fits(invalid, event, factors, emissions)
++ assert len(matching._ridge_fit_cache.get().batch_fit_entries) == 1
++
++ run()