Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
docs: assemble reviewed real audio candidates (#146)
Cinna Davis396c3c159a5c
diff --git a/benchmarks/audio-candidate-index-20261103.csv b/benchmarks/audio-candidate-index-20261103.csvnew file mode 100644index 0000000..55a7d81--- /dev/null+++ b/benchmarks/audio-candidate-index-20261103.csv@@ -0,0 +1,26 @@+candidate_id,source_type,source_parent_id,source_url,title,search_category,clip_duration_s,scanner_result,proposal_count,visual_review,recording_location_class,coarse_location_hint,reference_coordinate_key,recording_clock_status,aircraft_identity,location_evaluation_eligible+esid-706,Eclipse Soundscapes field recording,706,https://zenodo.org/records/22227872,2024-04-08 Eclipse Soundscapes ESID 706,publisher field site,600,5 moving-tone proposal groups,5,descending_band_visible: descending moving-tone proposal visible; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 706,esid-706,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-688,Eclipse Soundscapes field recording,688,https://zenodo.org/records/22025528,2024-04-08 Eclipse Soundscapes ESID 688,publisher field site,600,15 moving-tone proposal groups,15,descending_band_visible: descending tonal ridge visible; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 688,esid-688,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-656,Eclipse Soundscapes field recording,656,https://zenodo.org/records/22224306,2024-04-08 Eclipse Soundscapes ESID 656,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: several visible parallel descending tonal bands; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 656,esid-656,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-485,Eclipse Soundscapes field recording,485,https://zenodo.org/records/22224082,2024-04-08 Eclipse Soundscapes ESID 485,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: visible descending tonal ridge with an abrupt segment; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 485,esid-485,filename timestamp unverified; Needs to be set manually,unconfirmed,no+esid-281,Eclipse Soundscapes field recording,281,https://zenodo.org/records/22214458,2024-04-08 Eclipse Soundscapes ESID 281,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: visible descending tonal ridge; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 281,esid-281,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-847,Eclipse Soundscapes field recording,847,https://zenodo.org/records/22059635,2024-04-08 Eclipse Soundscapes ESID 847,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: clear descending tonal ridge; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 847,esid-847,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-696,Eclipse Soundscapes field recording,696,https://zenodo.org/records/22050432,2024-04-08 Eclipse Soundscapes ESID 696,publisher field site,600,2 moving-tone proposal groups,2,descending_band_visible: multiple visible descending tonal bands; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 696,esid-696,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-103,Eclipse Soundscapes field recording,103,https://zenodo.org/records/22049202,2024-04-08 Eclipse Soundscapes ESID 103,publisher field site,600,2 moving-tone proposal groups,2,descending_band_visible: clear descending tonal ridge; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 103,esid-103,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-756,Eclipse Soundscapes field recording,756,https://zenodo.org/records/22030700,2024-04-08 Eclipse Soundscapes ESID 756,publisher field site,600,6 moving-tone proposal groups,6,descending_band_visible: two clear descending tonal bands; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 756,esid-756,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-920,Eclipse Soundscapes field recording,920,https://zenodo.org/records/22014438,2024-04-08 Eclipse Soundscapes ESID 920,publisher field site,600,6 moving-tone proposal groups,6,descending_band_visible: two clear descending tonal bands; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 920,esid-920,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-962,Eclipse Soundscapes field recording,962,https://zenodo.org/records/22002491,2024-04-08 Eclipse Soundscapes ESID 962,publisher field site,600,2 moving-tone proposal groups,2,descending_band_visible: visible shallow descending tonal band; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 962,esid-962,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-665,Eclipse Soundscapes field recording,665,https://zenodo.org/records/21999916,2024-04-08 Eclipse Soundscapes ESID 665,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: visible descending band with a sharper transition; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 665,esid-665,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-762,Eclipse Soundscapes field recording,762,https://zenodo.org/records/21786378,2024-04-08 Eclipse Soundscapes ESID 762,publisher field site,600,4 moving-tone proposal groups,4,descending_band_visible: clear descending low-frequency band; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 762,esid-762,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-710,Eclipse Soundscapes field recording,710,https://zenodo.org/records/21722091,2024-04-08 Eclipse Soundscapes ESID 710,publisher field site,600,2 moving-tone proposal groups,2,descending_band_visible_weak_or_interrupted: fragmented and weak descending energy; visual support is limited; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 710,esid-710,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-443,Eclipse Soundscapes field recording,443,https://zenodo.org/records/21755346,2024-04-08 Eclipse Soundscapes ESID 443,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: visible low-frequency descending band; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 443,esid-443,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-445,Eclipse Soundscapes field recording,445,https://zenodo.org/records/21743960,2024-04-08 Eclipse Soundscapes ESID 445,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: clear descending low-frequency band; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 445,esid-445,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-439,Eclipse Soundscapes field recording,439,https://zenodo.org/records/21804576,2024-04-08 Eclipse Soundscapes ESID 439,publisher field site,600,6 moving-tone proposal groups,6,descending_band_visible: two visible descending bands with a later upward segment; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 439,esid-439,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-430,Eclipse Soundscapes field recording,430,https://zenodo.org/records/21772857,2024-04-08 Eclipse Soundscapes ESID 430,publisher field site,600,5 moving-tone proposal groups,5,descending_band_visible: clear descending tonal band; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 430,esid-430,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-923,Eclipse Soundscapes field recording,923,https://zenodo.org/records/21918793,2024-04-08 Eclipse Soundscapes ESID 923,publisher field site,600,4 moving-tone proposal groups,4,descending_band_visible: clear descending tonal band; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 923,esid-923,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+esid-791,Eclipse Soundscapes field recording,791,https://zenodo.org/records/21911431,2024-04-08 Eclipse Soundscapes ESID 791,publisher field site,600,4 moving-tone proposal groups,4,descending_band_visible_weak_or_interrupted: visible descending tonal energy with interruptions; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 791,esid-791,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no+yt--aolQUqqvXc,YouTube audio excerpt,-aolQUqqvXc,https://www.youtube.com/watch?v=-aolQUqqvXc,Prop Airplane Passing by Overhead with Birds in Background (Version 1),aircraft_targeted,22.0135,proposals,4|4,candidate_doppler_like,exact receiver coordinates unresolved,Title may name a broad area or site; recorder position is not independently established,,recording UTC not established,unconfirmed,no+yt-G0fjcT-SP1o,YouTube audio excerpt,G0fjcT-SP1o,https://www.youtube.com/watch?v=G0fjcT-SP1o,Planes Flying Overhead While the Birds Sing in the Distance,aircraft_targeted,180.0135,proposals,6,candidate_doppler_like,exact receiver coordinates unresolved,Title may name a broad area or site; recorder position is not independently established,,recording UTC not established,unconfirmed,no+yt-MEwEWurF2Ck,YouTube audio excerpt,MEwEWurF2Ck,https://www.youtube.com/watch?v=MEwEWurF2Ck,Small Plane Flies Over My House,aircraft_targeted,15.0135,proposals,2|2,candidate_doppler_like,exact receiver coordinates unresolved,Title may name a broad area or site; recorder position is not independently established,,recording UTC not established,unconfirmed,no+yt-Ri8dQWcn9hc,YouTube audio excerpt,Ri8dQWcn9hc,https://www.youtube.com/watch?v=Ri8dQWcn9hc,LAX Sepulveda In N Out Burger Plane Spotting 엘에이 공항 인앤아웃버거 비행기 출사,site_named_aircraft,180.0135,proposals,13,candidate_doppler_like,exact receiver coordinates unresolved,Title may name a broad area or site; recorder position is not independently established,,recording UTC not established,unconfirmed,no+yt-XWP9tkBS1nk,YouTube audio excerpt,XWP9tkBS1nk,https://www.youtube.com/watch?v=XWP9tkBS1nk,"Low Level Twilight Flyby Of Unusual Aircraft, June 19 2013 Syracuse NY -- THE DRONE ZONE",aircraft_targeted,168.0135,proposals,4,candidate_doppler_like,exact receiver coordinates unresolved,Title may name a broad area or site; recorder position is not independently established,,recording UTC not established,unconfirmed,nodiff --git a/benchmarks/truth-separated/audio-candidate-location-references-20261103.csv b/benchmarks/truth-separated/audio-candidate-location-references-20261103.csvnew file mode 100644index 0000000..685342f--- /dev/null+++ b/benchmarks/truth-separated/audio-candidate-location-references-20261103.csv@@ -0,0 +1,21 @@+candidate_id,source_parent_id,zenodo_record_id,record_url,publisher_metadata_url,site_latitude,site_longitude,coordinate_basis,microphone_coordinate_accuracy,clock_setting,filename_start_utc_claim+esid-706,706,22227872,https://zenodo.org/records/22227872,https://zenodo.org/api/records/22227872/files/total_eclipse_data.csv/content,37.45517,-89.2612,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:16+00:00+esid-688,688,22025528,https://zenodo.org/records/22025528,https://zenodo.org/api/records/22025528/files/total_eclipse_data.csv/content,40.53005,-90.2607,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:10+00:00+esid-656,656,22224306,https://zenodo.org/records/22224306,https://zenodo.org/api/records/22224306/files/total_eclipse_data.csv/content,40.10586,-88.2015,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:28+00:00+esid-485,485,22224082,https://zenodo.org/records/22224082,https://zenodo.org/api/records/22224082/files/total_eclipse_data.csv/content,33.03493,-96.79528,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Needs to be set manually,2024-04-08T04:47:16+00:00+esid-281,281,22214458,https://zenodo.org/records/22214458,https://zenodo.org/api/records/22214458/files/total_eclipse_data.csv/content,41.828606,-70.794038,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T04:25:48+00:00+esid-847,847,22059635,https://zenodo.org/records/22059635,https://zenodo.org/api/records/22059635/files/total_eclipse_data.csv/content,43.0539,-79.13847,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T03:59:26+00:00+esid-696,696,22050432,https://zenodo.org/records/22050432,https://zenodo.org/api/records/22050432/files/total_eclipse_data.csv/content,40.86539,-90.3474,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:55+00:00+esid-103,103,22049202,https://zenodo.org/records/22049202,https://zenodo.org/api/records/22049202/files/total_eclipse_data.csv/content,43.79568,-72.18011,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T04:24:53+00:00+esid-756,756,22030700,https://zenodo.org/records/22030700,https://zenodo.org/api/records/22030700/files/total_eclipse_data.csv/content,32.783939,-96.585562,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T05:56:00+00:00+esid-920,920,22014438,https://zenodo.org/records/22014438,https://zenodo.org/api/records/22014438/files/total_eclipse_data.csv/content,43.18657,-77.75444,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T02:41:49+00:00+esid-962,962,22002491,https://zenodo.org/records/22002491,https://zenodo.org/api/records/22002491/files/total_eclipse_data.csv/content,37.60759,-89.2466,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:23+00:00+esid-665,665,21999916,https://zenodo.org/records/21999916,https://zenodo.org/api/records/21999916/files/total_eclipse_data.csv/content,39.77373,-87.0203,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:51+00:00+esid-762,762,21786378,https://zenodo.org/records/21786378,https://zenodo.org/api/records/21786378/files/total_eclipse_data.csv/content,30.386361,-98.860931,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T04:27:49+00:00+esid-710,710,21722091,https://zenodo.org/records/21722091,https://zenodo.org/api/records/21722091/files/total_eclipse_data.csv/content,40.69459,-89.5904,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:20+00:00+esid-443,443,21755346,https://zenodo.org/records/21755346,https://zenodo.org/api/records/21755346/files/total_eclipse_data.csv/content,33.25035,-97.0949,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T04:53:15+00:00+esid-445,445,21743960,https://zenodo.org/records/21743960,https://zenodo.org/api/records/21743960/files/total_eclipse_data.csv/content,31.3897866,-97.258821,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T03:09:15+00:00+esid-439,439,21804576,https://zenodo.org/records/21804576,https://zenodo.org/api/records/21804576/files/total_eclipse_data.csv/content,32.6841,-95.5414,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T02:09:50+00:00+esid-430,430,21772857,https://zenodo.org/records/21772857,https://zenodo.org/api/records/21772857/files/total_eclipse_data.csv/content,32.6933,-97.43444,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T05:27:00+00:00+esid-923,923,21918793,https://zenodo.org/records/21918793,https://zenodo.org/api/records/21918793/files/total_eclipse_data.csv/content,38.03153,-78.52409,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T08:39:17+00:00+esid-791,791,21911431,https://zenodo.org/records/21911431,https://zenodo.org/api/records/21911431/files/total_eclipse_data.csv/content,39.39719,-96.73761,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T03:16:09+00:00diff --git a/docs/HANDOFF.md b/docs/HANDOFF.mdindex c749d54..48938a1 100644--- a/docs/HANDOFF.md+++ b/docs/HANDOFF.md@@ -1,28 +1,24 @@# Implementation handoff-## Current owner priority: optimization (November 3)--Strengthen the cheap-to-expensive search hierarchy before further large national-searches. Do not copy manually selected speed/distance bands into automatic-discovery, or silently remove jets, single-event hypotheses or missing data.--Two exploratory optimizations were tested and **not landed**, because the existing-batch fit cache changed their performance benefit. On real automatic associations,-early residual rejection reduced an uncached replay from about8.0–8.3s to5.75s,-but a production-cache comparison was slower. Bounded acoustic-sample reuse cut-101,955 retarded-time sample evaluations to4,259 and preserved all70 result rows-exactly, but still made602 rather than609 acoustic calls and did not improve-runtime consistently. Candidate elimination alone is not a measured speedup.-The next concrete experiment is batching the shared-clock sample union before-fitting, reducing function calls as well as sample count. Another available task-is processing larger matched subsets first and pruning below Top-K using the-certified score ceiling of one unit per matched group, while preserving finite-mode bounds. No experimental screening/cache change was applied to production.--The parallel [real-audio collection](REAL_AUDIO_CORPUS.md) has392 source records-and18 completed scans; the requested20-known/5-unknown usable benchmark is still-unmet. Its source table retains failures and negatives. No new corpus recordings-were matched to flights or localized.+## Optimization priority: merged improvements (November 3)++The requested cheap-to-expensive screening work is now on `main`: PR #158 added+the conservative broad-window measured-ridge screen; PR #169 added a fixed-clock+event/flight precheck; PR #170 batches compatible clock fits; PRs #173–174 advance+fit pruning and adaptive tile handling. On one real regional development point,+#158 preserved the same 92 retained rows while reducing full nonlinear fits from+30,361 to 17,202 and process CPU from 48.27 to 30.04 seconds. That is a measured+local improvement, not a nationwide runtime or localization-accuracy result. The+manually estimated narrow speed/range bands from the reference video were not+copied into automatic rejection rules.++The current priority is independent real-world evidence. The+[real-audio collection](REAL_AUDIO_CORPUS.md) now includes a 314-video scanner+survey and a 25-row reviewed candidate index. The requested 20 known-location+usable aircraft recordings plus five unknown-location usable aircraft recordings+is still unmet: publisher site coordinates do not establish microphone accuracy,+and moving-tone review does not confirm aircraft identity. No new corpus+recordings were matched to flights or localized.The [swarm task plan](AGENT_TASKS.md) provides 12 actionable GitHub issues,dependencies, suggested agent allocation and a copyable assignment prompt.diff --git a/docs/REAL_AUDIO_CORPUS.md b/docs/REAL_AUDIO_CORPUS.mdindex 93a97fc..fb7a7e3 100644--- a/docs/REAL_AUDIO_CORPUS.md+++ b/docs/REAL_AUDIO_CORPUS.md@@ -1,70 +1,107 @@-# Online audio collection and scanner survey--The owner's current priority is search optimization. Continue this collection in-parallel without launching geolocation or aircraft matching on its recordings.-The requested benchmark remains **20 location-known usable clips plus 5-location-unresolved usable clips**. It is not complete.--`scripts/collect_audio_corpus.py` searches public YouTube/Freesound pages, obtains-audio-only MP3 excerpts, runs the unchanged intake/discovery scanner, and exports a-single CSV retaining download failures and zero-proposal recordings. It accepts no-flight IDs, locations, frequency bands or event windows as scanner inputs.-Media, full ridges and source descriptions stay local. No source video is committed.--## Current collection--[Source and result table](../benchmarks/online-audio-survey-20261103.csv):--- 392 source records: 287 YouTube videos and 105 Freesound sounds.-- 18 YouTube excerpts scanned, totaling 45.57 minutes: 6 with moving-tone- proposals and 12 with none. These are scanner results, not confirmed aircraft.-- 246 acquisition failures, retained separately from scanner negatives.- Freesound preview URLs returned HTTP 403; later YouTube attempts failed with- connection refused. The remaining 128 sources have not been scanned.-- The initial C-130 pilot excerpt also yielded zero proposals; it is separate- from the table's 18 completed excerpts.--Queries cover targeted aircraft, outdoor walks, birds/nature, indoor recordings,-phone/movement and adverse conditions. The table records the search category;-actual equipment and environment still require source review. This is an English-search-selected sample, not a random or representative sample of YouTube.--Do not count a source as a usable benchmark merely because the scanner found a-tone. An onboard takeoff recording, for example, can contain engine-frequency-changes without representing a fixed observer hearing aircraft Doppler. Sound-effects and edited/looped ambience also require review. A zero result means the-current scanner found no qualifying proposal, not that no aircraft was audible.--Publisher map pins are labeled as such, with unknown microphone accuracy.-Upload time is not recording UTC. A source lacking independently established time-and precise location can still support acoustic tests, but cannot establish-100–1,000 m geolocation accuracy. Unknown-location clips must not be localized-as part of this collection task.--## Resume and coordination--Local corpus root:-`/home/cinna/.local/state/dopplergeist-audio-corpus-20261103`.-It contains `sources.jsonl`, `audio/`, `scans/`, `logs/`, the two search plans,-and `catalog.csv`. Decoded temporary audio is removed after scanning. The full-MP3 excerpts and measured ridges remain available to other local workers.+# Real audio collection and Doppler-candidate review++All new recordings in this collection were processed with the automatic audio+scanner only. No flight matching, aircraft association or geolocation was run on+the collected clips. No coordinates, flight IDs or recording windows were scanner+inputs. Downloaded media, detailed ridge outputs and review plots stay local.++The broad video survey and the smaller candidate index serve different purposes:+the survey preserves positive, negative and failed outcomes; the index collects+the strongest reviewed real-recording leads for follow-up. Neither is an accepted+aircraft-localization benchmark yet.++## Broad online-video survey++The [expanded case table](../benchmarks/youtube-expanded-scan-20261103.csv) and+[summary](../benchmarks/youtube-expanded-scan-20261103.md) contain 314 distinct+YouTube video IDs. The final per-ID outcomes are 273 successful scans, 41+unsuccessful acquisitions, 47 successful scans with moving-tone proposals and+226 successful scans with no proposals. All 47 proposal cases received a quick+single-reviewer spectrogram screen: five were labeled *candidate Doppler-like*,+five looked plausible but showed possible edits or repetition, and 37 remained+ambiguous. These labels do not confirm aircraft identity or continuous recording+time.++Successful-scan proposals by query category were:++| Search category | Scanned | With proposals |+| --- | ---: | ---: |+| Aircraft-targeted | 62 | 18 |+| Outdoor walks | 48 | 10 |+| Site-named aircraft | 26 | 8 |+| Adverse conditions | 39 | 4 |+| Birds and nature | 26 | 3 |+| Indoor | 29 | 3 |+| Phone movement | 43 | 1 |++These categories describe the searches, not verified microphones or recording+conditions. Results are query-selected, not random or representative of YouTube.+Most scans cover the first 180 seconds, or the entire source if shorter; this is+not full-video screening for long sources. The three indoor proposals were all+ambiguous. A no-proposal result applies only to the analyzed excerpt.++The original [392-source survey snapshot](../benchmarks/online-audio-survey-20261103.csv)+is retained unchanged as a historical artifact. The expanded table reconciles it+with retries and the separate site-named search, deduplicates by video ID and+preserves unsuccessful acquisition outcomes and multiple attempts.++## Candidate index++The [25-row candidate index](../benchmarks/audio-candidate-index-20261103.csv)+contains:++- **20 Eclipse Soundscapes site recordings** with a reviewed descending tonal+ band and a publisher-reported site coordinate. Two of the 20 have weak or+ interrupted visual support. The coordinate reference is kept separately in+ [the truth-separated table](../benchmarks/truth-separated/audio-candidate-location-references-20261103.csv).+ The publisher coordinate is for a deployment site; microphone placement error+ is unknown. Filename-derived times are claims, not independently verified UTC.+- **Five YouTube candidates** labeled *candidate Doppler-like* in the visual+ review. Their exact receiver coordinates are unprovided; some titles mention a+ coarse place. Aircraft identity, recording UTC and independence from editing+ are not established.++The Eclipse and YouTube candidates are real recordings and real moving-tone+evidence, but the source of the tonal bands has not been confirmed as aircraft.+The site coordinate is not a surveyed microphone coordinate. These clips are+useful candidate material for further evidence gathering; they cannot yet measure+100–1,000 m localization error. The requested **20 known-location usable aircraft+clips plus five usable unknown-location aircraft clips remains unmet** until+aircraft/source identity and the needed receiver position/time evidence are+independently established.++The candidate exporter uses only the committed scanner/review manifests and local+publisher metadata. It creates the location table separately:++```bash+python scripts/export_audio_candidate_index.py \+ --truth-root /home/cinna/.local/state/dopplergeist-eclipse-corpus-20261103 \+ --index benchmarks/audio-candidate-index-20261103.csv \+ --truth benchmarks/truth-separated/audio-candidate-location-references-20261103.csv+```++Never pass the truth-separated coordinate table into the scanner or use it to+narrow an evaluation search. The exporter does not download or rescan audio.++## Acquisition and scanner reproduction++The broad survey used the predeclared query plan+[`online-audio-search-plan-20261103.json`](../benchmarks/online-audio-search-plan-20261103.json)+and [`collect_audio_corpus.py`](../scripts/collect_audio_corpus.py). Collection+clips use the first available 180 seconds, or the full shorter source; sample+zero and upload time are not recording UTC. The collector's scanner settings are+recorded in the saved scan JSON under the local corpus roots. No source video or+audio is committed.++The expanded table can be regenerated without download or rescanning when its+six local collection roots remain available:```bash-PYTHONPATH=src python scripts/collect_audio_corpus.py search \- --root /path/to/corpus --plan benchmarks/online-audio-search-plan-20261103.json-PYTHONPATH=src python scripts/collect_audio_corpus.py run \- --root /path/to/corpus --workers 2 --seconds 180-PYTHONPATH=src python scripts/collect_audio_corpus.py export --root /path/to/corpus+python scripts/export_youtube_survey_20261103.py \+ --state-root /home/cinna/.local/state \+ --output /tmp/youtube-expanded-scan-20261103.csv```-Install `requests`, the project's scanner dependencies, FFmpeg and yt-dlp.-Completed/failed sources are skipped on subsequent runs; use `--retry` to retry-failures. `--category` selects a search category. Optional `reviews.json` maps-source IDs to reviewed fields/notes, which override catalog fields on export.-Clips use the first available 180 seconds, or the complete shorter source; a live-source's relative start is not established recording UTC.--Coordinate in [Discussion 39](https://github.com/1337isnot1337/DOPPLERGEIST/discussions/39):-W1 owns the broader YouTube survey, W5 distinct known-site YouTube clips, and W4-the separate AeroSonicDB sample. Check video IDs before downloading duplicates.-Keep externally sourced media local unless redistribution permission is clear.+Source rights and continued online availability are not established by this+survey. Retain failed downloads, ambiguous proposals and no-proposal excerpts in+any future update.diff --git a/scripts/export_audio_candidate_index.py b/scripts/export_audio_candidate_index.pynew file mode 100644index 0000000..2866c2d--- /dev/null+++ b/scripts/export_audio_candidate_index.py@@ -0,0 +1,158 @@+"""Export reviewed real-audio candidates with site coordinates kept separate."""+import argparse+import csv+import html+import json+import re+from pathlib import Path+++def read_csv(path):+ with path.open(newline="") as stream:+ return list(csv.DictReader(stream))+++def write_csv(path, fields, rows):+ path.parent.mkdir(parents=True, exist_ok=True)+ with path.open("w", newline="") as stream:+ writer = csv.DictWriter(stream, fieldnames=fields, lineterminator="\n")+ writer.writeheader()+ writer.writerows(rows)+++def coordinates(site, record_id, truth_root):+ files = sorted(truth_root.glob(f"*/truth/{site}.json"))+ for path in files:+ record = json.loads(path.read_text())+ if str(record.get("id")) != str(record_id):+ continue+ description = html.unescape(re.sub(+ r"<[^>]+>", " ", record["metadata"]["description"]+ ))+ latitude = re.search(r"Latitude\s*:\s*(-?\d+(?:\.\d+)?)", description, re.I)+ longitude = re.search(r"Longitude\s*:\s*(-?\d+(?:\.\d+)?)", description, re.I)+ if latitude and longitude:+ return latitude.group(1), longitude.group(1)+ raise ValueError(f"No matching publisher coordinate metadata for ESID {site}")+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--truth-root", type=Path, required=True,+ help="Local Eclipse collection root containing batch*/truth/*.json")+ parser.add_argument("--index", type=Path, required=True)+ parser.add_argument("--truth", type=Path, required=True)+ args = parser.parse_args()++ candidate_fields = [+ "candidate_id", "source_type", "source_parent_id", "source_url", "title",+ "search_category", "clip_duration_s", "scanner_result", "proposal_count",+ "visual_review", "recording_location_class", "coarse_location_hint",+ "reference_coordinate_key", "recording_clock_status", "aircraft_identity",+ "location_evaluation_eligible",+ ]+ truth_fields = [+ "candidate_id", "source_parent_id", "zenodo_record_id", "record_url",+ "publisher_metadata_url", "site_latitude", "site_longitude",+ "coordinate_basis", "microphone_coordinate_accuracy", "clock_setting",+ "filename_start_utc_claim",+ ]++ site_candidates = []+ for name in ("eclipse-field-screen-20261103.json",+ "eclipse-field-screen-batch2-20261103.json"):+ report = json.loads((Path("benchmarks") / name).read_text())+ for result in report["results"]:+ note = result.get("visual_review") or ""+ if (result.get("status") != "scanned"+ or int(result.get("proposal_groups") or 0) == 0+ or "not visually convincing" in note):+ continue+ site_candidates.append((name, result))++ videos = {+ row["id"]: row for row in read_csv(Path("benchmarks/youtube-expanded-scan-20261103.csv"))+ }+ unknown_candidates = [+ row for row in videos.values()+ if "candidate_doppler_like" in row["visual_reviews"].split("|")+ ]+ if len(site_candidates) != 20 or len(unknown_candidates) != 5:+ raise ValueError(f"Expected 20 site-pattern and 5 unresolved-coordinate video candidates; "+ f"found {len(site_candidates)} and {len(unknown_candidates)}")++ index_rows = []+ truth_rows = []+ for batch, result in site_candidates:+ site = result["site"]+ record_id = result["record_url"].rstrip("/").split("/")[-1]+ candidate_id = f"esid-{site}"+ pattern = "descending_band_visible"+ if any(word in (result.get("visual_review") or "").lower()+ for word in ("weak", "fragmented", "interruption")):+ pattern = "descending_band_visible_weak_or_interrupted"+ index_rows.append({+ "candidate_id": candidate_id,+ "source_type": "Eclipse Soundscapes field recording",+ "source_parent_id": site,+ "source_url": result["record_url"],+ "title": f"2024-04-08 Eclipse Soundscapes ESID {site}",+ "search_category": "publisher field site",+ "clip_duration_s": result.get("clip_duration_s", ""),+ "scanner_result": f"{result.get('proposal_groups', 0)} moving-tone proposal groups",+ "proposal_count": result.get("proposal_groups", 0),+ "visual_review": f"{pattern}: {result.get('visual_review', '')}",+ "recording_location_class": "publisher_site_coordinate; microphone accuracy unknown",+ "coarse_location_hint": f"Eclipse Soundscapes site {site}",+ "reference_coordinate_key": candidate_id,+ "recording_clock_status": (+ "filename timestamp unverified; " ++ str(result.get("publisher_clock_setting", "publisher clock setting unreported"))+ ),+ "aircraft_identity": "unconfirmed",+ "location_evaluation_eligible": "no",+ })+ latitude, longitude = coordinates(site, record_id, args.truth_root)+ truth_rows.append({+ "candidate_id": candidate_id,+ "source_parent_id": site,+ "zenodo_record_id": record_id,+ "record_url": result["record_url"],+ "publisher_metadata_url": result.get("publisher_site_metadata_url", ""),+ "site_latitude": latitude,+ "site_longitude": longitude,+ "coordinate_basis": "Publisher-reported deployment site coordinates",+ "microphone_coordinate_accuracy": "Unknown; not an independently surveyed microphone position",+ "clock_setting": result.get("publisher_clock_setting", ""),+ "filename_start_utc_claim": result.get("filename_timestamp_utc_unverified", ""),+ })++ for row in sorted(unknown_candidates, key=lambda item: item["id"]):+ durations = [float(v) for v in row["durations_s"].split("|") if v]+ index_rows.append({+ "candidate_id": f"yt-{row['id']}",+ "source_type": "YouTube audio excerpt",+ "source_parent_id": row["id"],+ "source_url": row["url"],+ "title": row["title"],+ "search_category": row["category"],+ "clip_duration_s": max(durations, default=""),+ "scanner_result": row["scanner_result"],+ "proposal_count": row["proposal_counts"],+ "visual_review": row["visual_reviews"],+ "recording_location_class": "exact receiver coordinates unresolved",+ "coarse_location_hint": "Title may name a broad area or site; recorder position is not independently established",+ "reference_coordinate_key": "",+ "recording_clock_status": "recording UTC not established",+ "aircraft_identity": "unconfirmed",+ "location_evaluation_eligible": "no",+ })++ write_csv(args.index, candidate_fields, index_rows)+ write_csv(args.truth, truth_fields, truth_rows)+ print(f"candidate_index={len(index_rows)} site_coordinates={len(truth_rows)} "+ f"unresolved_receiver_coordinate_videos={len(unknown_candidates)}")+++if __name__ == "__main__":+ main()