Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

docs: assemble reviewed real audio candidates (#146)

Cinna Davis396c3c159a5c
diff --git a/benchmarks/audio-candidate-index-20261103.csv b/benchmarks/audio-candidate-index-20261103.csv
new file mode 100644
index 0000000..55a7d81
--- /dev/null
+++ b/benchmarks/audio-candidate-index-20261103.csv
@@ -0,0 +1,26 @@
+candidate_id,source_type,source_parent_id,source_url,title,search_category,clip_duration_s,scanner_result,proposal_count,visual_review,recording_location_class,coarse_location_hint,reference_coordinate_key,recording_clock_status,aircraft_identity,location_evaluation_eligible
+esid-706,Eclipse Soundscapes field recording,706,https://zenodo.org/records/22227872,2024-04-08 Eclipse Soundscapes ESID 706,publisher field site,600,5 moving-tone proposal groups,5,descending_band_visible: descending moving-tone proposal visible; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 706,esid-706,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-688,Eclipse Soundscapes field recording,688,https://zenodo.org/records/22025528,2024-04-08 Eclipse Soundscapes ESID 688,publisher field site,600,15 moving-tone proposal groups,15,descending_band_visible: descending tonal ridge visible; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 688,esid-688,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-656,Eclipse Soundscapes field recording,656,https://zenodo.org/records/22224306,2024-04-08 Eclipse Soundscapes ESID 656,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: several visible parallel descending tonal bands; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 656,esid-656,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-485,Eclipse Soundscapes field recording,485,https://zenodo.org/records/22224082,2024-04-08 Eclipse Soundscapes ESID 485,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: visible descending tonal ridge with an abrupt segment; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 485,esid-485,filename timestamp unverified; Needs to be set manually,unconfirmed,no
+esid-281,Eclipse Soundscapes field recording,281,https://zenodo.org/records/22214458,2024-04-08 Eclipse Soundscapes ESID 281,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: visible descending tonal ridge; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 281,esid-281,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-847,Eclipse Soundscapes field recording,847,https://zenodo.org/records/22059635,2024-04-08 Eclipse Soundscapes ESID 847,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: clear descending tonal ridge; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 847,esid-847,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-696,Eclipse Soundscapes field recording,696,https://zenodo.org/records/22050432,2024-04-08 Eclipse Soundscapes ESID 696,publisher field site,600,2 moving-tone proposal groups,2,descending_band_visible: multiple visible descending tonal bands; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 696,esid-696,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-103,Eclipse Soundscapes field recording,103,https://zenodo.org/records/22049202,2024-04-08 Eclipse Soundscapes ESID 103,publisher field site,600,2 moving-tone proposal groups,2,descending_band_visible: clear descending tonal ridge; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 103,esid-103,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-756,Eclipse Soundscapes field recording,756,https://zenodo.org/records/22030700,2024-04-08 Eclipse Soundscapes ESID 756,publisher field site,600,6 moving-tone proposal groups,6,descending_band_visible: two clear descending tonal bands; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 756,esid-756,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-920,Eclipse Soundscapes field recording,920,https://zenodo.org/records/22014438,2024-04-08 Eclipse Soundscapes ESID 920,publisher field site,600,6 moving-tone proposal groups,6,descending_band_visible: two clear descending tonal bands; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 920,esid-920,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-962,Eclipse Soundscapes field recording,962,https://zenodo.org/records/22002491,2024-04-08 Eclipse Soundscapes ESID 962,publisher field site,600,2 moving-tone proposal groups,2,descending_band_visible: visible shallow descending tonal band; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 962,esid-962,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-665,Eclipse Soundscapes field recording,665,https://zenodo.org/records/21999916,2024-04-08 Eclipse Soundscapes ESID 665,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: visible descending band with a sharper transition; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 665,esid-665,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-762,Eclipse Soundscapes field recording,762,https://zenodo.org/records/21786378,2024-04-08 Eclipse Soundscapes ESID 762,publisher field site,600,4 moving-tone proposal groups,4,descending_band_visible: clear descending low-frequency band; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 762,esid-762,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-710,Eclipse Soundscapes field recording,710,https://zenodo.org/records/21722091,2024-04-08 Eclipse Soundscapes ESID 710,publisher field site,600,2 moving-tone proposal groups,2,descending_band_visible_weak_or_interrupted: fragmented and weak descending energy; visual support is limited; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 710,esid-710,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-443,Eclipse Soundscapes field recording,443,https://zenodo.org/records/21755346,2024-04-08 Eclipse Soundscapes ESID 443,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: visible low-frequency descending band; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 443,esid-443,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-445,Eclipse Soundscapes field recording,445,https://zenodo.org/records/21743960,2024-04-08 Eclipse Soundscapes ESID 445,publisher field site,600,1 moving-tone proposal groups,1,descending_band_visible: clear descending low-frequency band; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 445,esid-445,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-439,Eclipse Soundscapes field recording,439,https://zenodo.org/records/21804576,2024-04-08 Eclipse Soundscapes ESID 439,publisher field site,600,6 moving-tone proposal groups,6,descending_band_visible: two visible descending bands with a later upward segment; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 439,esid-439,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-430,Eclipse Soundscapes field recording,430,https://zenodo.org/records/21772857,2024-04-08 Eclipse Soundscapes ESID 430,publisher field site,600,5 moving-tone proposal groups,5,descending_band_visible: clear descending tonal band; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 430,esid-430,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-923,Eclipse Soundscapes field recording,923,https://zenodo.org/records/21918793,2024-04-08 Eclipse Soundscapes ESID 923,publisher field site,600,4 moving-tone proposal groups,4,descending_band_visible: clear descending tonal band; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 923,esid-923,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+esid-791,Eclipse Soundscapes field recording,791,https://zenodo.org/records/21911431,2024-04-08 Eclipse Soundscapes ESID 791,publisher field site,600,4 moving-tone proposal groups,4,descending_band_visible_weak_or_interrupted: visible descending tonal energy with interruptions; aircraft identity unconfirmed,publisher_site_coordinate; microphone accuracy unknown,Eclipse Soundscapes site 791,esid-791,filename timestamp unverified; Set with Automated AudioMoth Time Chime,unconfirmed,no
+yt--aolQUqqvXc,YouTube audio excerpt,-aolQUqqvXc,https://www.youtube.com/watch?v=-aolQUqqvXc,Prop Airplane Passing by Overhead with Birds in Background (Version 1),aircraft_targeted,22.0135,proposals,4|4,candidate_doppler_like,exact receiver coordinates unresolved,Title may name a broad area or site; recorder position is not independently established,,recording UTC not established,unconfirmed,no
+yt-G0fjcT-SP1o,YouTube audio excerpt,G0fjcT-SP1o,https://www.youtube.com/watch?v=G0fjcT-SP1o,Planes Flying Overhead While the Birds Sing in the Distance,aircraft_targeted,180.0135,proposals,6,candidate_doppler_like,exact receiver coordinates unresolved,Title may name a broad area or site; recorder position is not independently established,,recording UTC not established,unconfirmed,no
+yt-MEwEWurF2Ck,YouTube audio excerpt,MEwEWurF2Ck,https://www.youtube.com/watch?v=MEwEWurF2Ck,Small Plane Flies Over My House,aircraft_targeted,15.0135,proposals,2|2,candidate_doppler_like,exact receiver coordinates unresolved,Title may name a broad area or site; recorder position is not independently established,,recording UTC not established,unconfirmed,no
+yt-Ri8dQWcn9hc,YouTube audio excerpt,Ri8dQWcn9hc,https://www.youtube.com/watch?v=Ri8dQWcn9hc,LAX Sepulveda In N Out Burger Plane Spotting 엘에이 공항 인앤아웃버거 비행기 출사,site_named_aircraft,180.0135,proposals,13,candidate_doppler_like,exact receiver coordinates unresolved,Title may name a broad area or site; recorder position is not independently established,,recording UTC not established,unconfirmed,no
+yt-XWP9tkBS1nk,YouTube audio excerpt,XWP9tkBS1nk,https://www.youtube.com/watch?v=XWP9tkBS1nk,"Low Level Twilight Flyby Of Unusual Aircraft, June 19 2013 Syracuse NY -- THE DRONE ZONE",aircraft_targeted,168.0135,proposals,4,candidate_doppler_like,exact receiver coordinates unresolved,Title may name a broad area or site; recorder position is not independently established,,recording UTC not established,unconfirmed,no
diff --git a/benchmarks/truth-separated/audio-candidate-location-references-20261103.csv b/benchmarks/truth-separated/audio-candidate-location-references-20261103.csv
new file mode 100644
index 0000000..685342f
--- /dev/null
+++ b/benchmarks/truth-separated/audio-candidate-location-references-20261103.csv
@@ -0,0 +1,21 @@
+candidate_id,source_parent_id,zenodo_record_id,record_url,publisher_metadata_url,site_latitude,site_longitude,coordinate_basis,microphone_coordinate_accuracy,clock_setting,filename_start_utc_claim
+esid-706,706,22227872,https://zenodo.org/records/22227872,https://zenodo.org/api/records/22227872/files/total_eclipse_data.csv/content,37.45517,-89.2612,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:16+00:00
+esid-688,688,22025528,https://zenodo.org/records/22025528,https://zenodo.org/api/records/22025528/files/total_eclipse_data.csv/content,40.53005,-90.2607,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:10+00:00
+esid-656,656,22224306,https://zenodo.org/records/22224306,https://zenodo.org/api/records/22224306/files/total_eclipse_data.csv/content,40.10586,-88.2015,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:28+00:00
+esid-485,485,22224082,https://zenodo.org/records/22224082,https://zenodo.org/api/records/22224082/files/total_eclipse_data.csv/content,33.03493,-96.79528,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Needs to be set manually,2024-04-08T04:47:16+00:00
+esid-281,281,22214458,https://zenodo.org/records/22214458,https://zenodo.org/api/records/22214458/files/total_eclipse_data.csv/content,41.828606,-70.794038,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T04:25:48+00:00
+esid-847,847,22059635,https://zenodo.org/records/22059635,https://zenodo.org/api/records/22059635/files/total_eclipse_data.csv/content,43.0539,-79.13847,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T03:59:26+00:00
+esid-696,696,22050432,https://zenodo.org/records/22050432,https://zenodo.org/api/records/22050432/files/total_eclipse_data.csv/content,40.86539,-90.3474,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:55+00:00
+esid-103,103,22049202,https://zenodo.org/records/22049202,https://zenodo.org/api/records/22049202/files/total_eclipse_data.csv/content,43.79568,-72.18011,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T04:24:53+00:00
+esid-756,756,22030700,https://zenodo.org/records/22030700,https://zenodo.org/api/records/22030700/files/total_eclipse_data.csv/content,32.783939,-96.585562,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T05:56:00+00:00
+esid-920,920,22014438,https://zenodo.org/records/22014438,https://zenodo.org/api/records/22014438/files/total_eclipse_data.csv/content,43.18657,-77.75444,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T02:41:49+00:00
+esid-962,962,22002491,https://zenodo.org/records/22002491,https://zenodo.org/api/records/22002491/files/total_eclipse_data.csv/content,37.60759,-89.2466,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:23+00:00
+esid-665,665,21999916,https://zenodo.org/records/21999916,https://zenodo.org/api/records/21999916/files/total_eclipse_data.csv/content,39.77373,-87.0203,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:51+00:00
+esid-762,762,21786378,https://zenodo.org/records/21786378,https://zenodo.org/api/records/21786378/files/total_eclipse_data.csv/content,30.386361,-98.860931,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T04:27:49+00:00
+esid-710,710,21722091,https://zenodo.org/records/21722091,https://zenodo.org/api/records/21722091/files/total_eclipse_data.csv/content,40.69459,-89.5904,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T00:00:20+00:00
+esid-443,443,21755346,https://zenodo.org/records/21755346,https://zenodo.org/api/records/21755346/files/total_eclipse_data.csv/content,33.25035,-97.0949,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T04:53:15+00:00
+esid-445,445,21743960,https://zenodo.org/records/21743960,https://zenodo.org/api/records/21743960/files/total_eclipse_data.csv/content,31.3897866,-97.258821,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T03:09:15+00:00
+esid-439,439,21804576,https://zenodo.org/records/21804576,https://zenodo.org/api/records/21804576/files/total_eclipse_data.csv/content,32.6841,-95.5414,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T02:09:50+00:00
+esid-430,430,21772857,https://zenodo.org/records/21772857,https://zenodo.org/api/records/21772857/files/total_eclipse_data.csv/content,32.6933,-97.43444,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T05:27:00+00:00
+esid-923,923,21918793,https://zenodo.org/records/21918793,https://zenodo.org/api/records/21918793/files/total_eclipse_data.csv/content,38.03153,-78.52409,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T08:39:17+00:00
+esid-791,791,21911431,https://zenodo.org/records/21911431,https://zenodo.org/api/records/21911431/files/total_eclipse_data.csv/content,39.39719,-96.73761,Publisher-reported deployment site coordinates,Unknown; not an independently surveyed microphone position,Set with Automated AudioMoth Time Chime,2024-04-08T03:16:09+00:00
diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md
index c749d54..48938a1 100644
--- a/docs/HANDOFF.md
+++ b/docs/HANDOFF.md
@@ -1,28 +1,24 @@
# Implementation handoff
-## Current owner priority: optimization (November 3)
-
-Strengthen the cheap-to-expensive search hierarchy before further large national
-searches. Do not copy manually selected speed/distance bands into automatic
-discovery, or silently remove jets, single-event hypotheses or missing data.
-
-Two exploratory optimizations were tested and **not landed**, because the existing
-batch fit cache changed their performance benefit. On real automatic associations,
-early residual rejection reduced an uncached replay from about8.0–8.3s to5.75s,
-but a production-cache comparison was slower. Bounded acoustic-sample reuse cut
-101,955 retarded-time sample evaluations to4,259 and preserved all70 result rows
-exactly, but still made602 rather than609 acoustic calls and did not improve
-runtime consistently. Candidate elimination alone is not a measured speedup.
-The next concrete experiment is batching the shared-clock sample union before
-fitting, reducing function calls as well as sample count. Another available task
-is processing larger matched subsets first and pruning below Top-K using the
-certified score ceiling of one unit per matched group, while preserving finite
-mode bounds. No experimental screening/cache change was applied to production.
-
-The parallel [real-audio collection](REAL_AUDIO_CORPUS.md) has392 source records
-and18 completed scans; the requested20-known/5-unknown usable benchmark is still
-unmet. Its source table retains failures and negatives. No new corpus recordings
-were matched to flights or localized.
+## Optimization priority: merged improvements (November 3)
+
+The requested cheap-to-expensive screening work is now on `main`: PR #158 added
+the conservative broad-window measured-ridge screen; PR #169 added a fixed-clock
+event/flight precheck; PR #170 batches compatible clock fits; PRs #173–174 advance
+fit pruning and adaptive tile handling. On one real regional development point,
+#158 preserved the same 92 retained rows while reducing full nonlinear fits from
+30,361 to 17,202 and process CPU from 48.27 to 30.04 seconds. That is a measured
+local improvement, not a nationwide runtime or localization-accuracy result. The
+manually estimated narrow speed/range bands from the reference video were not
+copied into automatic rejection rules.
+
+The current priority is independent real-world evidence. The
+[real-audio collection](REAL_AUDIO_CORPUS.md) now includes a 314-video scanner
+survey and a 25-row reviewed candidate index. The requested 20 known-location
+usable aircraft recordings plus five unknown-location usable aircraft recordings
+is still unmet: publisher site coordinates do not establish microphone accuracy,
+and moving-tone review does not confirm aircraft identity. No new corpus
+recordings were matched to flights or localized.
The [swarm task plan](AGENT_TASKS.md) provides 12 actionable GitHub issues,
dependencies, suggested agent allocation and a copyable assignment prompt.
diff --git a/docs/REAL_AUDIO_CORPUS.md b/docs/REAL_AUDIO_CORPUS.md
index 93a97fc..fb7a7e3 100644
--- a/docs/REAL_AUDIO_CORPUS.md
+++ b/docs/REAL_AUDIO_CORPUS.md
@@ -1,70 +1,107 @@
-# Online audio collection and scanner survey
-
-The owner's current priority is search optimization. Continue this collection in
-parallel without launching geolocation or aircraft matching on its recordings.
-The requested benchmark remains **20 location-known usable clips plus 5
-location-unresolved usable clips**. It is not complete.
-
-`scripts/collect_audio_corpus.py` searches public YouTube/Freesound pages, obtains
-audio-only MP3 excerpts, runs the unchanged intake/discovery scanner, and exports a
-single CSV retaining download failures and zero-proposal recordings. It accepts no
-flight IDs, locations, frequency bands or event windows as scanner inputs.
-Media, full ridges and source descriptions stay local. No source video is committed.
-
-## Current collection
-
-[Source and result table](../benchmarks/online-audio-survey-20261103.csv):
-
-- 392 source records: 287 YouTube videos and 105 Freesound sounds.
-- 18 YouTube excerpts scanned, totaling 45.57 minutes: 6 with moving-tone
- proposals and 12 with none. These are scanner results, not confirmed aircraft.
-- 246 acquisition failures, retained separately from scanner negatives.
- Freesound preview URLs returned HTTP 403; later YouTube attempts failed with
- connection refused. The remaining 128 sources have not been scanned.
-- The initial C-130 pilot excerpt also yielded zero proposals; it is separate
- from the table's 18 completed excerpts.
-
-Queries cover targeted aircraft, outdoor walks, birds/nature, indoor recordings,
-phone/movement and adverse conditions. The table records the search category;
-actual equipment and environment still require source review. This is an English
-search-selected sample, not a random or representative sample of YouTube.
-
-Do not count a source as a usable benchmark merely because the scanner found a
-tone. An onboard takeoff recording, for example, can contain engine-frequency
-changes without representing a fixed observer hearing aircraft Doppler. Sound
-effects and edited/looped ambience also require review. A zero result means the
-current scanner found no qualifying proposal, not that no aircraft was audible.
-
-Publisher map pins are labeled as such, with unknown microphone accuracy.
-Upload time is not recording UTC. A source lacking independently established time
-and precise location can still support acoustic tests, but cannot establish
-100–1,000 m geolocation accuracy. Unknown-location clips must not be localized
-as part of this collection task.
-
-## Resume and coordination
-
-Local corpus root:
-`/home/cinna/.local/state/dopplergeist-audio-corpus-20261103`.
-It contains `sources.jsonl`, `audio/`, `scans/`, `logs/`, the two search plans,
-and `catalog.csv`. Decoded temporary audio is removed after scanning. The full
-MP3 excerpts and measured ridges remain available to other local workers.
+# Real audio collection and Doppler-candidate review
+
+All new recordings in this collection were processed with the automatic audio
+scanner only. No flight matching, aircraft association or geolocation was run on
+the collected clips. No coordinates, flight IDs or recording windows were scanner
+inputs. Downloaded media, detailed ridge outputs and review plots stay local.
+
+The broad video survey and the smaller candidate index serve different purposes:
+the survey preserves positive, negative and failed outcomes; the index collects
+the strongest reviewed real-recording leads for follow-up. Neither is an accepted
+aircraft-localization benchmark yet.
+
+## Broad online-video survey
+
+The [expanded case table](../benchmarks/youtube-expanded-scan-20261103.csv) and
+[summary](../benchmarks/youtube-expanded-scan-20261103.md) contain 314 distinct
+YouTube video IDs. The final per-ID outcomes are 273 successful scans, 41
+unsuccessful acquisitions, 47 successful scans with moving-tone proposals and
+226 successful scans with no proposals. All 47 proposal cases received a quick
+single-reviewer spectrogram screen: five were labeled *candidate Doppler-like*,
+five looked plausible but showed possible edits or repetition, and 37 remained
+ambiguous. These labels do not confirm aircraft identity or continuous recording
+time.
+
+Successful-scan proposals by query category were:
+
+| Search category | Scanned | With proposals |
+| --- | ---: | ---: |
+| Aircraft-targeted | 62 | 18 |
+| Outdoor walks | 48 | 10 |
+| Site-named aircraft | 26 | 8 |
+| Adverse conditions | 39 | 4 |
+| Birds and nature | 26 | 3 |
+| Indoor | 29 | 3 |
+| Phone movement | 43 | 1 |
+
+These categories describe the searches, not verified microphones or recording
+conditions. Results are query-selected, not random or representative of YouTube.
+Most scans cover the first 180 seconds, or the entire source if shorter; this is
+not full-video screening for long sources. The three indoor proposals were all
+ambiguous. A no-proposal result applies only to the analyzed excerpt.
+
+The original [392-source survey snapshot](../benchmarks/online-audio-survey-20261103.csv)
+is retained unchanged as a historical artifact. The expanded table reconciles it
+with retries and the separate site-named search, deduplicates by video ID and
+preserves unsuccessful acquisition outcomes and multiple attempts.
+
+## Candidate index
+
+The [25-row candidate index](../benchmarks/audio-candidate-index-20261103.csv)
+contains:
+
+- **20 Eclipse Soundscapes site recordings** with a reviewed descending tonal
+ band and a publisher-reported site coordinate. Two of the 20 have weak or
+ interrupted visual support. The coordinate reference is kept separately in
+ [the truth-separated table](../benchmarks/truth-separated/audio-candidate-location-references-20261103.csv).
+ The publisher coordinate is for a deployment site; microphone placement error
+ is unknown. Filename-derived times are claims, not independently verified UTC.
+- **Five YouTube candidates** labeled *candidate Doppler-like* in the visual
+ review. Their exact receiver coordinates are unprovided; some titles mention a
+ coarse place. Aircraft identity, recording UTC and independence from editing
+ are not established.
+
+The Eclipse and YouTube candidates are real recordings and real moving-tone
+evidence, but the source of the tonal bands has not been confirmed as aircraft.
+The site coordinate is not a surveyed microphone coordinate. These clips are
+useful candidate material for further evidence gathering; they cannot yet measure
+100–1,000 m localization error. The requested **20 known-location usable aircraft
+clips plus five usable unknown-location aircraft clips remains unmet** until
+aircraft/source identity and the needed receiver position/time evidence are
+independently established.
+
+The candidate exporter uses only the committed scanner/review manifests and local
+publisher metadata. It creates the location table separately:
+
+```bash
+python scripts/export_audio_candidate_index.py \
+ --truth-root /home/cinna/.local/state/dopplergeist-eclipse-corpus-20261103 \
+ --index benchmarks/audio-candidate-index-20261103.csv \
+ --truth benchmarks/truth-separated/audio-candidate-location-references-20261103.csv
+```
+
+Never pass the truth-separated coordinate table into the scanner or use it to
+narrow an evaluation search. The exporter does not download or rescan audio.
+
+## Acquisition and scanner reproduction
+
+The broad survey used the predeclared query plan
+[`online-audio-search-plan-20261103.json`](../benchmarks/online-audio-search-plan-20261103.json)
+and [`collect_audio_corpus.py`](../scripts/collect_audio_corpus.py). Collection
+clips use the first available 180 seconds, or the full shorter source; sample
+zero and upload time are not recording UTC. The collector's scanner settings are
+recorded in the saved scan JSON under the local corpus roots. No source video or
+audio is committed.
+
+The expanded table can be regenerated without download or rescanning when its
+six local collection roots remain available:
```bash
-PYTHONPATH=src python scripts/collect_audio_corpus.py search \
- --root /path/to/corpus --plan benchmarks/online-audio-search-plan-20261103.json
-PYTHONPATH=src python scripts/collect_audio_corpus.py run \
- --root /path/to/corpus --workers 2 --seconds 180
-PYTHONPATH=src python scripts/collect_audio_corpus.py export --root /path/to/corpus
+python scripts/export_youtube_survey_20261103.py \
+ --state-root /home/cinna/.local/state \
+ --output /tmp/youtube-expanded-scan-20261103.csv
```
-Install `requests`, the project's scanner dependencies, FFmpeg and yt-dlp.
-Completed/failed sources are skipped on subsequent runs; use `--retry` to retry
-failures. `--category` selects a search category. Optional `reviews.json` maps
-source IDs to reviewed fields/notes, which override catalog fields on export.
-Clips use the first available 180 seconds, or the complete shorter source; a live
-source's relative start is not established recording UTC.
-
-Coordinate in [Discussion 39](https://github.com/1337isnot1337/DOPPLERGEIST/discussions/39):
-W1 owns the broader YouTube survey, W5 distinct known-site YouTube clips, and W4
-the separate AeroSonicDB sample. Check video IDs before downloading duplicates.
-Keep externally sourced media local unless redistribution permission is clear.
+Source rights and continued online availability are not established by this
+survey. Retain failed downloads, ambiguous proposals and no-proposal excerpts in
+any future update.
diff --git a/scripts/export_audio_candidate_index.py b/scripts/export_audio_candidate_index.py
new file mode 100644
index 0000000..2866c2d
--- /dev/null
+++ b/scripts/export_audio_candidate_index.py
@@ -0,0 +1,158 @@
+"""Export reviewed real-audio candidates with site coordinates kept separate."""
+import argparse
+import csv
+import html
+import json
+import re
+from pathlib import Path
+
+
+def read_csv(path):
+ with path.open(newline="") as stream:
+ return list(csv.DictReader(stream))
+
+
+def write_csv(path, fields, rows):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with path.open("w", newline="") as stream:
+ writer = csv.DictWriter(stream, fieldnames=fields, lineterminator="\n")
+ writer.writeheader()
+ writer.writerows(rows)
+
+
+def coordinates(site, record_id, truth_root):
+ files = sorted(truth_root.glob(f"*/truth/{site}.json"))
+ for path in files:
+ record = json.loads(path.read_text())
+ if str(record.get("id")) != str(record_id):
+ continue
+ description = html.unescape(re.sub(
+ r"<[^>]+>", " ", record["metadata"]["description"]
+ ))
+ latitude = re.search(r"Latitude\s*:\s*(-?\d+(?:\.\d+)?)", description, re.I)
+ longitude = re.search(r"Longitude\s*:\s*(-?\d+(?:\.\d+)?)", description, re.I)
+ if latitude and longitude:
+ return latitude.group(1), longitude.group(1)
+ raise ValueError(f"No matching publisher coordinate metadata for ESID {site}")
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--truth-root", type=Path, required=True,
+ help="Local Eclipse collection root containing batch*/truth/*.json")
+ parser.add_argument("--index", type=Path, required=True)
+ parser.add_argument("--truth", type=Path, required=True)
+ args = parser.parse_args()
+
+ candidate_fields = [
+ "candidate_id", "source_type", "source_parent_id", "source_url", "title",
+ "search_category", "clip_duration_s", "scanner_result", "proposal_count",
+ "visual_review", "recording_location_class", "coarse_location_hint",
+ "reference_coordinate_key", "recording_clock_status", "aircraft_identity",
+ "location_evaluation_eligible",
+ ]
+ truth_fields = [
+ "candidate_id", "source_parent_id", "zenodo_record_id", "record_url",
+ "publisher_metadata_url", "site_latitude", "site_longitude",
+ "coordinate_basis", "microphone_coordinate_accuracy", "clock_setting",
+ "filename_start_utc_claim",
+ ]
+
+ site_candidates = []
+ for name in ("eclipse-field-screen-20261103.json",
+ "eclipse-field-screen-batch2-20261103.json"):
+ report = json.loads((Path("benchmarks") / name).read_text())
+ for result in report["results"]:
+ note = result.get("visual_review") or ""
+ if (result.get("status") != "scanned"
+ or int(result.get("proposal_groups") or 0) == 0
+ or "not visually convincing" in note):
+ continue
+ site_candidates.append((name, result))
+
+ videos = {
+ row["id"]: row for row in read_csv(Path("benchmarks/youtube-expanded-scan-20261103.csv"))
+ }
+ unknown_candidates = [
+ row for row in videos.values()
+ if "candidate_doppler_like" in row["visual_reviews"].split("|")
+ ]
+ if len(site_candidates) != 20 or len(unknown_candidates) != 5:
+ raise ValueError(f"Expected 20 site-pattern and 5 unresolved-coordinate video candidates; "
+ f"found {len(site_candidates)} and {len(unknown_candidates)}")
+
+ index_rows = []
+ truth_rows = []
+ for batch, result in site_candidates:
+ site = result["site"]
+ record_id = result["record_url"].rstrip("/").split("/")[-1]
+ candidate_id = f"esid-{site}"
+ pattern = "descending_band_visible"
+ if any(word in (result.get("visual_review") or "").lower()
+ for word in ("weak", "fragmented", "interruption")):
+ pattern = "descending_band_visible_weak_or_interrupted"
+ index_rows.append({
+ "candidate_id": candidate_id,
+ "source_type": "Eclipse Soundscapes field recording",
+ "source_parent_id": site,
+ "source_url": result["record_url"],
+ "title": f"2024-04-08 Eclipse Soundscapes ESID {site}",
+ "search_category": "publisher field site",
+ "clip_duration_s": result.get("clip_duration_s", ""),
+ "scanner_result": f"{result.get('proposal_groups', 0)} moving-tone proposal groups",
+ "proposal_count": result.get("proposal_groups", 0),
+ "visual_review": f"{pattern}: {result.get('visual_review', '')}",
+ "recording_location_class": "publisher_site_coordinate; microphone accuracy unknown",
+ "coarse_location_hint": f"Eclipse Soundscapes site {site}",
+ "reference_coordinate_key": candidate_id,
+ "recording_clock_status": (
+ "filename timestamp unverified; " +
+ str(result.get("publisher_clock_setting", "publisher clock setting unreported"))
+ ),
+ "aircraft_identity": "unconfirmed",
+ "location_evaluation_eligible": "no",
+ })
+ latitude, longitude = coordinates(site, record_id, args.truth_root)
+ truth_rows.append({
+ "candidate_id": candidate_id,
+ "source_parent_id": site,
+ "zenodo_record_id": record_id,
+ "record_url": result["record_url"],
+ "publisher_metadata_url": result.get("publisher_site_metadata_url", ""),
+ "site_latitude": latitude,
+ "site_longitude": longitude,
+ "coordinate_basis": "Publisher-reported deployment site coordinates",
+ "microphone_coordinate_accuracy": "Unknown; not an independently surveyed microphone position",
+ "clock_setting": result.get("publisher_clock_setting", ""),
+ "filename_start_utc_claim": result.get("filename_timestamp_utc_unverified", ""),
+ })
+
+ for row in sorted(unknown_candidates, key=lambda item: item["id"]):
+ durations = [float(v) for v in row["durations_s"].split("|") if v]
+ index_rows.append({
+ "candidate_id": f"yt-{row['id']}",
+ "source_type": "YouTube audio excerpt",
+ "source_parent_id": row["id"],
+ "source_url": row["url"],
+ "title": row["title"],
+ "search_category": row["category"],
+ "clip_duration_s": max(durations, default=""),
+ "scanner_result": row["scanner_result"],
+ "proposal_count": row["proposal_counts"],
+ "visual_review": row["visual_reviews"],
+ "recording_location_class": "exact receiver coordinates unresolved",
+ "coarse_location_hint": "Title may name a broad area or site; recorder position is not independently established",
+ "reference_coordinate_key": "",
+ "recording_clock_status": "recording UTC not established",
+ "aircraft_identity": "unconfirmed",
+ "location_evaluation_eligible": "no",
+ })
+
+ write_csv(args.index, candidate_fields, index_rows)
+ write_csv(args.truth, truth_fields, truth_rows)
+ print(f"candidate_index={len(index_rows)} site_coordinates={len(truth_rows)} "
+ f"unresolved_receiver_coordinate_videos={len(unknown_candidates)}")
+
+
+if __name__ == "__main__":
+ main()