Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
#!/usr/bin/env python3
"""Run fresh HTTP jobs for a predeclared real-recording representation matrix.
Prepare freezes configuration, inputs and implementation before any arm runs.
Run preserves failed, inconclusive and unusable arms. Report only compares them.
"""
from __future__ import annotations
import argparse
from copy import deepcopy
from datetime import datetime, timedelta, timezone
from importlib import metadata
from importlib.machinery import EXTENSION_SUFFIXES
import inspect
import itertools
import json
import math
import os
from pathlib import Path
import shlex
import signal
import subprocess
import sys
import tempfile
import threading
import numpy as np
from pyproj import Geod
from scipy import signal as scipy_signal
from scipy.optimize import linear_sum_assignment
import dopplergeist
from dopplergeist.contracts import validate_request
from dopplergeist.discovery import DiscoverySettings
from dopplergeist.intake import IntakeRecording, intake_recording, probe_media
import verify_upload_http as http_verifier
ROOT = Path(__file__).resolve().parents[1]
PROTOCOL = ROOT / "docs/verification/metamorphic_protocol_v1.json"
GEOD = Geod(ellps="WGS84")
def read_json(path, default=None):
path = Path(path)
return json.loads(path.read_text()) if path.is_file() else default
def write_json(path, value):
"""Atomic progress reports; immutable plans use exclusive creation."""
path = Path(path)
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n")
temporary.replace(path)
def identity(path):
path = Path(path).absolute()
resolved = path.resolve(strict=True)
return {"path": str(path), "resolved_path": str(resolved), "bytes": path.stat().st_size,
"sha256": http_verifier.sha256(path)}
def parse_utc(value):
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
if parsed.tzinfo is None:
raise ValueError("UTC endpoints must include a timezone")
return parsed.astimezone(timezone.utc)
def format_utc(value):
return value.isoformat().replace("+00:00", "Z")
def shifted_request(request, silence_s):
result = deepcopy(request)
window = result.get("time_window")
if not isinstance(window, dict):
raise ValueError("The matrix needs a defensible sample-zero UTC window")
for key in ("start_utc", "end_utc"):
window[key] = format_utc(parse_utc(window[key]) - timedelta(seconds=silence_s))
return result
def implementation():
from dopplergeist import data_access, matching, native_acoustic, native_cpa
package = Path(dopplergeist.__file__).resolve().parent
result = {
"package_files": {p.relative_to(package).as_posix(): http_verifier.sha256(p)
for p in sorted(package.rglob("*")) if p.is_file() and (
p.suffix in {".py", ".c"} or any(p.name.endswith(suffix) for suffix in EXTENSION_SUFFIXES))},
"runner_sha256": http_verifier.sha256(__file__),
"verifier_sha256": http_verifier.sha256(http_verifier.__file__),
"python": sys.version, "executable": sys.executable,
"packages": {name: metadata.version(name) for name in
("numpy", "scipy", "pandas", "pyproj", "rasterio", "pyarrow", "matplotlib")},
"backends": {"ridge": matching.RIDGE_FIT_BACKEND, "acoustic": native_acoustic.ACOUSTIC_BACKEND,
"cpa": native_cpa.CPA_BACKEND,
"index_payload": "native" if data_access._native_payload_columns is not None else "python"},
"environment": {name: value for name, value in sorted(os.environ.items())
if name in {"DOPPLERGEIST_CPA_BACKEND", "DOPPLERGEIST_ACOUSTIC_BACKEND",
"DOPPLERGEIST_RIDGE_BACKEND",
"OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS",
"NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS", "PYTHONHASHSEED"}},
"missing_runtime_pins": [],
}
for name in ("ffmpeg", "ffprobe"):
result[name] = subprocess.run([name, "-version"], check=True,
capture_output=True, text=True, timeout=10).stdout.splitlines()[0]
try:
result["packages"]["shapely"] = metadata.version("shapely")
except metadata.PackageNotFoundError:
result["packages"]["shapely"] = None
result["geos_version"] = None
else:
import shapely
result["geos_version"] = getattr(shapely, "geos_version_string", None)
if not isinstance(result["geos_version"], str) or not result["geos_version"]:
result["missing_runtime_pins"].append("geos_version")
return result
def operator_inputs(operator, additional):
"""Pin local data named by the operator, without following active job state."""
if operator.get("offline", True) is not True:
raise ValueError("Use an offline operator with frozen historical data and terrain")
paths = set()
for value in operator.get("flight_indexes", []):
paths.add(Path(value).resolve(strict=True))
ground = operator.get("terrain", {})
for value in ground.get("rasters", []):
paths.add(Path(value).resolve(strict=True))
if ground.get("geoid_raster"):
paths.add(Path(ground["geoid_raster"]).resolve(strict=True))
if operator.get("cache_root"):
cache = Path(operator["cache_root"]).resolve(strict=True)
paths.update(p.absolute() for p in cache.rglob("*") if p.is_file())
paths.update(Path(p).resolve(strict=True) for p in additional)
if not paths:
raise ValueError("No immutable historical/terrain inputs were supplied")
return [identity(path) for path in sorted(paths)]
def prepare(audio, request, operator, output, *, source_note, scope_note,
additional_inputs=(), job_max_runtime_s=None, http_timeout_s=None):
audio = Path(audio).resolve(strict=True)
operator = deepcopy(operator)
for name in ("flight_indexes",):
if name in operator:
operator[name] = [str(Path(p).resolve(strict=True)) for p in operator[name]]
ground = operator.get("terrain", {})
if "rasters" in ground:
ground["rasters"] = [str(Path(p).resolve(strict=True)) for p in ground["rasters"]]
if ground.get("geoid_raster"):
ground["geoid_raster"] = str(Path(ground["geoid_raster"]).resolve(strict=True))
if operator.get("cache_root"):
operator["cache_root"] = str(Path(operator["cache_root"]).resolve(strict=True))
if set(operator.get("discovery", {})) - {"chunk_s", "overlap_s"}:
raise ValueError("The upload operator exposes only discovery chunk_s and overlap_s")
protocol = read_json(PROTOCOL)
for key, override in (("job_max_runtime_s", job_max_runtime_s),
("http_timeout_s", http_timeout_s)):
if override is not None:
if not math.isfinite(override) or override <= 0:
raise ValueError(f"{key} must be finite and positive")
protocol["budgets"][key] = float(override)
if protocol["budgets"]["job_max_runtime_s"] >= protocol["budgets"]["http_timeout_s"]:
raise ValueError("The client deadline must exceed the job runtime limit")
if not source_note.strip() or not scope_note.strip():
raise ValueError("Declare the real source and the evaluation scope")
request = dict(validate_request(dict(request, audio_path=str(audio))))
request.pop("audio_path")
shifted_request(request, 0)
DiscoverySettings(**operator.get("discovery", {}))
inputs = operator_inputs(operator, additional_inputs)
cases = []
for case in protocol["cases"]:
settings = dict(protocol["baseline"], **case["changes"])
arm_operator = deepcopy(operator)
if "discovery" in settings:
arm_operator["discovery"] = settings.pop("discovery")
DiscoverySettings(**arm_operator.get("discovery", {}))
cases.append({"id": case["id"], "encoding": settings,
"request": shifted_request(request, settings["leading_silence_s"]),
"operator": arm_operator})
pinned_implementation = implementation()
if pinned_implementation["missing_runtime_pins"]:
raise ValueError("Cannot freeze missing runtime pins: "
+ ", ".join(pinned_implementation["missing_runtime_pins"]))
plan = {"schema_version": 1, "prepared_utc": http_verifier.now(),
"protocol": protocol, "protocol_sha256": http_verifier.sha256(PROTOCOL),
"source": identity(audio), "source_note": source_note, "scope_note": scope_note,
"request": request, "operator": operator, "inputs": inputs, "cases": cases,
"additional_inputs": [str(Path(p).resolve(strict=True)) for p in additional_inputs],
"implementation": pinned_implementation,
"source_commit": subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT,
check=True, capture_output=True, text=True).stdout.strip(),
"truth_evaluation": {"eligible": False, "evaluated": False,
"reason": "No independent truth supplied to this protocol"},
"national_gate_satisfied": False}
output = Path(output).resolve()
output.mkdir(parents=True, exist_ok=False)
with (output / "plan.json").open("x") as stream:
json.dump(plan, stream, indent=2, allow_nan=False)
stream.write("\n")
(output / "plan.sha256").write_text(http_verifier.sha256(output / "plan.json") + "\n")
return output / "plan.json"
def check_frozen(plan_path, plan):
expected = plan_path.with_name("plan.sha256").read_text().strip()
if http_verifier.sha256(plan_path) != expected:
raise ValueError("The frozen plan changed; prepare a new experiment")
if implementation() != plan["implementation"]:
raise ValueError("Implementation, runtime or backend configuration changed after preparation")
if identity(plan["source"]["path"]) != plan["source"]:
raise ValueError("The recorded source changed")
if operator_inputs(plan["operator"], plan["additional_inputs"]) != plan["inputs"]:
raise ValueError("Pinned data bytes, paths or cache file set changed")
def run_command(command, directory, name, timeout_s, *, env=None):
write_json(directory / f"{name}-command.json", command)
with (directory / f"{name}.log").open("wb") as log:
process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
env=env, start_new_session=True)
try:
returncode = process.wait(timeout=timeout_s)
if returncode:
raise subprocess.CalledProcessError(returncode, command)
except BaseException:
# Intake starts FFmpeg. Stop this command's whole process group.
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
pass
try:
process.wait(timeout=2)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait()
raise
def preflight_decode(audio, directory, protocol):
"""Use actual intake in a bounded child; HTTP intake is still fresh."""
command = [sys.executable, "-c",
"import sys; from dopplergeist.intake import intake_recording; "
"intake_recording(sys.argv[1], sys.argv[2], sample_rate_hz=int(sys.argv[3]))",
str(audio), str(directory), str(protocol["pcm_comparison"]["sample_rate_hz"])]
env = dict(os.environ, PYTHONPATH=str(Path(dopplergeist.__file__).resolve().parent.parent)
+ os.pathsep + os.environ.get("PYTHONPATH", ""))
run_command(command, directory.parent, directory.name + "-decode",
protocol["budgets"]["encoding_timeout_s"], env=env)
return IntakeRecording(directory / "recording.f32", read_json(directory / "recording.json"),
read_json(directory / "intake.json"))
def make_canonical(source, directory, protocol):
probe = probe_media(source)
audio = next(s for s in probe["streams"] if s["codec_type"] == "audio")
channels = int(audio["channels"])
downmix = "pan=mono|c0=" + "+".join(f"{1/channels:.17g}*c{i}" for i in range(channels))
path = directory / "canonical.wav"
command = ["ffmpeg", "-nostdin", "-v", "error", "-xerror", "-i", str(source),
"-map", "0:a:0", "-vn", "-sn", "-dn", "-af", downmix, "-ac", "1",
"-ar", str(protocol["canonical_pcm"]["sample_rate_hz"]),
"-c:a", "pcm_f32le", "-map_metadata", "-1", "-rf64", "auto", str(path)]
run_command(command, directory, "canonical", protocol["budgets"]["encoding_timeout_s"])
reference = preflight_decode(path, directory / "reference-pcm", protocol)
return path, reference
def encode(canonical, directory, settings, protocol):
filters = []
if settings["gain_db"]:
filters.append(f"volume={10**(settings['gain_db']/20):.17g}:precision=double")
if settings["leading_silence_s"]:
samples = round(settings["leading_silence_s"] * protocol["canonical_pcm"]["sample_rate_hz"])
filters.append(f"adelay={samples}S:all=1")
if settings["channels"] == 2:
filters.append("pan=stereo|c0=c0|c1=c0")
path = directory / "input.mp3"
command = ["ffmpeg", "-nostdin", "-v", "error", "-xerror", "-i", str(canonical),
"-map", "0:a:0", "-vn", "-sn", "-dn"]
if filters:
command += ["-af", ",".join(filters)]
command += ["-ac", str(settings["channels"]), "-ar", str(settings["sample_rate_hz"]),
"-c:a", "libmp3lame", "-b:a", f"{settings['bitrate_kbps']}k",
"-write_xing", "1", "-map_metadata", "-1", str(path)]
run_command(command, directory, "encode", protocol["budgets"]["encoding_timeout_s"])
return path
def native_decode_metrics(audio, directory, protocol):
"""Scan blocks of a temporary native-rate decode; keep no second large PCM."""
probe = probe_media(audio)
stream = next(s for s in probe["streams"] if s["codec_type"] == "audio")
channels = int(stream["channels"])
command = ["ffmpeg", "-nostdin", "-v", "error", "-xerror", "-i", str(audio),
"-map", "0:a:0", "-f", "f32le", "-c:a", "pcm_f32le", "pipe:1"]
peak, frames, channel_difference = 0.0, 0, 0.0
with tempfile.TemporaryFile(dir=directory) as raw, (directory / "native-decode.log").open("wb") as log:
subprocess.run(command, stdout=raw, stderr=log, check=True,
timeout=protocol["budgets"]["encoding_timeout_s"])
if raw.tell() % (4 * channels):
raise ValueError("Native decode has an incomplete audio frame")
raw.seek(0)
while len(block := np.fromfile(raw, dtype="<f4", count=65536 * channels)):
if not np.isfinite(block).all():
raise ValueError("Native decode contains nonfinite samples")
values = block.reshape(-1, channels)
peak = max(peak, float(np.max(np.abs(values))))
if channels == 2:
channel_difference = max(channel_difference, float(np.max(np.abs(values[:, 0] - values[:, 1]))))
frames += len(values)
packets = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "a:0",
"-read_intervals", "%+#1", "-show_packets", "-of", "json", str(audio)],
check=True, capture_output=True, text=True, timeout=10)
return {"sample_rate_hz": int(stream["sample_rate"]), "channels": channels,
"sample_count": frames, "duration_s": frames / int(stream["sample_rate"]),
"absolute_peak": peak, "identical_channel_max_difference": channel_difference if channels == 2 else None,
"first_packet": json.loads(packets.stdout).get("packets", [])[:1],
"container_start_time": probe["format"].get("start_time"),
"container_duration": probe["format"].get("duration"), "decode_command": command}
def compare_pcm(reference_path, variant_path, settings, protocol):
config, tol = protocol["pcm_comparison"], protocol["tolerances"]
sr = config["sample_rate_hz"]
left = np.memmap(reference_path, dtype="<f4", mode="r")
right = np.memmap(variant_path, dtype="<f4", mode="r")
padding = round(settings["leading_silence_s"] * sr)
gain = 10 ** (settings["gain_db"] / 20)
count = min(len(left), max(0, len(right) - padding))
if count == 0:
raise ValueError("No recorded samples remain after the declared padding")
squared_error, energy = 0.0, 0.0
for first in range(0, count, config["block_samples"]):
last = min(count, first + config["block_samples"])
a = np.asarray(left[first:last], dtype=np.float64)
b = np.asarray(right[first+padding:last+padding], dtype=np.float64) / gain
squared_error += float(np.dot(a-b, a-b))
energy += float(np.dot(a, a))
duration_error = (len(right) - padding - len(left)) / sr
nrmse = math.sqrt(squared_error / energy) if energy > 0 else None
radius = round(config["lag_search_radius_s"] * sr)
width = min(round(config["lag_window_s"] * sr), count - 2*radius)
lags = []
if width > 0:
for first in np.linspace(radius, count-radius-width, config["lag_windows"]).astype(int):
a = np.asarray(left[first:first+width], dtype=np.float64)
a = a - a.mean()
if float(np.dot(a, a)) <= 1e-14:
lags.append({"start_s": int(first)/sr, "lag_s": None, "reason": "no_alignment_energy"})
continue
b = np.asarray(right[first+padding-radius:first+padding+width+radius], dtype=np.float64) / gain
values = scipy_signal.correlate(b, a, mode="valid", method="fft")
sums, squares = np.r_[0.0, np.cumsum(b)], np.r_[0.0, np.cumsum(b*b)]
variance = squares[width:] - squares[:-width] - (sums[width:] - sums[:-width])**2/width
normalized = values / np.sqrt(np.maximum(variance, np.finfo(float).tiny))
lag = (int(np.argmax(normalized)) - radius) / sr
lags.append({"start_s": int(first)/sr, "lag_s": lag})
measured = [abs(row["lag_s"]) for row in lags if row["lag_s"] is not None]
max_lag = max(measured) if measured else None
return {"reference_samples": len(left), "variant_samples": len(right),
"compared_samples": count, "padding_samples": padding, "declared_gain": gain,
"duration_error_s": duration_error, "gain_corrected_nrmse": nrmse,
"fixed_window_lags": lags, "maximum_absolute_lag_s": max_lag,
"passed": (abs(duration_error) <= tol["decoded_duration_error_s"]
and nrmse is not None and nrmse <= tol["gain_corrected_pcm_nrmse"]
and max_lag is not None and max_lag <= tol["residual_alignment_lag_s"])}
class MemorySampler:
"""Sample only this runner's descendants, never another worker's jobs."""
STOP_TIMEOUT_S = 2.0
def __init__(self):
self.stop = threading.Event()
self.peak = 0
self.samples = 0
self.thread_error = None
self.stop_timed_out = False
self.skipped_process_reads = 0
try:
# /proc can be mounted from an outer PID namespace. os.getpid()
# then names a different entry in that proc mount.
self.proc_root_pid = int(Path("/proc/self").resolve(strict=True).name)
except (OSError, RuntimeError, ValueError):
self.proc_root_pid = None
self.traversal = ("task_children" if self.proc_root_pid is not None
and any(Path(f"/proc/{self.proc_root_pid}/task").glob("*/children"))
else "ppid_snapshot")
self.thread = threading.Thread(target=self.sample_guarded, daemon=True)
def sample_guarded(self):
try:
self.sample()
except BaseException as exc:
# Instrumentation must preserve an HTTP result even if the sampler
# fails after observing a positive, incomplete peak.
self.thread_error = f"{type(exc).__name__}: {exc}"
@staticmethod
def process_status(pid, *, include_rss):
parent, rss = None, None
with Path(f"/proc/{pid}/status").open() as status:
for line in status:
if line.startswith("PPid:"):
parent = int(line.split()[1])
if not include_rss:
break
elif include_rss and line.startswith("VmRSS:"):
rss = int(line.split()[1]) * 1024
if parent is not None and (rss is not None or not include_rss):
break
return parent, rss
def parent_snapshot(self):
# Some proc mounts omit task/*/children. Retain only parent relations
# during this scan; read RSS only for the selected descendant tree.
children = {}
for entry in Path("/proc").iterdir():
if self.stop.is_set():
break
if not entry.name.isdecimal():
continue
try:
parent, _ = self.process_status(int(entry.name), include_rss=False)
if parent is not None:
children.setdefault(parent, []).append(int(entry.name))
except (OSError, ValueError):
self.skipped_process_reads += 1
return children
def sample(self):
if self.proc_root_pid is None:
return
while not self.stop.is_set():
snapshot = self.parent_snapshot() if self.traversal == "ppid_snapshot" else None
pending, visited, total = [(self.proc_root_pid, None)], set(), 0
while pending and not self.stop.is_set():
pid, expected_parent = pending.pop()
if pid in visited:
continue
visited.add(pid)
try:
if pid != self.proc_root_pid:
parent, rss = self.process_status(pid, include_rss=True)
# A process can disappear, be reparented, or have its PID
# reused after the snapshot. Exclude a changed relation.
if parent != expected_parent:
self.skipped_process_reads += 1
continue
total += rss or 0
if snapshot is not None:
pending += [(child, pid) for child in snapshot.get(pid, [])]
else:
# The service can spawn a worker from any of its threads.
for children in Path(f"/proc/{pid}/task").glob("*/children"):
if self.stop.is_set():
break
pending += [(int(x), pid) for x in children.read_text().split()]
except (OSError, ValueError):
self.skipped_process_reads += 1
self.peak = max(self.peak, total)
self.samples += 1
self.stop.wait(0.1)
def __enter__(self):
self.thread.start()
return self
def __exit__(self, *_):
self.stop.set()
self.thread.join(timeout=self.STOP_TIMEOUT_S)
self.stop_timed_out = self.thread.is_alive()
def result(self):
return {"sampled_descendant_peak_rss_mib": self.peak/(1024**2) if self.peak else None,
"interval_s": 0.1, "samples": self.samples,
"thread_error": self.thread_error, "stop_timed_out": self.stop_timed_out,
"partial_measurement": bool(self.thread_error or self.stop_timed_out),
"skipped_process_reads": self.skipped_process_reads,
"traversal": self.traversal,
"unavailable_reason": ("sampler_error" if self.thread_error else
"proc_self_unavailable" if self.proc_root_pid is None
else "no_descendant_rss_observed") if not self.peak else None,
"scope": "Sum of local service and worker descendant RSS; sampled lower bound, not unique physical memory"}
def mapped_events(events, padding):
result = deepcopy(events)
for event in result:
event["start_s"] -= padding
event["end_s"] -= padding
for sample in event["ridge"]:
sample["time_s"] -= padding
return result
def observe_case(directory, case, receipt):
job_id = receipt.get("job_id")
job = directory / "http" / "jobs" / job_id if job_id else None
final = read_json(directory / "http/result.json", {})
if not final and job is not None:
final = read_json(job / "result.json", {})
discovery = read_json(job / "discovery/discovery.json", {}) if job else {}
events = final.get("events", discovery.get("events", []))
search = read_json(job / "search/search-report.json", {}) if job else {}
refined = read_json(job / "refined-candidates.json", {}) if job else {}
decision = read_json(job / "decision.json", {}) if job else {}
evidence = read_json(directory / "http/pipeline-evidence.json", {})
padding = case["encoding"]["leading_silence_s"]
candidates = []
groups = {event["event_id"]: event.get("harmonic_group_id", event["event_id"]) for event in events}
for i, candidate in enumerate(final.get("candidates", [])):
row = refined.get("ranked", [])[i] if i < len(refined.get("ranked", [])) else {}
copied = deepcopy(candidate)
copied["mapped_recording_start_utc"] = format_utc(
parse_utc(candidate["recording_start_utc"]) + timedelta(seconds=padding))
copied["matched_group_count"] = len({groups[e] for e in candidate["event_ids"] if e in groups})
copied["normalized_score"] = candidate["score"] / max(1, copied["matched_group_count"])
sensitivity = row.get("sensitivity", {})
copied["uncertainty_provenance"] = {key: sensitivity.get(key) for key in
("kind", "method", "sources", "complete", "boundary_limited")}
copied["uncertainty_radius_m"] = sensitivity.get("radius_m")
copied["mode_id"] = row.get("mode_id")
candidates.append(copied)
keys = ("status", "tiles_total", "tiles_completed", "tiles_resumed", "unsearched_tiles",
"grid_points_searched", "refinement_points_searched", "refinement_points_skipped",
"refinement_stages_resumed", "model_assessment_complete", "model_support_complete",
"data_scope_complete", "all_event_groups_considered", "event_groups_proposed",
"alternative_modes_exhaustively_assessed", "elapsed_s", "process_peak_rss_mib",
"bounded_concurrency")
return {"job_id": job_id, "job_directory": str(job) if job else None,
"status": final.get("status"), "reason_code": final.get("reason_code"),
"stage": final.get("stage"), "accepted_location": final.get("status") == "succeeded",
"verification_passed": receipt.get("verification_passed", False),
"full_pipeline_observed": receipt.get("full_pipeline_observed", False),
"full_pipeline_requirements_met": receipt.get("full_pipeline_requirements_met", False),
"stages_verified": evidence.get("stages_verified", []),
"http_elapsed_s": receipt.get("elapsed_s"), "interrupted": receipt.get("interrupted", False),
"verification_errors": {key: receipt[key] for key in
("error", "finalization_error", "requirement_error", "pipeline_evidence_errors") if key in receipt},
"mapped_events": mapped_events(events, padding),
"ridge_count": len(events), "group_count": len(set(groups.values())),
"candidates": candidates, "coverage": final.get("coverage"),
"work": {key: search.get(key) for key in keys},
"finite_search_evidence": search.get("finite_search_evidence"),
"decision_provenance": decision.get("diagnostics"),
"acquisition": read_json(job / "data-provenance.json", {}) if job else {}}
def assign_pairs(left_count, right_count, eligible):
"""Maximum-cardinality, then minimum-cost one-to-one correspondence."""
if not left_count or not right_count:
return []
penalty = 10 * (left_count + right_count + 1)
costs = np.full((left_count, right_count+left_count), float(penalty))
costs[:, :right_count] = 3 * penalty
for (i, j), row in eligible.items():
costs[i, j] = row["cost"]
rows, columns = linear_sum_assignment(costs)
return [(int(i), int(j)) for i, j in zip(rows, columns) if (i, j) in eligible]
def event_correspondence(left, right, protocol):
rules = protocol["event_correspondence"]
eligible = {}
for i, a in enumerate(left):
for j, b in enumerate(right):
first, last = max(a["start_s"], b["start_s"]), min(a["end_s"], b["end_s"])
overlap = max(0.0, last-first) / min(a["end_s"]-a["start_s"], b["end_s"]-b["start_s"])
if overlap < rules["minimum_overlap_fraction_of_shorter_event"]:
continue
ta, fa = np.array([(r["time_s"], r["frequency_hz"]) for r in a["ridge"]]).T
tb, fb = np.array([(r["time_s"], r["frequency_hz"]) for r in b["ridge"]]).T
mask = (ta >= tb[0]) & (ta <= tb[-1])
if not mask.any():
continue
frequency = float(np.median(np.abs(fa[mask] - np.interp(ta[mask], tb, fb))))
if frequency > rules["maximum_matching_median_frequency_hz"]:
continue
eligible[i, j] = {"left_event_id": a["event_id"], "right_event_id": b["event_id"],
"left_group_id": a.get("harmonic_group_id", a["event_id"]),
"right_group_id": b.get("harmonic_group_id", b["event_id"]),
"start_delta_s": b["start_s"]-a["start_s"],
"end_delta_s": b["end_s"]-a["end_s"],
"overlap_fraction": overlap, "median_frequency_delta_hz": frequency,
"compared_ridge_samples": int(mask.sum()),
"cost": 1-overlap + frequency/rules["maximum_matching_median_frequency_hz"]}
matches = [eligible[pair] for pair in assign_pairs(len(left), len(right), eligible)]
left_ids = {row["left_event_id"] for row in matches}
right_ids = {row["right_event_id"] for row in matches}
forward, reverse, possible_forward, possible_reverse = {}, {}, {}, {}
for rows, fwd, rev in ((matches, forward, reverse),
(eligible.values(), possible_forward, possible_reverse)):
for row in rows:
fwd.setdefault(row["left_group_id"], set()).add(row["right_group_id"])
rev.setdefault(row["right_group_id"], set()).add(row["left_group_id"])
return {"matches": matches,
"unmatched_left": [e["event_id"] for e in left if e["event_id"] not in left_ids],
"unmatched_right": [e["event_id"] for e in right if e["event_id"] not in right_ids],
"unmatched_left_groups": sorted({e.get("harmonic_group_id", e["event_id"]) for e in left} - forward.keys()),
"unmatched_right_groups": sorted({e.get("harmonic_group_id", e["event_id"]) for e in right} - reverse.keys()),
"split_groups": {key: sorted(value) for key, value in forward.items() if len(value) > 1},
"merged_groups": {key: sorted(value) for key, value in reverse.items() if len(value) > 1},
"possible_split_or_ambiguous_groups": {key: sorted(value) for key, value in possible_forward.items() if len(value) > 1},
"possible_merge_or_ambiguous_groups": {key: sorted(value) for key, value in possible_reverse.items() if len(value) > 1}}
def distance(a, b):
return float(GEOD.inv(a["lon"], a["lat"], b["lon"], b["lat"])[2])
def associations_equal(a, b, mapping):
first = list(zip(a["event_ids"], a["flight_ids"]))
if any(event not in mapping for event, _ in first):
return False
return sorted((mapping[event], flight) for event, flight in first) == sorted(zip(b["event_ids"], b["flight_ids"]))
def coverage_state(coverage):
"""Comparable coverage evidence; preserve raw limitations in the report."""
if not isinstance(coverage, dict):
return None
return {key: coverage.get(key) for key in
("search_complete", "data_status", "missing_data", "requested_area")}
def completion_state(observation):
"""Completion gates, excluding elapsed time and performance-only counters."""
work = observation.get("work") or {}
return {
"verification_passed": observation.get("verification_passed"),
"full_pipeline_observed": observation.get("full_pipeline_observed"),
"full_pipeline_requirements_met": observation.get("full_pipeline_requirements_met"),
**{key: work.get(key) for key in
("status", "unsearched_tiles", "model_assessment_complete",
"model_support_complete", "data_scope_complete",
"all_event_groups_considered",
"alternative_modes_exhaustively_assessed")},
}
def compare_cases(left, right, protocol):
a, b = left["observation"], right["observation"]
tol = protocol["tolerances"]
correspondence = event_correspondence(a["mapped_events"], b["mapped_events"], protocol)
mapping = {m["left_event_id"]: m["right_event_id"] for m in correspondence["matches"]}
coverage_states = [coverage_state(a.get("coverage")), coverage_state(b.get("coverage"))]
completion_states = [completion_state(a), completion_state(b)]
result = {"left": left["id"], "right": right["id"], "event_correspondence": correspondence,
"representation_usable": bool(left.get("usability", {}).get("passed") and right.get("usability", {}).get("passed")),
"complete_pipelines": bool(a["full_pipeline_requirements_met"] and b["full_pipeline_requirements_met"]),
"status_transition": [a["status"], b["status"]],
"reason_transition": [a["reason_code"], b["reason_code"]],
"group_count_delta": b["group_count"] - a["group_count"],
"ridge_count_delta": b["ridge_count"] - a["ridge_count"],
"coverage_equal": a["coverage"] == b["coverage"],
"coverage_state_transition": coverage_states,
"completion_state_transition": completion_states,
"coverage_transition": [a["coverage"], b["coverage"]],
"work_transition": [a["work"], b["work"]],
"decision_provenance_equal": a["decision_provenance"] == b["decision_provenance"],
"diagnostic_candidate_comparison": None, "accepted_location_displacement_m": None,
"serious_accepted_jump": False, "violations": []}
checks = {
"coverage_state_changed": coverage_states[0] != coverage_states[1],
"work_completion_changed": completion_states[0] != completion_states[1],
"status_changed": a["status"] != b["status"],
"reason_changed": a["reason_code"] != b["reason_code"],
"group_count_changed": a["group_count"] != b["group_count"],
"ridge_correspondence_incomplete": bool(correspondence["unmatched_left"] or correspondence["unmatched_right"]),
"group_split_or_merge": bool(correspondence["split_groups"] or correspondence["merged_groups"]),
"mapped_event_edges": any(max(abs(m["start_delta_s"]), abs(m["end_delta_s"])) > tol["mapped_event_edge_s"]
for m in correspondence["matches"]),
"mapped_ridge_frequency": any(m["median_frequency_delta_hz"] > tol["median_ridge_frequency_hz"]
for m in correspondence["matches"]),
}
if a["candidates"] and b["candidates"]:
best_a, best_b = a["candidates"][0], b["candidates"][0]
displacement = distance(best_a, best_b)
clock_delta = abs((parse_utc(best_b["mapped_recording_start_utc"])
- parse_utc(best_a["mapped_recording_start_utc"])).total_seconds())
same_associations = associations_equal(best_a, best_b, mapping)
same_units = best_a["score_kind"] == best_b["score_kind"]
comparable = same_associations and same_units and best_a["matched_group_count"] == best_b["matched_group_count"]
score_delta = abs(best_a["normalized_score"] - best_b["normalized_score"]) if comparable else None
result["diagnostic_candidate_comparison"] = {
"displacement_m": displacement, "within_threshold_m": {
str(int(limit)): displacement <= limit for limit in tol["displacement_thresholds_m"]},
"mapped_clock_delta_s": clock_delta, "associated_flights_equal": same_associations,
"score_units_equal": same_units, "normalized_score_comparable": comparable,
"normalized_score_delta": score_delta,
"uncertainty_provenance_equal": best_a["uncertainty_provenance"] == best_b["uncertainty_provenance"],
"uncertainty_radius_m": [best_a["uncertainty_radius_m"], best_b["uncertainty_radius_m"]]}
checks.update(associated_flights_changed=not same_associations, score_units_changed=not same_units,
normalized_score_changed=score_delta is not None and score_delta > tol["normalized_score_per_matched_group"],
fitted_clock_changed=clock_delta > tol["mapped_clock_s"],
diagnostic_candidate_jump=displacement > tol["accepted_jump_m"],
uncertainty_provenance_changed=best_a["uncertainty_provenance"] != best_b["uncertainty_provenance"])
if a["accepted_location"] and b["accepted_location"]:
result["accepted_location_displacement_m"] = displacement
result["serious_accepted_jump"] = displacement > tol["accepted_jump_m"]
eligible = {}
for i, first in enumerate(a["candidates"]):
for j, second in enumerate(b["candidates"]):
if associations_equal(first, second, mapping):
separation = distance(first, second)
clock = abs((parse_utc(first["mapped_recording_start_utc"])
- parse_utc(second["mapped_recording_start_utc"])).total_seconds())
same_units = first["score_kind"] == second["score_kind"]
if separation <= tol["distant_mode_m"] and clock <= tol["mapped_clock_s"] and same_units:
eligible[i, j] = {"cost": separation/tol["distant_mode_m"] + clock/tol["mapped_clock_s"],
"distance_m": separation, "mapped_clock_delta_s": clock,
"score_kind": first["score_kind"],
"normalized_score_delta": abs(first["normalized_score"]-second["normalized_score"])
if first["matched_group_count"] == second["matched_group_count"] else None,
"uncertainty_provenance_equal": first["uncertainty_provenance"] == second["uncertainty_provenance"]}
pairs = assign_pairs(len(a["candidates"]), len(b["candidates"]), eligible)
result["retained_mode_correspondence"] = [
dict(left_index=i, right_index=j, **{key: value for key, value in eligible[i, j].items() if key != "cost"})
for i, j in pairs]
result["unmatched_retained_left_indices"] = sorted(set(range(len(a["candidates"]))) - {i for i, _ in pairs})
result["unmatched_retained_right_indices"] = sorted(set(range(len(b["candidates"]))) - {j for _, j in pairs})
checks["retained_mode_score_changed"] = any(row["normalized_score_delta"] is not None
and row["normalized_score_delta"] > tol["normalized_score_per_matched_group"]
for row in result["retained_mode_correspondence"])
checks["retained_mode_uncertainty_provenance_changed"] = any(
not row["uncertainty_provenance_equal"] for row in result["retained_mode_correspondence"])
distant = []
for label, source, matched in (("left", a, {i for i, _ in pairs}),
("right", b, {j for _, j in pairs})):
for index, candidate in enumerate(source["candidates"]):
offset = distance(source["candidates"][0], candidate)
if offset >= tol["distant_mode_m"]:
distant.append({"side": label, "candidate_index": index,
"distance_from_winner_m": offset, "retained_within_1000m": index in matched})
result["distant_modes"] = distant
checks["distant_mode_lost"] = any(not row["retained_within_1000m"] for row in distant)
elif bool(a["candidates"]) != bool(b["candidates"]):
checks["candidate_presence_changed"] = True
result["violations"] = [name for name, failed in checks.items() if failed]
return result
def build_report(plan_path, plan, state):
cases = state["cases"]
observed = [row for row in cases if row.get("observation")]
pairs = []
for a, b in itertools.combinations(observed, 2):
try:
pairs.append(compare_cases(a, b, plan["protocol"]))
except Exception as exc:
pairs.append({"left": a["id"], "right": b["id"],
"error": f"{type(exc).__name__}: {exc}",
"violations": ["comparison_error"], "serious_accepted_jump": False})
completed = [row for row in cases if row["state"] == "finished"]
passed = [row for row in completed if row.get("observation", {}).get("verification_passed")]
stages = [row for row in passed if row["observation"]["full_pipeline_observed"]]
full = [row for row in passed if row["observation"]["full_pipeline_requirements_met"]]
violations = [pair for pair in pairs if pair["violations"]]
return {"schema_version": 1, "plan": str(plan_path), "plan_sha256": http_verifier.sha256(plan_path),
"updated_utc": http_verifier.now(), "source_note": plan["source_note"], "scope_note": plan["scope_note"],
"declared_case_count": len(plan["cases"]), "finished_cases": len(completed),
"verified_cases": len(passed), "complete_stage_cases": len(stages), "full_pipeline_cases": len(full),
"run_state": state["state"], "error": state.get("error"), "cases": cases, "pairwise": pairs,
"pairs_with_observed_differences": len(violations),
"serious_accepted_jumps": [p for p in pairs if p["serious_accepted_jump"]],
"all_complete_usable_and_within_tolerances": (state["state"] == "finished"
and len(full) == len(plan["cases"])
and all(row.get("usability", {}).get("passed") for row in cases) and not violations),
"truth_evaluation": plan["truth_evaluation"], "national_gate_satisfied": False,
"interpretation": "Inconclusive decisions remain inconclusive. Pairwise agreement is not location accuracy. Early exits do not complete the pipeline requirement."}
def save_report(plan_path, plan, state, output):
report = build_report(plan_path, plan, state)
write_json(output / "report.json", report)
lines = ["# Real recording representation matrix", "", plan["scope_note"], "",
f"Full pipeline observed in {report['full_pipeline_cases']}/{report['declared_case_count']} declared cases.",
"No independent truth error was evaluated. National acceptance remains open.", "",
"| Case | State | Status / reason | Groups / ridges | Usable | Full pipeline | HTTP seconds |",
"| --- | --- | --- | --- | --- | --- | --- |"]
for row in state["cases"]:
obs = row.get("observation", {})
seconds = obs.get("http_elapsed_s")
lines.append(f"| {row['id']} | {row['state']} | {obs.get('status')} / {obs.get('reason_code')} | "
f"{obs.get('group_count')} / {obs.get('ridge_count')} | {row.get('usability', {}).get('passed')} | "
f"{obs.get('full_pipeline_requirements_met')} | {round(seconds, 2) if seconds is not None else 'unavailable'} |")
lines += ["", f"Pairs with observed differences: {report['pairs_with_observed_differences']}.",
f"Serious accepted location jumps: {len(report['serious_accepted_jumps'])}.", "",
"The JSON report retains mapped ridges, correspondence, diagnostic candidates,",
"coverage/work transitions, uncertainty provenance, and per-arm reproducers."]
if state.get("error"):
lines += ["", "Run error: " + state["error"]]
(output / "report.md").write_text("\n".join(lines) + "\n")
return report
def run(plan_path, output, *, selected_case=None):
plan_path, output = Path(plan_path).resolve(strict=True), Path(output).resolve()
plan = read_json(plan_path)
check_frozen(plan_path, plan)
if "job_max_runtime_s" not in inspect.signature(http_verifier.verify_http).parameters:
raise ValueError("This runner needs the HTTP verifier's job_max_runtime_s API")
if selected_case and selected_case not in {case["id"] for case in plan["cases"]}:
raise ValueError("Unknown case identifier")
output.mkdir(parents=True, exist_ok=False)
state = {"state": "running", "runner_pid": os.getpid(), "started_utc": http_verifier.now(),
"cases": [{"id": case["id"], "state": "pending"} for case in plan["cases"]]}
write_json(output / "state.json", state)
save_report(plan_path, plan, state, output)
def interrupted(signum, _frame):
raise KeyboardInterrupt(f"Received {signal.Signals(signum).name}")
previous = signal.signal(signal.SIGTERM, interrupted)
active = None
try:
canonical, reference = make_canonical(plan["source"]["path"], output, plan["protocol"])
write_json(output / "canonical.json", identity(canonical))
baseline_input = None
if selected_case and selected_case.startswith("chunks_"):
baseline = next(case for case in plan["cases"] if case["id"] == "baseline")
baseline_dir = output / "baseline-encoding"
baseline_dir.mkdir()
baseline_input = identity(encode(canonical, baseline_dir, baseline["encoding"], plan["protocol"]))
for case, row in zip(plan["cases"], state["cases"]):
if selected_case and case["id"] != selected_case:
row["state"] = "not_selected"
continue
check_frozen(plan_path, plan)
active = row
directory = output / case["id"]
directory.mkdir()
row.update(state="running", directory=str(directory),
reproducer=shlex.join([sys.executable, str(Path(__file__).resolve()), "run",
"--plan", str(plan_path), "--output", str(directory)+"-reproducer", "--case", case["id"]]))
write_json(output / "state.json", state)
try:
audio = encode(canonical, directory, case["encoding"], plan["protocol"])
row["encoded_input"] = identity(audio)
try:
decoded = preflight_decode(audio, directory / "preflight", plan["protocol"])
native = native_decode_metrics(audio, directory, plan["protocol"])
pcm = compare_pcm(reference.samples_path, decoded.samples_path, case["encoding"], plan["protocol"])
row["usability"] = {"pcm": pcm, "native_decode": native,
"passed": bool(pcm["passed"] and native["absolute_peak"] < plan["protocol"]["tolerances"]["decoded_absolute_peak_exclusive"]
and (native["channels"] != 2 or native["identical_channel_max_difference"] == 0.0))}
except Exception as exc:
row["usability"] = {"passed": False, "error": f"{type(exc).__name__}: {exc}"}
if case["id"] == "baseline":
baseline_input = row["encoded_input"]
if case["id"].startswith("chunks_"):
if baseline_input is None:
raise ValueError("Chunk input cannot be verified without the baseline encoding")
same_input = row["encoded_input"]["sha256"] == baseline_input["sha256"]
row["usability"]["chunk_input_matches_baseline"] = same_input
row["usability"]["passed"] = bool(row["usability"]["passed"] and same_input)
write_json(directory / "representation.json", {"encoding": case["encoding"],
"encoded_input": row["encoded_input"], "usability": row["usability"],
"request": case["request"], "operator": case["operator"]})
# Failed usability checks do not remove encoded arms or cause
# outcome-dependent settings changes. The upload remains fresh.
memory = MemorySampler()
try:
with memory:
receipt = http_verifier.verify_http(audio, directory / "http",
request=case["request"], operator=case["operator"], expect_status=None,
job_max_runtime_s=plan["protocol"]["budgets"]["job_max_runtime_s"],
timeout_s=plan["protocol"]["budgets"]["http_timeout_s"],
require_full_pipeline=False)
finally:
row["memory"] = memory.result()
row["observation"] = observe_case(directory, case, receipt)
row["state"] = "finished"
if receipt.get("interrupted"):
raise KeyboardInterrupt("HTTP verification was interrupted")
except Exception as exc:
row.update(state="error", error=f"{type(exc).__name__}: {exc}")
if (directory / "http/http-verification.json").is_file() and "observation" not in row:
try:
row["observation"] = observe_case(directory, case,
read_json(directory / "http/http-verification.json"))
except Exception as parse_error:
row["observation_error"] = f"{type(parse_error).__name__}: {parse_error}"
finally:
write_json(output / "state.json", state)
save_report(plan_path, plan, state, output)
print(json.dumps({"case": row["id"], "state": row["state"],
"status": row.get("observation", {}).get("status"),
"reason": row.get("observation", {}).get("reason_code"),
"representation_usable": row.get("usability", {}).get("passed")}), flush=True)
active = None
check_frozen(plan_path, plan)
state["state"] = "finished"
except BaseException as exc:
state.update(state="interrupted" if isinstance(exc, KeyboardInterrupt) else "error",
error=f"{type(exc).__name__}: {exc}")
if active is not None and active["state"] == "running":
active.update(state="interrupted", error=state["error"])
finally:
signal.signal(signal.SIGTERM, previous)
write_json(output / "state.json", state)
report = save_report(plan_path, plan, state, output)
return report
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
pre = commands.add_parser("prepare", help="Freeze a protocol before executing its cases")
pre.add_argument("--audio", type=Path, required=True)
pre.add_argument("--request", type=Path, required=True)
pre.add_argument("--operator", type=Path, required=True)
pre.add_argument("--output", type=Path, required=True)
pre.add_argument("--source-note", required=True)
pre.add_argument("--scope-note", required=True)
pre.add_argument("--frozen-input", type=Path, action="append", default=[])
pre.add_argument("--job-max-runtime-s", type=float)
pre.add_argument("--http-timeout-s", type=float)
execute = commands.add_parser("run", help="Execute fresh jobs; never resume old scientific work")
execute.add_argument("--plan", type=Path, required=True)
execute.add_argument("--output", type=Path, required=True)
execute.add_argument("--case", help="One-arm reproducer; never counts as the full matrix")
report = commands.add_parser("report", help="Compare preserved observations without executing the pipeline")
report.add_argument("--plan", type=Path, required=True)
report.add_argument("--output", type=Path, required=True)
args = parser.parse_args(argv)
try:
if args.command == "prepare":
path = prepare(args.audio, read_json(args.request), read_json(args.operator), args.output,
source_note=args.source_note, scope_note=args.scope_note, additional_inputs=args.frozen_input,
job_max_runtime_s=args.job_max_runtime_s, http_timeout_s=args.http_timeout_s)
print(path)
return 0
if args.command == "run":
result = run(args.plan, args.output, selected_case=args.case)
else:
plan_path = args.plan.resolve(strict=True)
plan = read_json(plan_path)
if http_verifier.sha256(plan_path) != plan_path.with_name("plan.sha256").read_text().strip():
raise ValueError("The frozen plan changed")
if http_verifier.sha256(__file__) != plan["implementation"]["runner_sha256"]:
raise ValueError("Use the frozen runner to reproduce these comparisons")
result = save_report(plan_path, plan, read_json(args.output / "state.json"), args.output)
print(json.dumps({key: result[key] for key in
("run_state", "finished_cases", "full_pipeline_cases", "pairs_with_observed_differences",
"all_complete_usable_and_within_tolerances")}))
return 0 if result["all_complete_usable_and_within_tolerances"] else 1
except (OSError, ValueError, TypeError, subprocess.SubprocessError) as exc:
parser.error(str(exc))
if __name__ == "__main__":
raise SystemExit(main())