Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
feat: continue saved searches with compatible staged scoring (#7)
Cinna Davisc023bee89fd4
diff --git a/src/dopplergeist/automatic.py b/src/dopplergeist/automatic.pyindex 0ed79b9..1a62b3f 100644--- a/src/dopplergeist/automatic.py+++ b/src/dopplergeist/automatic.py@@ -276,6 +276,8 @@ def _search(request, recording, events, output, config, search_progress):scoring = {"mode": "robust_ridge", "c": c, "time_origin": lo if lo == hi else None,"time_sweep_s": 0 if lo == hi else 5, "time_step_s": 0.5,"recording_start_window_s": [lo, hi]}+ if config.get("resume_legacy_first_batch"):+ scoring["legacy_first_batch"] = Truereport = search_national(indexes, terrain, prepared, measurements, time_range_s=time_range, output_dir=output / "search",bounds=bounds, budget=SearchBudget(**config.get("search_budget", {})),scoring=scoring, filters=filters, input_hashes=hashes,diff --git a/src/dopplergeist/batch_ridge.py b/src/dopplergeist/batch_ridge.pyindex 78995bd..ae2d257 100644--- a/src/dopplergeist/batch_ridge.py+++ b/src/dopplergeist/batch_ridge.py@@ -1,10 +1,10 @@"""Batched robust ridge fits with the scalar model's arithmetic order."""import numpy as np-from .matching import _frequency_scale, _ridge_fit_inputs+from .matching import _fit_ridge_uncached, _frequency_scale, _ridge_fit_inputs-def first_event_fits(measurement, event, factors, emissions):+def first_event_fits(measurement, event, factors, emissions, *, legacy_arithmetic=False):"""Fit supported clocks together, or return None for the ordinary fitter.This uses the same six iterations, box constraints, and diagnostics as the@@ -23,7 +23,7 @@ def first_event_fits(measurement, event, factors, emissions):return Nonelow, high = event["f0_range_hz"]# Keep scalar numeric promotion for noncanonical parameter types.- if any(type(value) not in (int, float) for value in (sigma, fraction, low, high)):+ if not legacy_arithmetic and any(type(value) not in (int, float) for value in (sigma, fraction, low, high)):return Noney, w = ridge[measured], weights[measured]independent = len(np.unique(times[measured]))@@ -40,8 +40,9 @@ def first_event_fits(measurement, event, factors, emissions):# The scalar fitter squares Python floats here. NumPy's array square# can differ by one ULP, which changes later IRLS steps and mode scores.try:- prior = np.fromiter(((2 * sigma / float(value))**2 for value in cap),- dtype=float, count=len(cap))+ prior = ((2 * sigma / cap)**2 if legacy_arithmetic else+ np.fromiter(((2 * sigma / float(value))**2 for value in cap),+ dtype=float, count=len(cap)))except (OverflowError, ZeroDivisionError):return Nonefor iteration in range(6):@@ -80,9 +81,10 @@ def first_event_fits(measurement, event, factors, emissions):effective = w / residual_scaleloss = np.average(2 * sigma**2 * (residual_scale - 1), weights=w, axis=1)try:- penalty = np.fromiter(- ((2 * sigma * float(drift) / float(bound))**2 / independent- for drift, bound in zip(beta, cap)), dtype=float, count=len(cap))+ penalty = ((2 * sigma * beta / cap)**2 / independent if legacy_arithmetic else+ np.fromiter(+ ((2 * sigma * float(drift) / float(bound))**2 / independent+ for drift, bound in zip(beta, cap)), dtype=float, count=len(cap)))except (OverflowError, ZeroDivisionError):return Nonescore = -np.sqrt(loss + penalty)@@ -97,4 +99,13 @@ def first_event_fits(measurement, event, factors, emissions):"rmse_hz": float(rmse[i]), "support_fraction": 1.0, "sample_count": int(len(y)),"independent_sample_count": independent, "outlier_fraction": float(outlier[i])}for i in range(len(alpha))]+ if legacy_arithmetic:+ # Saved searches before the scalar-consistent correction used this+ # array arithmetic and these three fallback checks for the first group.+ # Preserve their exact scores while batching later groups normally.+ for i in {0, len(results)//2, len(results)-1}:+ scalar = _fit_ridge_uncached(None, None, measurement, 0., event["f0_range_hz"],+ 343., sigma, fraction, acoustic_result=(factors[i], emissions[i]))+ if results[i] != scalar:+ return Nonereturn resultsdiff --git a/src/dopplergeist/checkpoint_resume.py b/src/dopplergeist/checkpoint_resume.pynew file mode 100644index 0000000..68faee0--- /dev/null+++ b/src/dopplergeist/checkpoint_resume.py@@ -0,0 +1,128 @@+"""Upgrade the known pre-batching save without changing its scoring arithmetic."""+from copy import deepcopy+from datetime import datetime, timezone+import json+from importlib.metadata import version+from pathlib import Path+import platform+import shutil+import sqlite3+import uuid++from .data_access import content_hash, write_json+++# Scoring implementation of 225bf6b, including the old first-group batch rule.+# Other saved implementations retain the ordinary strict resume checks.+LEGACY_SOURCES = {+ "matching.py": "1a0df93bd2ab9bf67253680d0933d5900eb2b5e4ba9346f17c8180565922cc6c",+ "search.py": "d7927fd0e0a5373c45be16812bbd29439d66a225eb8c8e744ced65158f4a9f09",+ "batch_ridge.py": "0042a55f576d385f39aeec9f3cbbfa14ffa70a58283437b14530cfb72d58a59f",+ "geometry.py": "71ea6ca7402ae91a55efdb0f8a307d10b80f8218350f077fecaa850523c1ce41",+ "tiled_search.py": "cd6eccc357241c1eec0f1d800f3aea3a2691b74b2736625da44ebe4686f15150",+ "acoustic_screen.py": "fc17f54913c066a2dedfd0a9ed0699cf9f93d0a72cb2c5a89e7803542a207d95",+ "finite_search.py": "78ed9fca6cb13c9754ea0f4438f1bb54bbcdc488ead2a4d77a15f149b6d77b1c",+}+++def upgrade_legacy_job(directory):+ """Prepare a replacement search before swapping it into a stopped job.++ Completed payloads and retention records stay unchanged. The old directory+ and audio/operator identities remain available for rollback. New work uses+ staged batching with the saved first-group arithmetic and fallback checks.+ """+ from .automatic import _audio_identity, _digest+ from . import matching++ directory = Path(directory)+ search = directory / "search"+ if not (search / "manifest.json").exists():+ return None+ manifest = json.loads((search / "manifest.json").read_text())+ old = manifest["configuration"]+ if any(old["code"].get(name) != digest for name, digest in LEGACY_SOURCES.items()):+ return None+ if getattr(matching, "_native_fit_irls", None) is not None:+ raise ValueError("Legacy checkpoint continuation requires the Python ridge fitter")+ if old["scoring"].get("mode") != "robust_ridge" or old.get("coarse_floor_policy") != "per_tile":+ raise ValueError("This legacy checkpoint mode has no compatible upgrade")+ runtime = {"python": platform.python_version(), "machine": platform.machine(),+ "packages": {name: version(name) for name in old["runtime"]["packages"]}}+ if runtime != old["runtime"]:+ raise ValueError("Saved search runtime changed; the legacy arithmetic requires its original runtime")+ state = json.loads((directory / "result.json").read_text())+ if state["status"] not in {"cancelled", "failed", "inconclusive"}:+ raise ValueError("Stop the previous job before upgrading its checkpoints")+ request = json.loads((directory / "request.json").read_text())+ operator = json.loads((directory / "operator.json").read_text())+ audio = json.loads((directory / "resume.json").read_text())+ identity = _audio_identity(request, operator)+ for key in set(audio["identity"]) | set(identity):+ if key != "modules" and audio["identity"].get(key) != identity.get(key):+ raise ValueError("Saved audio, request, settings or runtime changed")+ if any(_digest(directory / name) != digest for name, digest in audio["files"].items()):+ raise ValueError("Saved audio checkpoint changed")+ discovery = json.loads((directory / "discovery/discovery.json").read_text())+ events, _ = matching.prepare_events(discovery["events"])+ if events != old["events"]:+ raise ValueError("Prepared events no longer match the saved search")+ current = deepcopy(old)+ current["code"] = {p.name: _digest(p) for p in sorted(Path(__file__).parent.glob("*.py"))}+ current["scoring"]["legacy_first_batch"] = True+ new_hash = content_hash(current)+ suffix = uuid.uuid4().hex[:12]+ staged = directory / f".search-upgrade-{suffix}"+ previous = directory / f"search-before-upgrade-{suffix}"+ backup = directory / f"checkpoint-upgrade-{suffix}"+ backup.mkdir()+ for name in ("resume.json", "operator.json", "result.json", "provenance.json"):+ if (directory / name).exists():+ shutil.copy2(directory / name, backup / name)++ def ignore(path, names):+ return [name for name in names if name == ".tile-workers" or name.endswith(".tmp")+ or name.startswith("retention.sqlite")]++ # JSON/support files are immutable until atomically replaced. Hard links+ # retain their original bytes without copying the whole saved search twice.+ shutil.copytree(search, staged, copy_function=lambda src, dst: Path(dst).hardlink_to(src), ignore=ignore)+ with sqlite3.connect((search / "retention.sqlite").resolve().as_uri() + "?mode=ro", uri=True) as source:+ if source.execute("SELECT identity FROM run").fetchone() != (manifest["run_hash"],):+ raise ValueError("Saved retention ledger identity differs from its manifest")+ with sqlite3.connect(staged / "retention.sqlite") as target:+ source.backup(target)+ target.execute("UPDATE run SET identity=?", (new_hash,))+ tiles = 0+ for pattern in ("tile-*.json", "split-*.json"):+ for path in staged.glob(pattern):+ record = json.loads(path.read_text())+ if record["run_hash"] != manifest["run_hash"]:+ raise ValueError(f"Saved checkpoint has a different run identity: {path.name}")+ record["run_hash"] = new_hash+ write_json(path, record)+ tiles += pattern == "tile-*.json"+ write_json(staged / "manifest.json", {"run_hash": new_hash, "configuration": current})+ report = {"version": 1, "created_utc": datetime.now(timezone.utc).isoformat(),+ "old_run_hash": manifest["run_hash"], "new_run_hash": new_hash,+ "tiles_preserved": tiles, "legacy_first_batch": True,+ "previous_search": str(previous), "identity_backup": str(backup)}+ write_json(backup / "upgrade.json", report)+ search.rename(previous)+ try:+ staged.rename(search)+ audio["identity"] = identity+ operator["resume_legacy_first_batch"] = True+ write_json(directory / "resume.json", audio)+ write_json(directory / "operator.json", operator)+ write_json(directory / "checkpoint-upgrade.json", report)+ except BaseException:+ # No worker has been launched yet. Restore the original save if the+ # directory/identity commit cannot finish.+ if search.exists():+ search.rename(staged)+ previous.rename(search)+ for name in ("resume.json", "operator.json"):+ shutil.copy2(backup / name, directory / name)+ raise+ return reportdiff --git a/src/dopplergeist/jobs.py b/src/dopplergeist/jobs.pyindex 444d8ea..62c1025 100644--- a/src/dopplergeist/jobs.py+++ b/src/dopplergeist/jobs.py@@ -204,6 +204,8 @@ class JobManager:raise ValueError("The previous worker is still stopping; try again")if sum(thread.is_alive() for thread in self._threads.values()) >= self.max_pending:raise OverflowError("Job queue is full; try again after a job finishes")+ from .checkpoint_resume import upgrade_legacy_job+ upgrade_legacy_job(directory)from .automatic import _load_audio_checkpoint_load_audio_checkpoint(directory, read_json(directory / "request.json"), read_json(directory / "operator.json"))write_json(directory / f"result-before-resume-{uuid.uuid4().hex}.json", current)diff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.pyindex fb717cb..428d981 100644--- a/src/dopplergeist/search.py+++ b/src/dopplergeist/search.py@@ -450,7 +450,7 @@ def _ridge_result(diagnostics):def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,- acoustic_batches, score_floor, use_dual):+ acoustic_batches, score_floor, use_dual, legacy_first_batch=False):"""Batch only clocks that survive the preceding event's existing screen.Event order within each clock is unchanged. A later event sees precisely the@@ -491,7 +491,8 @@ def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,fitted = Noneif batch is not None and len(eligible) >= 4:fitted = first_event_fits(measurement, event,- batch[0][eligible], batch[1][eligible])+ batch[0][eligible], batch[1][eligible],+ legacy_arithmetic=legacy_first_batch and index == 0)remaining = len(events) - index - 1next_active = []for fit_index, trial in enumerate(eligible):@@ -535,7 +536,7 @@ def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,def score_candidate(candidate, point, tracks, events, measurements, mode="brightness",time_origin=None, time_sweep_s=5.0, time_step_s=0.5, c=343.0,- score_floor=None):+ score_floor=None, legacy_first_batch=False):"""Score one hypothesis; a bounded score-floor result has no numeric score."""all_events, all_measurements = events, measurementsassociation = "event_indices" in candidate@@ -630,7 +631,8 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="brightselected_tracks, events, measurements, observer, anchor + shifts, c,lazy_batches if lazy_batches is not None else batches,score_floor if association else None,- association and score_floor is not None and time_origin is None)+ association and score_floor is not None and time_origin is None,+ legacy_first_batch and association and score_floor is not None and time_origin is None)for trial, shift in enumerate(() if floor_pruned else shifts):if mode == "robust_ridge":from .matching import score_ridges@@ -688,7 +690,8 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="brightfrom .batch_ridge import first_event_fitsfactors, emissions = batches[0]fitted = first_event_fits(measurements[0], events[0],- factors[eligible], emissions[eligible])+ factors[eligible], emissions[eligible],+ legacy_arithmetic=legacy_first_batch)if fitted is not None:first_fits = [None] * len(shifts)for index, fit in zip(eligible, fitted):diff --git a/tests/test_real_checkpoint_upgrade.py b/tests/test_real_checkpoint_upgrade.pynew file mode 100644index 0000000..a3d13dc--- /dev/null+++ b/tests/test_real_checkpoint_upgrade.py@@ -0,0 +1,56 @@+"""Upgrade a subset of an actual stopped job; no recording or tracks generated."""+import json+import os+from pathlib import Path+import shutil+import sqlite3+import tempfile++import pytest++from dopplergeist.automatic import _load_audio_checkpoint+from dopplergeist.checkpoint_resume import upgrade_legacy_job+from dopplergeist.finite_search import RetentionLedger+++def test_saved_scored_tile_and_ledger_survive_upgrade(request):+ location = os.environ.get("DOPPLERGEIST_REAL_SAVED_JOB")+ if not location:+ pytest.skip("Supply the actual stopped pre-batching job")+ source = Path(location)+ temporary = tempfile.TemporaryDirectory(prefix="checkpoint-upgrade-test-", dir=source.parent.parent)+ request.addfinalizer(temporary.cleanup)+ tmp_path = Path(temporary.name)+ for name in ("request.json", "operator.json", "resume.json", "result.json"):+ shutil.copy2(source / name, tmp_path / name)+ for name in ("intake/recording.f32", "intake/recording.json", "intake/intake.json",+ "discovery/discovery.json", "search/manifest.json", "search/retention.sqlite"):+ path = tmp_path / name+ path.parent.mkdir(parents=True, exist_ok=True)+ path.hardlink_to(source / name)+ tile = source / "search/tile-30_7.json"+ if not tile.exists():+ pytest.skip("The saved real scored tile is unavailable")+ original = tile.read_bytes()+ payload = json.loads(original)["payload"]+ assert payload["counts"]["candidates"] > 0+ (tmp_path / "search" / tile.name).hardlink_to(tile)+ with sqlite3.connect(source / "search/retention.sqlite") as db:+ counts = {name: db.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0]+ for name in ("stages", "records", "bounds")}+ report = upgrade_legacy_job(tmp_path)+ assert report["tiles_preserved"] == 1+ assert json.loads((tmp_path / "search" / tile.name).read_text())["payload"] == payload+ assert (Path(report["previous_search"]) / tile.name).read_bytes() == original+ assert tile.read_bytes() == original+ with sqlite3.connect(tmp_path / "search/retention.sqlite") as db:+ assert db.execute("SELECT identity FROM run").fetchone() == (report["new_run_hash"],)+ assert {name: db.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0]+ for name in counts} == counts+ ledger = RetentionLedger(tmp_path / "search/retention.sqlite", report["new_run_hash"],+ "matched_group_support_minus_normalized_residual")+ assert ledger.stage_valid("coarse:30_7", payload["retention_checksum"])+ operator = json.loads((tmp_path / "operator.json").read_text())+ assert operator["resume_legacy_first_batch"] is True+ _load_audio_checkpoint(tmp_path, json.loads((tmp_path / "request.json").read_text()), operator)+ assert upgrade_legacy_job(tmp_path) is None