Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

feat: continue saved searches with compatible staged scoring (#7)

Cinna Davisc023bee89fd4
diff --git a/src/dopplergeist/automatic.py b/src/dopplergeist/automatic.py
index 0ed79b9..1a62b3f 100644
--- a/src/dopplergeist/automatic.py
+++ b/src/dopplergeist/automatic.py
@@ -276,6 +276,8 @@ def _search(request, recording, events, output, config, search_progress):
scoring = {"mode": "robust_ridge", "c": c, "time_origin": lo if lo == hi else None,
"time_sweep_s": 0 if lo == hi else 5, "time_step_s": 0.5,
"recording_start_window_s": [lo, hi]}
+ if config.get("resume_legacy_first_batch"):
+ scoring["legacy_first_batch"] = True
report = search_national(indexes, terrain, prepared, measurements, time_range_s=time_range, output_dir=output / "search",
bounds=bounds, budget=SearchBudget(**config.get("search_budget", {})),
scoring=scoring, filters=filters, input_hashes=hashes,
diff --git a/src/dopplergeist/batch_ridge.py b/src/dopplergeist/batch_ridge.py
index 78995bd..ae2d257 100644
--- a/src/dopplergeist/batch_ridge.py
+++ b/src/dopplergeist/batch_ridge.py
@@ -1,10 +1,10 @@
"""Batched robust ridge fits with the scalar model's arithmetic order."""
import numpy as np
-from .matching import _frequency_scale, _ridge_fit_inputs
+from .matching import _fit_ridge_uncached, _frequency_scale, _ridge_fit_inputs
-def first_event_fits(measurement, event, factors, emissions):
+def first_event_fits(measurement, event, factors, emissions, *, legacy_arithmetic=False):
"""Fit supported clocks together, or return None for the ordinary fitter.
This uses the same six iterations, box constraints, and diagnostics as the
@@ -23,7 +23,7 @@ def first_event_fits(measurement, event, factors, emissions):
return None
low, high = event["f0_range_hz"]
# Keep scalar numeric promotion for noncanonical parameter types.
- if any(type(value) not in (int, float) for value in (sigma, fraction, low, high)):
+ if not legacy_arithmetic and any(type(value) not in (int, float) for value in (sigma, fraction, low, high)):
return None
y, w = ridge[measured], weights[measured]
independent = len(np.unique(times[measured]))
@@ -40,8 +40,9 @@ def first_event_fits(measurement, event, factors, emissions):
# The scalar fitter squares Python floats here. NumPy's array square
# can differ by one ULP, which changes later IRLS steps and mode scores.
try:
- prior = np.fromiter(((2 * sigma / float(value))**2 for value in cap),
- dtype=float, count=len(cap))
+ prior = ((2 * sigma / cap)**2 if legacy_arithmetic else
+ np.fromiter(((2 * sigma / float(value))**2 for value in cap),
+ dtype=float, count=len(cap)))
except (OverflowError, ZeroDivisionError):
return None
for iteration in range(6):
@@ -80,9 +81,10 @@ def first_event_fits(measurement, event, factors, emissions):
effective = w / residual_scale
loss = np.average(2 * sigma**2 * (residual_scale - 1), weights=w, axis=1)
try:
- penalty = np.fromiter(
- ((2 * sigma * float(drift) / float(bound))**2 / independent
- for drift, bound in zip(beta, cap)), dtype=float, count=len(cap))
+ penalty = ((2 * sigma * beta / cap)**2 / independent if legacy_arithmetic else
+ np.fromiter(
+ ((2 * sigma * float(drift) / float(bound))**2 / independent
+ for drift, bound in zip(beta, cap)), dtype=float, count=len(cap)))
except (OverflowError, ZeroDivisionError):
return None
score = -np.sqrt(loss + penalty)
@@ -97,4 +99,13 @@ def first_event_fits(measurement, event, factors, emissions):
"rmse_hz": float(rmse[i]), "support_fraction": 1.0, "sample_count": int(len(y)),
"independent_sample_count": independent, "outlier_fraction": float(outlier[i])}
for i in range(len(alpha))]
+ if legacy_arithmetic:
+ # Saved searches before the scalar-consistent correction used this
+ # array arithmetic and these three fallback checks for the first group.
+ # Preserve their exact scores while batching later groups normally.
+ for i in {0, len(results)//2, len(results)-1}:
+ scalar = _fit_ridge_uncached(None, None, measurement, 0., event["f0_range_hz"],
+ 343., sigma, fraction, acoustic_result=(factors[i], emissions[i]))
+ if results[i] != scalar:
+ return None
return results
diff --git a/src/dopplergeist/checkpoint_resume.py b/src/dopplergeist/checkpoint_resume.py
new file mode 100644
index 0000000..68faee0
--- /dev/null
+++ b/src/dopplergeist/checkpoint_resume.py
@@ -0,0 +1,128 @@
+"""Upgrade the known pre-batching save without changing its scoring arithmetic."""
+from copy import deepcopy
+from datetime import datetime, timezone
+import json
+from importlib.metadata import version
+from pathlib import Path
+import platform
+import shutil
+import sqlite3
+import uuid
+
+from .data_access import content_hash, write_json
+
+
+# Scoring implementation of 225bf6b, including the old first-group batch rule.
+# Other saved implementations retain the ordinary strict resume checks.
+LEGACY_SOURCES = {
+ "matching.py": "1a0df93bd2ab9bf67253680d0933d5900eb2b5e4ba9346f17c8180565922cc6c",
+ "search.py": "d7927fd0e0a5373c45be16812bbd29439d66a225eb8c8e744ced65158f4a9f09",
+ "batch_ridge.py": "0042a55f576d385f39aeec9f3cbbfa14ffa70a58283437b14530cfb72d58a59f",
+ "geometry.py": "71ea6ca7402ae91a55efdb0f8a307d10b80f8218350f077fecaa850523c1ce41",
+ "tiled_search.py": "cd6eccc357241c1eec0f1d800f3aea3a2691b74b2736625da44ebe4686f15150",
+ "acoustic_screen.py": "fc17f54913c066a2dedfd0a9ed0699cf9f93d0a72cb2c5a89e7803542a207d95",
+ "finite_search.py": "78ed9fca6cb13c9754ea0f4438f1bb54bbcdc488ead2a4d77a15f149b6d77b1c",
+}
+
+
+def upgrade_legacy_job(directory):
+ """Prepare a replacement search before swapping it into a stopped job.
+
+ Completed payloads and retention records stay unchanged. The old directory
+ and audio/operator identities remain available for rollback. New work uses
+ staged batching with the saved first-group arithmetic and fallback checks.
+ """
+ from .automatic import _audio_identity, _digest
+ from . import matching
+
+ directory = Path(directory)
+ search = directory / "search"
+ if not (search / "manifest.json").exists():
+ return None
+ manifest = json.loads((search / "manifest.json").read_text())
+ old = manifest["configuration"]
+ if any(old["code"].get(name) != digest for name, digest in LEGACY_SOURCES.items()):
+ return None
+ if getattr(matching, "_native_fit_irls", None) is not None:
+ raise ValueError("Legacy checkpoint continuation requires the Python ridge fitter")
+ if old["scoring"].get("mode") != "robust_ridge" or old.get("coarse_floor_policy") != "per_tile":
+ raise ValueError("This legacy checkpoint mode has no compatible upgrade")
+ runtime = {"python": platform.python_version(), "machine": platform.machine(),
+ "packages": {name: version(name) for name in old["runtime"]["packages"]}}
+ if runtime != old["runtime"]:
+ raise ValueError("Saved search runtime changed; the legacy arithmetic requires its original runtime")
+ state = json.loads((directory / "result.json").read_text())
+ if state["status"] not in {"cancelled", "failed", "inconclusive"}:
+ raise ValueError("Stop the previous job before upgrading its checkpoints")
+ request = json.loads((directory / "request.json").read_text())
+ operator = json.loads((directory / "operator.json").read_text())
+ audio = json.loads((directory / "resume.json").read_text())
+ identity = _audio_identity(request, operator)
+ for key in set(audio["identity"]) | set(identity):
+ if key != "modules" and audio["identity"].get(key) != identity.get(key):
+ raise ValueError("Saved audio, request, settings or runtime changed")
+ if any(_digest(directory / name) != digest for name, digest in audio["files"].items()):
+ raise ValueError("Saved audio checkpoint changed")
+ discovery = json.loads((directory / "discovery/discovery.json").read_text())
+ events, _ = matching.prepare_events(discovery["events"])
+ if events != old["events"]:
+ raise ValueError("Prepared events no longer match the saved search")
+ current = deepcopy(old)
+ current["code"] = {p.name: _digest(p) for p in sorted(Path(__file__).parent.glob("*.py"))}
+ current["scoring"]["legacy_first_batch"] = True
+ new_hash = content_hash(current)
+ suffix = uuid.uuid4().hex[:12]
+ staged = directory / f".search-upgrade-{suffix}"
+ previous = directory / f"search-before-upgrade-{suffix}"
+ backup = directory / f"checkpoint-upgrade-{suffix}"
+ backup.mkdir()
+ for name in ("resume.json", "operator.json", "result.json", "provenance.json"):
+ if (directory / name).exists():
+ shutil.copy2(directory / name, backup / name)
+
+ def ignore(path, names):
+ return [name for name in names if name == ".tile-workers" or name.endswith(".tmp")
+ or name.startswith("retention.sqlite")]
+
+ # JSON/support files are immutable until atomically replaced. Hard links
+ # retain their original bytes without copying the whole saved search twice.
+ shutil.copytree(search, staged, copy_function=lambda src, dst: Path(dst).hardlink_to(src), ignore=ignore)
+ with sqlite3.connect((search / "retention.sqlite").resolve().as_uri() + "?mode=ro", uri=True) as source:
+ if source.execute("SELECT identity FROM run").fetchone() != (manifest["run_hash"],):
+ raise ValueError("Saved retention ledger identity differs from its manifest")
+ with sqlite3.connect(staged / "retention.sqlite") as target:
+ source.backup(target)
+ target.execute("UPDATE run SET identity=?", (new_hash,))
+ tiles = 0
+ for pattern in ("tile-*.json", "split-*.json"):
+ for path in staged.glob(pattern):
+ record = json.loads(path.read_text())
+ if record["run_hash"] != manifest["run_hash"]:
+ raise ValueError(f"Saved checkpoint has a different run identity: {path.name}")
+ record["run_hash"] = new_hash
+ write_json(path, record)
+ tiles += pattern == "tile-*.json"
+ write_json(staged / "manifest.json", {"run_hash": new_hash, "configuration": current})
+ report = {"version": 1, "created_utc": datetime.now(timezone.utc).isoformat(),
+ "old_run_hash": manifest["run_hash"], "new_run_hash": new_hash,
+ "tiles_preserved": tiles, "legacy_first_batch": True,
+ "previous_search": str(previous), "identity_backup": str(backup)}
+ write_json(backup / "upgrade.json", report)
+ search.rename(previous)
+ try:
+ staged.rename(search)
+ audio["identity"] = identity
+ operator["resume_legacy_first_batch"] = True
+ write_json(directory / "resume.json", audio)
+ write_json(directory / "operator.json", operator)
+ write_json(directory / "checkpoint-upgrade.json", report)
+ except BaseException:
+ # No worker has been launched yet. Restore the original save if the
+ # directory/identity commit cannot finish.
+ if search.exists():
+ search.rename(staged)
+ previous.rename(search)
+ for name in ("resume.json", "operator.json"):
+ shutil.copy2(backup / name, directory / name)
+ raise
+ return report
diff --git a/src/dopplergeist/jobs.py b/src/dopplergeist/jobs.py
index 444d8ea..62c1025 100644
--- a/src/dopplergeist/jobs.py
+++ b/src/dopplergeist/jobs.py
@@ -204,6 +204,8 @@ class JobManager:
raise ValueError("The previous worker is still stopping; try again")
if sum(thread.is_alive() for thread in self._threads.values()) >= self.max_pending:
raise OverflowError("Job queue is full; try again after a job finishes")
+ from .checkpoint_resume import upgrade_legacy_job
+ upgrade_legacy_job(directory)
from .automatic import _load_audio_checkpoint
_load_audio_checkpoint(directory, read_json(directory / "request.json"), read_json(directory / "operator.json"))
write_json(directory / f"result-before-resume-{uuid.uuid4().hex}.json", current)
diff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.py
index fb717cb..428d981 100644
--- a/src/dopplergeist/search.py
+++ b/src/dopplergeist/search.py
@@ -450,7 +450,7 @@ def _ridge_result(diagnostics):
def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,
- acoustic_batches, score_floor, use_dual):
+ acoustic_batches, score_floor, use_dual, legacy_first_batch=False):
"""Batch only clocks that survive the preceding event's existing screen.
Event order within each clock is unchanged. A later event sees precisely the
@@ -491,7 +491,8 @@ def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,
fitted = None
if batch is not None and len(eligible) >= 4:
fitted = first_event_fits(measurement, event,
- batch[0][eligible], batch[1][eligible])
+ batch[0][eligible], batch[1][eligible],
+ legacy_arithmetic=legacy_first_batch and index == 0)
remaining = len(events) - index - 1
next_active = []
for fit_index, trial in enumerate(eligible):
@@ -535,7 +536,7 @@ def _batched_ridge_trials(tracks, events, measurements, observer, starts, c,
def score_candidate(candidate, point, tracks, events, measurements, mode="brightness",
time_origin=None, time_sweep_s=5.0, time_step_s=0.5, c=343.0,
- score_floor=None):
+ score_floor=None, legacy_first_batch=False):
"""Score one hypothesis; a bounded score-floor result has no numeric score."""
all_events, all_measurements = events, measurements
association = "event_indices" in candidate
@@ -630,7 +631,8 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="bright
selected_tracks, events, measurements, observer, anchor + shifts, c,
lazy_batches if lazy_batches is not None else batches,
score_floor if association else None,
- association and score_floor is not None and time_origin is None)
+ association and score_floor is not None and time_origin is None,
+ legacy_first_batch and association and score_floor is not None and time_origin is None)
for trial, shift in enumerate(() if floor_pruned else shifts):
if mode == "robust_ridge":
from .matching import score_ridges
@@ -688,7 +690,8 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="bright
from .batch_ridge import first_event_fits
factors, emissions = batches[0]
fitted = first_event_fits(measurements[0], events[0],
- factors[eligible], emissions[eligible])
+ factors[eligible], emissions[eligible],
+ legacy_arithmetic=legacy_first_batch)
if fitted is not None:
first_fits = [None] * len(shifts)
for index, fit in zip(eligible, fitted):
diff --git a/tests/test_real_checkpoint_upgrade.py b/tests/test_real_checkpoint_upgrade.py
new file mode 100644
index 0000000..a3d13dc
--- /dev/null
+++ b/tests/test_real_checkpoint_upgrade.py
@@ -0,0 +1,56 @@
+"""Upgrade a subset of an actual stopped job; no recording or tracks generated."""
+import json
+import os
+from pathlib import Path
+import shutil
+import sqlite3
+import tempfile
+
+import pytest
+
+from dopplergeist.automatic import _load_audio_checkpoint
+from dopplergeist.checkpoint_resume import upgrade_legacy_job
+from dopplergeist.finite_search import RetentionLedger
+
+
+def test_saved_scored_tile_and_ledger_survive_upgrade(request):
+ location = os.environ.get("DOPPLERGEIST_REAL_SAVED_JOB")
+ if not location:
+ pytest.skip("Supply the actual stopped pre-batching job")
+ source = Path(location)
+ temporary = tempfile.TemporaryDirectory(prefix="checkpoint-upgrade-test-", dir=source.parent.parent)
+ request.addfinalizer(temporary.cleanup)
+ tmp_path = Path(temporary.name)
+ for name in ("request.json", "operator.json", "resume.json", "result.json"):
+ shutil.copy2(source / name, tmp_path / name)
+ for name in ("intake/recording.f32", "intake/recording.json", "intake/intake.json",
+ "discovery/discovery.json", "search/manifest.json", "search/retention.sqlite"):
+ path = tmp_path / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.hardlink_to(source / name)
+ tile = source / "search/tile-30_7.json"
+ if not tile.exists():
+ pytest.skip("The saved real scored tile is unavailable")
+ original = tile.read_bytes()
+ payload = json.loads(original)["payload"]
+ assert payload["counts"]["candidates"] > 0
+ (tmp_path / "search" / tile.name).hardlink_to(tile)
+ with sqlite3.connect(source / "search/retention.sqlite") as db:
+ counts = {name: db.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0]
+ for name in ("stages", "records", "bounds")}
+ report = upgrade_legacy_job(tmp_path)
+ assert report["tiles_preserved"] == 1
+ assert json.loads((tmp_path / "search" / tile.name).read_text())["payload"] == payload
+ assert (Path(report["previous_search"]) / tile.name).read_bytes() == original
+ assert tile.read_bytes() == original
+ with sqlite3.connect(tmp_path / "search/retention.sqlite") as db:
+ assert db.execute("SELECT identity FROM run").fetchone() == (report["new_run_hash"],)
+ assert {name: db.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0]
+ for name in counts} == counts
+ ledger = RetentionLedger(tmp_path / "search/retention.sqlite", report["new_run_hash"],
+ "matched_group_support_minus_normalized_residual")
+ assert ledger.stage_valid("coarse:30_7", payload["retention_checksum"])
+ operator = json.loads((tmp_path / "operator.json").read_text())
+ assert operator["resume_legacy_first_batch"] is True
+ _load_audio_checkpoint(tmp_path, json.loads((tmp_path / "request.json").read_text()), operator)
+ assert upgrade_legacy_job(tmp_path) is None