Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Continue the saved reference search with faster Python scoring (#263)

Cinna Davis611dca2a4398
diff --git a/src/dopplergeist/checkpoint_resume.py b/src/dopplergeist/checkpoint_resume.py
index 5ff63b4..31fbf1a 100644
--- a/src/dopplergeist/checkpoint_resume.py
+++ b/src/dopplergeist/checkpoint_resume.py
@@ -1,4 +1,4 @@
-"""Upgrade the known pre-batching save without changing its scoring arithmetic."""
+"""Continue known saved searches with exactly compatible Python scoring."""
from copy import deepcopy
from datetime import datetime, timezone
import json
@@ -25,6 +25,25 @@ LEGACY_SOURCES = {
}
+# The already upgraded reference search has been replayed exactly against this
+# Python scoring family. Keep this transition bound to its complete input hash.
+BATCHED_REFERENCE_RUN = "9890647b03bc1f966f3b039727e7c0f00ccef0b4250b7312d5b423c191ca3d3f"
+ORIGINAL_REFERENCE_RUN = "d1ef045417a64a792b81c7355e508bb204046a70b614fb5815aba14b74a75456"
+THROUGHPUT_SOURCES = {
+ "matching.py": "8c2ba9b32ce035f07a8357bf0da05f63a5ac693baad5f6dc52930a82f94439c9",
+ "search.py": "be84feaef43fa356e3e77e0132e1e427b677216f4c3e84166e566a1d9787c99e",
+ "batch_ridge.py": "314d4837fadd80f94410d21c8d5115fa65aff0515e3de2cf4bb08fefd427680a",
+ "geometry.py": "f30f6aa755387f7c28f4bb49d0750e7afc42551f3e5af34f9d8be87c5f0cdc26",
+ "tiled_search.py": "8b54a363611863e896cb86b388daa0fd5722161eac7b19ea8dbe1937b10c45ea",
+ "acoustic_screen.py": "ce2b4e86fed8dfba92ae4742f8cd273f80711976df6ca0a399f1f233d6a6ff32",
+ "finite_search.py": "f1bae3b5a947a3d3234df3b3b1bc8edecca5cf7682ed561d45ad38fe34681fe6",
+ "parallel_tiles.py": "6b5ff3def69b2433c00f9c4afbd1c52ccc0a79e0f623bdf0dff3f61c0d7ea16a",
+ "data_access.py": "1b599aadb8cafa6be0c6aeec959e0c450c1b5c515c24fad6e33cb97719bce6c1",
+ "tracks.py": "543f483cee756fdb9efdd69c8e8f05b32e290f19187af3523160f19d2eba90e0",
+ "native_acoustic.py": "a048cb3bd2d308c91995c97e587f181fbf6d74d6dd9598ab3ab3fd1cc4a50796",
+}
+
+
def upgrade_legacy_job(directory):
"""Prepare a replacement search before swapping it into a stopped job.
@@ -41,12 +60,30 @@ def upgrade_legacy_job(directory):
return None
manifest = json.loads((search / "manifest.json").read_text())
old = manifest["configuration"]
- if any(old["code"].get(name) != digest for name, digest in LEGACY_SOURCES.items()):
+ if content_hash(old) != manifest.get("run_hash"):
return None
+ previous_upgrade = None
+ if any(old["code"].get(name) != digest for name, digest in LEGACY_SOURCES.items()):
+ report_path = directory / "checkpoint-upgrade.json"
+ if manifest["run_hash"] != BATCHED_REFERENCE_RUN or not report_path.exists():
+ return None
+ previous_upgrade = json.loads(report_path.read_text())
+ if (previous_upgrade.get("version") != 1
+ or previous_upgrade.get("old_run_hash") != ORIGINAL_REFERENCE_RUN
+ or previous_upgrade.get("new_run_hash") != BATCHED_REFERENCE_RUN
+ or previous_upgrade.get("legacy_first_batch") is not True
+ or old["scoring"].get("legacy_first_batch") is not True):
+ return None
if getattr(matching, "_native_fit_irls", None) is not None:
raise ValueError("Legacy checkpoint continuation requires the Python ridge fitter")
if geometry.ACOUSTIC_BACKEND != "python-acoustic-v1":
raise ValueError("Legacy checkpoint continuation requires the Python acoustic solver")
+ from .provenance import implementation_files
+ package_root = Path(__file__).parent
+ code = {p.relative_to(package_root).as_posix(): _digest(p)
+ for p in implementation_files(package_root, recursive=False)}
+ if previous_upgrade and any(code.get(name) != digest for name, digest in THROUGHPUT_SOURCES.items()):
+ return None
if (old["scoring"].get("mode") != "robust_ridge" or old.get("coarse_floor_policy") != "per_tile"
or old["scoring"].get("time_origin") is not None
or old.get("filters", {}).get("association") != "subsets"):
@@ -72,10 +109,7 @@ def upgrade_legacy_job(directory):
if events != old["events"]:
raise ValueError("Prepared events no longer match the saved search")
current = deepcopy(old)
- from .provenance import implementation_files
- package_root = Path(__file__).parent
- current["code"] = {p.relative_to(package_root).as_posix(): _digest(p)
- for p in implementation_files(package_root, recursive=False)}
+ current["code"] = code
current["scoring"]["legacy_first_batch"] = True
new_hash = content_hash(current)
suffix = uuid.uuid4().hex[:12]
@@ -83,7 +117,7 @@ def upgrade_legacy_job(directory):
previous = directory / f"search-before-upgrade-{suffix}"
backup = directory / f"checkpoint-upgrade-{suffix}"
backup.mkdir()
- for name in ("resume.json", "operator.json", "result.json", "provenance.json"):
+ for name in ("resume.json", "operator.json", "result.json", "provenance.json", "checkpoint-upgrade.json"):
if (directory / name).exists():
shutil.copy2(directory / name, backup / name)
@@ -105,10 +139,15 @@ def upgrade_legacy_job(directory):
# each payload and retention stage before reuse.
tiles = len(list(staged.glob("tile-*.json")))
write_json(staged / "manifest.json", {"run_hash": new_hash, "configuration": current})
- report = {"version": 1, "created_utc": datetime.now(timezone.utc).isoformat(),
+ report = {"version": 2 if previous_upgrade else 1,
+ "created_utc": datetime.now(timezone.utc).isoformat(),
"old_run_hash": manifest["run_hash"], "new_run_hash": new_hash,
"tiles_preserved": tiles, "legacy_first_batch": True,
"previous_search": str(previous), "identity_backup": str(backup)}
+ if previous_upgrade:
+ report["compatible_run_hashes"] = sorted([ORIGINAL_REFERENCE_RUN, BATCHED_REFERENCE_RUN])
+ report["upgrade_kind"] = "exact_python_throughput_v1"
+ report["previous_upgrade_report"] = str(backup / "checkpoint-upgrade.json")
write_json(backup / "upgrade.json", report)
search.rename(previous)
try:
@@ -124,8 +163,12 @@ def upgrade_legacy_job(directory):
if search.exists():
search.rename(staged)
previous.rename(search)
- for name in ("resume.json", "operator.json"):
- shutil.copy2(backup / name, directory / name)
+ for name in ("resume.json", "operator.json", "checkpoint-upgrade.json"):
+ saved = backup / name
+ if saved.exists():
+ shutil.copy2(saved, directory / name)
+ else:
+ (directory / name).unlink(missing_ok=True)
raise
return report
@@ -136,7 +179,11 @@ def compatible_checkpoint_runs(output, run_hash):
path = Path(output).parent / "checkpoint-upgrade.json"
if path.exists():
report = json.loads(path.read_text())
- if (report.get("version") == 1 and report.get("new_run_hash") == run_hash
- and report.get("legacy_first_batch") is True):
- accepted.add(report["old_run_hash"])
+ if report.get("new_run_hash") == run_hash and report.get("legacy_first_batch") is True:
+ if report.get("version") == 1:
+ accepted.add(report["old_run_hash"])
+ elif (report.get("version") == 2 and report.get("upgrade_kind") == "exact_python_throughput_v1"
+ and report.get("old_run_hash") == BATCHED_REFERENCE_RUN
+ and report.get("compatible_run_hashes") == sorted([ORIGINAL_REFERENCE_RUN, BATCHED_REFERENCE_RUN])):
+ accepted.update([ORIGINAL_REFERENCE_RUN, BATCHED_REFERENCE_RUN])
return accepted
diff --git a/tests/test_real_checkpoint_upgrade.py b/tests/test_real_checkpoint_upgrade.py
index e57c2d3..aa6d756 100644
--- a/tests/test_real_checkpoint_upgrade.py
+++ b/tests/test_real_checkpoint_upgrade.py
@@ -8,13 +8,16 @@ import tempfile
import pytest
+import dopplergeist.checkpoint_resume as checkpoint_resume
from dopplergeist.automatic import _load_audio_checkpoint
-from dopplergeist.checkpoint_resume import compatible_checkpoint_runs, upgrade_legacy_job
+from dopplergeist.checkpoint_resume import (BATCHED_REFERENCE_RUN, ORIGINAL_REFERENCE_RUN,
+ compatible_checkpoint_runs, upgrade_legacy_job)
from dopplergeist.finite_search import RetentionLedger
-def test_saved_scored_tile_and_ledger_survive_upgrade(request):
- location = os.environ.get("DOPPLERGEIST_REAL_SAVED_JOB")
[email protected]("source_variable", ["DOPPLERGEIST_REAL_SAVED_JOB", "DOPPLERGEIST_REAL_BATCHED_JOB"])
+def test_saved_scored_tile_and_ledger_survive_upgrade(request, source_variable):
+ location = os.environ.get(source_variable)
if not location:
pytest.skip("Supply the actual stopped pre-batching job")
source = Path(location)
@@ -23,6 +26,8 @@ def test_saved_scored_tile_and_ledger_survive_upgrade(request):
tmp_path = Path(temporary.name)
for name in ("request.json", "operator.json", "resume.json", "result.json"):
shutil.copy2(source / name, tmp_path / name)
+ if (source / "checkpoint-upgrade.json").exists():
+ shutil.copy2(source / "checkpoint-upgrade.json", tmp_path / "checkpoint-upgrade.json")
for name in ("intake/recording.f32", "intake/recording.json", "intake/intake.json",
"discovery/discovery.json", "search/manifest.json", "search/retention.sqlite"):
path = tmp_path / name
@@ -41,8 +46,14 @@ def test_saved_scored_tile_and_ledger_survive_upgrade(request):
report = upgrade_legacy_job(tmp_path)
assert report["tiles_preserved"] == 1
assert (tmp_path / "search" / tile.name).read_bytes() == original
- assert compatible_checkpoint_runs(tmp_path / "search", report["new_run_hash"]) == {
- report["old_run_hash"], report["new_run_hash"]}
+ accepted = {report["old_run_hash"], report["new_run_hash"]}
+ if source_variable == "DOPPLERGEIST_REAL_BATCHED_JOB":
+ assert report["version"] == 2
+ accepted.update([ORIGINAL_REFERENCE_RUN, BATCHED_REFERENCE_RUN])
+ assert json.loads(Path(report["previous_upgrade_report"]).read_text()) == json.loads(
+ (source / "checkpoint-upgrade.json").read_text())
+ assert compatible_checkpoint_runs(tmp_path / "search", report["new_run_hash"]) == accepted
+ assert compatible_checkpoint_runs(tmp_path / "search", report["old_run_hash"]) == {report["old_run_hash"]}
assert json.loads((tmp_path / "search" / tile.name).read_text())["payload"] == payload
assert (Path(report["previous_search"]) / tile.name).read_bytes() == original
assert tile.read_bytes() == original
@@ -57,3 +68,48 @@ def test_saved_scored_tile_and_ledger_survive_upgrade(request):
assert operator["resume_legacy_first_batch"] is True
_load_audio_checkpoint(tmp_path, json.loads((tmp_path / "request.json").read_text()), operator)
assert upgrade_legacy_job(tmp_path) is None
+
+
[email protected]("change", ["parent_report", "scoring_source"])
+def test_batched_upgrade_requires_known_parent_and_target(tmp_path, monkeypatch, change):
+ location = os.environ.get("DOPPLERGEIST_REAL_BATCHED_JOB")
+ if not location:
+ pytest.skip("Supply an actual saved batched job")
+ source = Path(location)
+ (tmp_path / "search").mkdir()
+ for name in ("search/manifest.json", "checkpoint-upgrade.json"):
+ shutil.copy2(source / name, tmp_path / name)
+ if change == "parent_report":
+ path = tmp_path / "checkpoint-upgrade.json"
+ report = json.loads(path.read_text())
+ report["legacy_first_batch"] = False
+ path.write_text(json.dumps(report))
+ else:
+ monkeypatch.setitem(checkpoint_resume.THROUGHPUT_SOURCES, "search.py", "0" * 64)
+ before = (tmp_path / "search/manifest.json").read_bytes()
+ assert upgrade_legacy_job(tmp_path) is None
+ assert (tmp_path / "search/manifest.json").read_bytes() == before
+
+
+def test_batched_upgrade_restores_parent_binding_on_failure(tmp_path, monkeypatch):
+ location = os.environ.get("DOPPLERGEIST_REAL_BATCHED_JOB")
+ if not location:
+ pytest.skip("Supply an actual saved batched job")
+ source = Path(location)
+ job = tmp_path / "job"
+ shutil.copytree(source, job)
+ names = ("checkpoint-upgrade.json", "resume.json", "operator.json", "search/manifest.json")
+ before = {name: (job / name).read_bytes() for name in names}
+ original = checkpoint_resume.write_json
+
+ def fail_operator(path, value):
+ if Path(path) == job / "operator.json":
+ raise OSError("interrupted upgrade")
+ return original(path, value)
+
+ monkeypatch.setattr(checkpoint_resume, "write_json", fail_operator)
+ with pytest.raises(OSError, match="interrupted upgrade"):
+ upgrade_legacy_job(job)
+ assert {name: (job / name).read_bytes() for name in names} == before
+ with sqlite3.connect(job / "search/retention.sqlite") as db:
+ assert db.execute("SELECT identity FROM run").fetchone() == (BATCHED_REFERENCE_RUN,)