Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Verify and import real worker retention stages
Cinna Davis0484235adb52
diff --git a/src/dopplergeist/parallel_retention.py b/src/dopplergeist/parallel_retention.pynew file mode 100644index 0000000..2c7788b--- /dev/null+++ b/src/dopplergeist/parallel_retention.py@@ -0,0 +1,53 @@+"""Move a completed worker stage into the search's retention ledger."""+import json+from pathlib import Path+import sqlite3++from .data_access import DataUnavailable+++def import_retention_stage(ledger, worker_path, stage, expected_checksum):+ """Atomically import one verified worker stage without changing its evidence.++ Workers use private ledgers; only the parent writes the destination ledger.+ The stage checksum, run identity, scores, discarded bounds and kept modes+ must all match before the parent can checkpoint that stage.+ """+ worker_path = Path(worker_path)+ if not worker_path.is_file() or worker_path.resolve() == ledger.path.resolve():+ raise DataUnavailable("Missing or invalid worker retention ledger")+ try:+ db = sqlite3.connect(ledger.path, timeout=30)+ try:+ db.execute("ATTACH DATABASE ? AS worker", (str(worker_path),))+ db.execute("BEGIN")+ identity = db.execute("SELECT identity, score_kind FROM worker.run").fetchone()+ if identity != (ledger.run_hash, ledger.score_kind):+ raise DataUnavailable("Worker retention ledger identity mismatch")+ stage_row = db.execute(+ "SELECT checksum, kept, occurrences, duplicates FROM worker.stages WHERE name=?", (stage,)+ ).fetchone()+ if stage_row is None or stage_row[0] != expected_checksum:+ raise DataUnavailable("Worker retention stage is missing or has the wrong checksum")+ records = db.execute(+ "SELECT mode_id, score, groups_count, kind, seed FROM worker.records WHERE stage=? ORDER BY mode_id",+ (stage,),+ ).fetchall()+ bounds = db.execute(+ "SELECT bound_id, upper_score, groups_count, detail FROM worker.bounds WHERE stage=? ORDER BY bound_id",+ (stage,),+ ).fetchall()+ if ledger._checksum(records, json.loads(stage_row[1]), stage_row[2], stage_row[3], bounds) != expected_checksum:+ raise DataUnavailable("Worker retention stage failed its checksum")+ for table in ("records", "bounds"):+ db.execute(f"DELETE FROM {table} WHERE stage=?", (stage,))+ db.execute(f"INSERT INTO {table} SELECT * FROM worker.{table} WHERE stage=?", (stage,))+ db.execute("DELETE FROM stages WHERE name=?", (stage,))+ db.execute("INSERT INTO stages SELECT * FROM worker.stages WHERE name=?", (stage,))+ db.commit()+ finally:+ db.close()+ except (sqlite3.DatabaseError, OSError, ValueError, TypeError) as error:+ raise DataUnavailable("Cannot import worker retention stage") from error+ if not ledger.stage_valid(stage, expected_checksum):+ raise DataUnavailable("Imported retention stage failed verification")diff --git a/tests/test_real_parallel_retention.py b/tests/test_real_parallel_retention.pynew file mode 100644index 0000000..15aa756--- /dev/null+++ b/tests/test_real_parallel_retention.py@@ -0,0 +1,40 @@+"""Verify worker-ledger imports using an actual running search's evidence."""+import os+from pathlib import Path+import sqlite3++import pytest++from dopplergeist.finite_search import RetentionLedger+from dopplergeist.parallel_retention import import_retention_stage+++def test_real_retention_stages_survive_worker_import(tmp_path):+ source = os.environ.get("DOPPLERGEIST_REAL_RETENTION_LEDGER")+ if not source:+ pytest.skip("set DOPPLERGEIST_REAL_RETENTION_LEDGER to a real search ledger")+ source = Path(source)+ snapshot = tmp_path / "worker.sqlite"+ with sqlite3.connect(f"{source.resolve().as_uri()}?mode=ro", uri=True) as current:+ with sqlite3.connect(snapshot) as worker:+ current.backup(worker)+ with sqlite3.connect(snapshot) as worker:+ run_hash, score_kind = worker.execute("SELECT identity, score_kind FROM run").fetchone()+ populated = worker.execute(+ "SELECT s.name, s.checksum FROM stages s JOIN bounds b ON b.stage=s.name "+ "WHERE s.name LIKE 'coarse:%' GROUP BY s.name ORDER BY COUNT(*) DESC LIMIT 1"+ ).fetchone()+ empty = worker.execute(+ "SELECT name, checksum FROM stages WHERE name LIKE 'coarse:%' AND occurrences=0 LIMIT 1"+ ).fetchone()+ assert populated is not None and empty is not None+ target = RetentionLedger(tmp_path / "parent.sqlite", run_hash, score_kind)+ for stage, checksum in (populated, empty, populated):+ import_retention_stage(target, snapshot, stage, checksum)+ assert target.stage_valid(stage, checksum)+ with sqlite3.connect(snapshot) as worker, sqlite3.connect(target.path) as parent:+ for stage, _ in (populated, empty):+ for table, column in (("stages", "name"), ("records", "stage"), ("bounds", "stage")):+ expected = worker.execute(f"SELECT * FROM {table} WHERE {column}=? ORDER BY 1,2", (stage,)).fetchall()+ actual = parent.execute(f"SELECT * FROM {table} WHERE {column}=? ORDER BY 1,2", (stage,)).fetchall()+ assert actual == expected