Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Avoid scanning the entire retention database at startup (#259)
Cinna Davisb4cd00ad46f0
diff --git a/src/dopplergeist/finite_search.py b/src/dopplergeist/finite_search.pyindex b817f46..a9f9040 100644--- a/src/dopplergeist/finite_search.py+++ b/src/dopplergeist/finite_search.py@@ -137,8 +137,8 @@ class RetentionLedger:groups_count INTEGER, detail TEXT, PRIMARY KEY(stage,bound_id));CREATE INDEX IF NOT EXISTS bound_index ON bounds(bound_id);""")- if db.execute("PRAGMA quick_check").fetchone()[0] != "ok":- raise DataUnavailable("Corrupt finite retention ledger")+ # Saved stages are checksum-verified before reuse and evidence.+ # Opening a national ledger must not scan its entire database.identity = db.execute("SELECT identity,score_kind FROM run").fetchone()if identity is None:db.execute("INSERT INTO run VALUES(?,?)", (run_hash, score_kind))diff --git a/src/dopplergeist/parallel_retention.py b/src/dopplergeist/parallel_retention.pyindex 814f598..0d6af89 100644--- a/src/dopplergeist/parallel_retention.py+++ b/src/dopplergeist/parallel_retention.py@@ -32,11 +32,11 @@ def import_retention_stage(ledger, worker_path, stage, expected_checksum):records = db.execute("SELECT mode_id, score, groups_count, kind, seed FROM worker.records WHERE stage=? ORDER BY mode_id",(stage,),- ).fetchall()+ )bounds = db.execute("SELECT bound_id, upper_score, groups_count, detail FROM worker.bounds WHERE stage=? ORDER BY bound_id",(stage,),- ).fetchall()+ )if ledger._checksum(records, json.loads(stage_row[1]), stage_row[2], stage_row[3], bounds) != expected_checksum:raise DataUnavailable("Worker retention stage failed its checksum")for table in ("records", "bounds"):diff --git a/tests/test_real_retention_batch.py b/tests/test_real_retention_batch.pyindex 499d92f..4eddf57 100644--- a/tests/test_real_retention_batch.py+++ b/tests/test_real_retention_batch.py@@ -7,6 +7,7 @@ import pytestfrom dopplergeist.data_access import DataUnavailable, write_jsonfrom dopplergeist.finite_search import RetentionLedger+from dopplergeist.parallel_retention import import_retention_stagefrom dopplergeist.tiled_search import merge_candidates@@ -39,6 +40,31 @@ def contents(target):for table in ("run", "stages", "records", "bounds")}[email protected]("damage", ["score", "missing_record", "sqlite_header"])+def test_reopened_recorded_stage_rejects_damage(real_tiles, tmp_path, damage):+ rows = json.loads((Path(__file__).parent / "data/reference-retention-real.json").read_text())["rows"]+ target = ledger(tmp_path / "damaged.sqlite", real_tiles)+ stage = "recorded-fits"+ merge_candidates(rows, 3, ledger=target, stage=stage)+ expected = target.checksum(stage)+ assert target.stage_valid(stage, expected)+ if damage == "sqlite_header":+ data = target.path.read_bytes()+ target.path.write_bytes(b"damaged header!!\0" + data[16:])+ else:+ with sqlite3.connect(target.path) as db:+ if damage == "score":+ db.execute("UPDATE records SET score=score+1 WHERE stage=?", (stage,))+ else:+ db.execute("DELETE FROM records WHERE stage=? AND mode_id=("+ "SELECT min(mode_id) FROM records WHERE stage=?)", (stage, stage))+ try:+ reopened = ledger(target.path, real_tiles)+ assert not reopened.stage_valid(stage, expected)+ except DataUnavailable:+ pass++def test_recorded_empty_stage_rows_and_checksums_match_in_batches(real_tiles, tmp_path):serial = ledger(tmp_path / "serial.sqlite", real_tiles)batched = ledger(tmp_path / "batched.sqlite", real_tiles)@@ -55,6 +81,22 @@ def test_recorded_empty_stage_rows_and_checksums_match_in_batches(real_tiles, tmassert batched.stage_valid(f"{phase}:{tile['tile_id']}", expected)+def test_recorded_modes_import_with_complete_checksums(real_tiles, tmp_path):+ rows = json.loads((Path(__file__).parent / "data/reference-retention-real.json").read_text())["rows"]+ worker = ledger(tmp_path / "worker.sqlite", real_tiles)+ parent = ledger(tmp_path / "parent.sqlite", real_tiles)+ stage = "recorded-fits"+ merge_candidates(rows, 3, ledger=worker, stage=stage)+ checksum = worker.checksum(stage)+ import_retention_stage(parent, worker.path, stage, checksum)+ assert contents(parent) == contents(worker)+ with worker.connect() as db:+ db.execute("UPDATE records SET score=score+1 WHERE stage=?", (stage,))+ with pytest.raises(DataUnavailable, match="failed its checksum"):+ import_retention_stage(parent, worker.path, stage, checksum)+ assert parent.stage_valid(stage, checksum)++def test_empty_tile_batches_preserve_previous_real_retained_modes(real_tiles, tmp_path):# Empty tiles can still write nonempty global stages after a supported tile.# These are archived fits from the original recording, not invented scores.