Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Avoid scanning the entire retention database at startup (#259)

Cinna Davisb4cd00ad46f0
diff --git a/src/dopplergeist/finite_search.py b/src/dopplergeist/finite_search.py
index b817f46..a9f9040 100644
--- a/src/dopplergeist/finite_search.py
+++ b/src/dopplergeist/finite_search.py
@@ -137,8 +137,8 @@ class RetentionLedger:
groups_count INTEGER, detail TEXT, PRIMARY KEY(stage,bound_id));
CREATE INDEX IF NOT EXISTS bound_index ON bounds(bound_id);
""")
- if db.execute("PRAGMA quick_check").fetchone()[0] != "ok":
- raise DataUnavailable("Corrupt finite retention ledger")
+ # Saved stages are checksum-verified before reuse and evidence.
+ # Opening a national ledger must not scan its entire database.
identity = db.execute("SELECT identity,score_kind FROM run").fetchone()
if identity is None:
db.execute("INSERT INTO run VALUES(?,?)", (run_hash, score_kind))
diff --git a/src/dopplergeist/parallel_retention.py b/src/dopplergeist/parallel_retention.py
index 814f598..0d6af89 100644
--- a/src/dopplergeist/parallel_retention.py
+++ b/src/dopplergeist/parallel_retention.py
@@ -32,11 +32,11 @@ def import_retention_stage(ledger, worker_path, stage, expected_checksum):
records = db.execute(
"SELECT mode_id, score, groups_count, kind, seed FROM worker.records WHERE stage=? ORDER BY mode_id",
(stage,),
- ).fetchall()
+ )
bounds = db.execute(
"SELECT bound_id, upper_score, groups_count, detail FROM worker.bounds WHERE stage=? ORDER BY bound_id",
(stage,),
- ).fetchall()
+ )
if ledger._checksum(records, json.loads(stage_row[1]), stage_row[2], stage_row[3], bounds) != expected_checksum:
raise DataUnavailable("Worker retention stage failed its checksum")
for table in ("records", "bounds"):
diff --git a/tests/test_real_retention_batch.py b/tests/test_real_retention_batch.py
index 499d92f..4eddf57 100644
--- a/tests/test_real_retention_batch.py
+++ b/tests/test_real_retention_batch.py
@@ -7,6 +7,7 @@ import pytest
from dopplergeist.data_access import DataUnavailable, write_json
from dopplergeist.finite_search import RetentionLedger
+from dopplergeist.parallel_retention import import_retention_stage
from dopplergeist.tiled_search import merge_candidates
@@ -39,6 +40,31 @@ def contents(target):
for table in ("run", "stages", "records", "bounds")}
[email protected]("damage", ["score", "missing_record", "sqlite_header"])
+def test_reopened_recorded_stage_rejects_damage(real_tiles, tmp_path, damage):
+ rows = json.loads((Path(__file__).parent / "data/reference-retention-real.json").read_text())["rows"]
+ target = ledger(tmp_path / "damaged.sqlite", real_tiles)
+ stage = "recorded-fits"
+ merge_candidates(rows, 3, ledger=target, stage=stage)
+ expected = target.checksum(stage)
+ assert target.stage_valid(stage, expected)
+ if damage == "sqlite_header":
+ data = target.path.read_bytes()
+ target.path.write_bytes(b"damaged header!!\0" + data[16:])
+ else:
+ with sqlite3.connect(target.path) as db:
+ if damage == "score":
+ db.execute("UPDATE records SET score=score+1 WHERE stage=?", (stage,))
+ else:
+ db.execute("DELETE FROM records WHERE stage=? AND mode_id=("
+ "SELECT min(mode_id) FROM records WHERE stage=?)", (stage, stage))
+ try:
+ reopened = ledger(target.path, real_tiles)
+ assert not reopened.stage_valid(stage, expected)
+ except DataUnavailable:
+ pass
+
+
def test_recorded_empty_stage_rows_and_checksums_match_in_batches(real_tiles, tmp_path):
serial = ledger(tmp_path / "serial.sqlite", real_tiles)
batched = ledger(tmp_path / "batched.sqlite", real_tiles)
@@ -55,6 +81,22 @@ def test_recorded_empty_stage_rows_and_checksums_match_in_batches(real_tiles, tm
assert batched.stage_valid(f"{phase}:{tile['tile_id']}", expected)
+def test_recorded_modes_import_with_complete_checksums(real_tiles, tmp_path):
+ rows = json.loads((Path(__file__).parent / "data/reference-retention-real.json").read_text())["rows"]
+ worker = ledger(tmp_path / "worker.sqlite", real_tiles)
+ parent = ledger(tmp_path / "parent.sqlite", real_tiles)
+ stage = "recorded-fits"
+ merge_candidates(rows, 3, ledger=worker, stage=stage)
+ checksum = worker.checksum(stage)
+ import_retention_stage(parent, worker.path, stage, checksum)
+ assert contents(parent) == contents(worker)
+ with worker.connect() as db:
+ db.execute("UPDATE records SET score=score+1 WHERE stage=?", (stage,))
+ with pytest.raises(DataUnavailable, match="failed its checksum"):
+ import_retention_stage(parent, worker.path, stage, checksum)
+ assert parent.stage_valid(stage, checksum)
+
+
def test_empty_tile_batches_preserve_previous_real_retained_modes(real_tiles, tmp_path):
# Empty tiles can still write nonempty global stages after a supported tile.
# These are archived fits from the original recording, not invented scores.