Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
"""Transaction checks against recorded empty tiles from the real reference run."""
import json
from pathlib import Path
import sqlite3
import pytest
from dopplergeist.data_access import DataUnavailable, write_json
from dopplergeist.finite_search import RetentionLedger
from dopplergeist.parallel_retention import import_retention_stage
from dopplergeist.tiled_search import merge_candidates
@pytest.fixture
def real_tiles():
path = Path(__file__).parent / "data/reference-empty-tiles-real.json"
fixture = json.loads(path.read_text())
assert fixture["tiles"]
for tile in fixture["tiles"]:
payload = tile["checkpoint"]["payload"]
assert payload["tracks"] == 0 and payload["ranked"] == [] and payload["counts"] == {}
return fixture
def ledger(path, fixture):
return RetentionLedger(path, fixture["tiles"][0]["checkpoint"]["run_hash"], fixture["score_kind"])
def record_tile(target, tile):
checksum = tile["checkpoint"]["payload"]["retention_checksum"]
for phase in ("coarse", "global"):
stage = f"{phase}:{tile['tile_id']}"
assert target.record(stage, [], [], 0) == checksum
assert target.checksum(stage) == checksum
def contents(target):
with target.connect() as db:
return {table: db.execute(f"SELECT * FROM {table} ORDER BY 1,2").fetchall()
for table in ("run", "stages", "records", "bounds")}
@pytest.mark.parametrize("damage", ["score", "missing_record", "sqlite_header"])
def test_reopened_recorded_stage_rejects_damage(real_tiles, tmp_path, damage):
rows = json.loads((Path(__file__).parent / "data/reference-retention-real.json").read_text())["rows"]
target = ledger(tmp_path / "damaged.sqlite", real_tiles)
stage = "recorded-fits"
merge_candidates(rows, 3, ledger=target, stage=stage)
expected = target.checksum(stage)
assert target.stage_valid(stage, expected)
if damage == "sqlite_header":
data = target.path.read_bytes()
target.path.write_bytes(b"damaged header!!\0" + data[16:])
else:
with sqlite3.connect(target.path) as db:
if damage == "score":
db.execute("UPDATE records SET score=score+1 WHERE stage=?", (stage,))
else:
db.execute("DELETE FROM records WHERE stage=? AND mode_id=("
"SELECT min(mode_id) FROM records WHERE stage=?)", (stage, stage))
try:
reopened = ledger(target.path, real_tiles)
assert not reopened.stage_valid(stage, expected)
except DataUnavailable:
pass
def test_recorded_empty_stage_rows_and_checksums_match_in_batches(real_tiles, tmp_path):
serial = ledger(tmp_path / "serial.sqlite", real_tiles)
batched = ledger(tmp_path / "batched.sqlite", real_tiles)
for tile in real_tiles["tiles"]:
record_tile(serial, tile)
for first in range(0, len(real_tiles["tiles"]), 3):
with batched.batch():
for tile in real_tiles["tiles"][first:first + 3]:
record_tile(batched, tile)
assert contents(batched) == contents(serial)
for tile in real_tiles["tiles"]:
expected = tile["checkpoint"]["payload"]["retention_checksum"]
for phase in ("coarse", "global"):
assert batched.stage_valid(f"{phase}:{tile['tile_id']}", expected)
def test_recorded_modes_import_with_complete_checksums(real_tiles, tmp_path):
rows = json.loads((Path(__file__).parent / "data/reference-retention-real.json").read_text())["rows"]
worker = ledger(tmp_path / "worker.sqlite", real_tiles)
parent = ledger(tmp_path / "parent.sqlite", real_tiles)
stage = "recorded-fits"
merge_candidates(rows, 3, ledger=worker, stage=stage)
checksum = worker.checksum(stage)
import_retention_stage(parent, worker.path, stage, checksum)
assert contents(parent) == contents(worker)
with worker.connect() as db:
db.execute("UPDATE records SET score=score+1 WHERE stage=?", (stage,))
with pytest.raises(DataUnavailable, match="failed its checksum"):
import_retention_stage(parent, worker.path, stage, checksum)
assert parent.stage_valid(stage, checksum)
def test_empty_tile_batches_preserve_previous_real_retained_modes(real_tiles, tmp_path):
# Empty tiles can still write nonempty global stages after a supported tile.
# These are archived fits from the original recording, not invented scores.
rows = json.loads((Path(__file__).parent / "data/reference-retention-real.json").read_text())["rows"]
serial = ledger(tmp_path / "retained-serial.sqlite", real_tiles)
batched = ledger(tmp_path / "retained-batched.sqlite", real_tiles)
def replay(target):
ranked = merge_candidates(rows, 3, ledger=target, stage="recorded-fits")
for tile in real_tiles["tiles"]:
target.record(f"coarse:{tile['tile_id']}", [], [], 0)
ranked = merge_candidates(ranked, 3, ledger=target, stage=f"global:{tile['tile_id']}")
return ranked
expected = replay(serial)
with batched.batch():
actual = replay(batched)
assert actual == expected
assert contents(batched) == contents(serial)
for tile in real_tiles["tiles"]:
stage = f"global:{tile['tile_id']}"
assert batched.stage_valid(stage, serial.checksum(stage))
def test_interrupted_batch_invalidates_checkpoint_until_replayed(real_tiles, tmp_path):
writer = ledger(tmp_path / "resume.sqlite", real_tiles)
reader = ledger(writer.path, real_tiles)
tile = real_tiles["tiles"][0]
stage = f"coarse:{tile['tile_id']}"
expected = tile["checkpoint"]["payload"]["retention_checksum"]
checkpoint = tmp_path / f"tile-{tile['tile_id']}.json"
with pytest.raises(RuntimeError, match="interrupted"):
with writer.batch():
record_tile(writer, tile)
write_json(checkpoint, tile["checkpoint"])
assert writer.stage_valid(stage, expected)
assert not reader.stage_valid(stage, expected)
raise RuntimeError("interrupted batch")
assert checkpoint.exists()
assert not reader.stage_valid(stage, expected)
with writer.batch():
record_tile(writer, tile)
assert reader.stage_valid(stage, expected)
def test_caught_connection_error_does_not_commit_partial_batch(real_tiles, tmp_path):
target = ledger(tmp_path / "caught.sqlite", real_tiles)
tile = real_tiles["tiles"][0]
with pytest.raises(DataUnavailable, match="not committed"):
with target.batch():
record_tile(target, tile)
try:
with target.connect() as db:
db.execute("SELECT * FROM nonexistent_batch_test_table")
except sqlite3.OperationalError:
pass
assert not target.stage_valid(f"coarse:{tile['tile_id']}", tile["checkpoint"]["payload"]["retention_checksum"])
def test_nested_batch_is_rejected_and_outer_work_rolls_back(real_tiles, tmp_path):
target = ledger(tmp_path / "nested.sqlite", real_tiles)
tile = real_tiles["tiles"][0]
with pytest.raises(RuntimeError, match="Nested retention batches"):
with target.batch():
record_tile(target, tile)
with target.batch():
pass
assert not target.stage_valid(f"coarse:{tile['tile_id']}", tile["checkpoint"]["payload"]["retention_checksum"])