Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Allow transient readers to release retention ledger locks (#298)

Cinna Davis2fcf7e7f86ad
diff --git a/docs/DATA_AND_SCALE.md b/docs/DATA_AND_SCALE.md
index 839f294..73421c6 100644
--- a/docs/DATA_AND_SCALE.md
+++ b/docs/DATA_AND_SCALE.md
@@ -123,6 +123,11 @@ shared-machine replay of two real grid points from a full-day search, one worker
took 64.93 seconds and two took 38.95 seconds including startup, with identical
payloads and ledgers. This small sample does not predict nationwide runtime.
+Retention-ledger connections wait up to 30 seconds for a transient SQLite
+lock, matching the worker-stage import. Longer contention can still fail a run;
+this does not replace the stopped-job snapshot procedure or save work that has
+not been checkpointed.
+
Before the spline-hull improvement below, one real full-day grid cell's track
retry reduced 1,929 queried tracks to 614 within the configured memory limit,
retaining all 200 tracks with unsupported motion. Complete scoring with the full
diff --git a/src/dopplergeist/finite_search.py b/src/dopplergeist/finite_search.py
index a9f9040..e08f0b9 100644
--- a/src/dopplergeist/finite_search.py
+++ b/src/dopplergeist/finite_search.py
@@ -148,7 +148,9 @@ class RetentionLedger:
raise DataUnavailable("Cannot read finite retention ledger") from error
def _open_connection(self):
- db = sqlite3.connect(self.path)
+ # Match the worker-stage import's bounded wait so a short-lived reader
+ # cannot fail an otherwise valid search at the default five seconds.
+ db = sqlite3.connect(self.path, timeout=30)
try:
db.execute("PRAGMA cache_size=-8192")
db.execute("PRAGMA temp_store=FILE")
diff --git a/tests/test_real_retention_lock.py b/tests/test_real_retention_lock.py
new file mode 100644
index 0000000..6e008c8
--- /dev/null
+++ b/tests/test_real_retention_lock.py
@@ -0,0 +1,49 @@
+"""Exercise a real saved retention ledger under a temporary reader lock."""
+import os
+from pathlib import Path
+import sqlite3
+import threading
+import time
+
+import pytest
+
+from dopplergeist.finite_search import RetentionLedger
+
+
+def test_real_ledger_writer_waits_for_transient_reader(tmp_path):
+ source = os.environ.get("DOPPLERGEIST_REAL_RETENTION_LEDGER")
+ if not source:
+ pytest.skip("set a completed real retention ledger path")
+ path = tmp_path / "retention.sqlite"
+ with sqlite3.connect(Path(source).resolve().as_uri() + "?mode=ro", uri=True) as src, sqlite3.connect(path) as dst:
+ src.backup(dst)
+ with sqlite3.connect(path) as db:
+ identity, kind = db.execute("SELECT identity,score_kind FROM run").fetchone()
+ stage, checksum = db.execute("SELECT name,checksum FROM stages LIMIT 1").fetchone()
+ ledger = RetentionLedger(path, identity, kind)
+ ready, release = threading.Event(), threading.Event()
+
+ def hold_reader():
+ with sqlite3.connect(path.resolve().as_uri() + "?mode=ro", uri=True) as db:
+ db.execute("BEGIN")
+ db.execute("SELECT checksum FROM stages WHERE name=?", (stage,)).fetchone()
+ ready.set()
+ release.wait(20)
+
+ reader = threading.Thread(target=hold_reader)
+ reader.start()
+ assert ready.wait(5)
+ timer = threading.Timer(7, release.set)
+ started = time.monotonic()
+ timer.start()
+ try:
+ with ledger.connect() as db:
+ db.execute("UPDATE stages SET checksum=checksum WHERE name=?", (stage,))
+ finally:
+ release.set()
+ reader.join()
+ timer.cancel()
+ assert time.monotonic() - started >= 6
+ assert ledger.stage_valid(stage, checksum)
+ with sqlite3.connect(path) as db:
+ assert db.execute("PRAGMA quick_check").fetchall() == [("ok",)]