Skip to content

Universal SEO skill for Claude Code. 26 sub-skills + 19 sub-agents covering technical SEO, E-E-A-T, schema, GEO/AEO, agent readiness (Lighthouse Agentic Browsing, WebMCP, llms.txt), backlinks, local SEO, e-commerce, international SEO, Google APIs, and PDF/Excel reporting. 9 optional extensions for live SEO data.

fix(dataforseo): hold one lock across the ledger read-modify-write

Adam Burgess227a00a4166a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8c34c37..7c80228 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- **Spend ledger lost concurrent writes (`dataforseo_costs.py`).** The ledger took its
+ file lock twice, once to read and once to write, and released it in between, so two
+ concurrent `log` calls both read the same ledger and the second silently overwrote the
+ first. Measured 3 entries lost out of 20 concurrent writes, with 1 to 3 lost on every
+ repeat run. This is not platform-specific. It matters because `seo-audit` fans out to
+ up to 15 parallel specialists, three of which call DataForSEO and each of which logs
+ after every call, so the ledger under-reported most during the runs that spend the
+ most. One exclusive lock now spans the entire read-modify-write in both `log` and
+ `reset`.
+- **No file locking at all on Windows.** The module set `fcntl = None` when the import
+ failed and then took every unlocked code path, so Windows ran the ledger with no
+ locking whatsoever. Locking now falls back to `msvcrt`, and when neither primitive is
+ available the operation fails loudly rather than proceeding unlocked.
+- **A corrupt ledger silently reset the budget to zero.** The non-atomic
+ `open(..., "w")` write could leave a truncated file if the process died mid-write, and
+ the reader caught `JSONDecodeError` and returned an empty ledger, which the next write
+ then persisted, discarding the whole spend history. Ledger writes are now atomic
+ (tempfile plus `os.replace`, matching the idiom already used in `google_auth.py`), and
+ an unparseable ledger fails closed with the file left in place for inspection.
+- **`CLAUDE_SEO_CONFIG_DIR`** now overrides the config and ledger location, so the new
+ `tests/test_dataforseo_costs.py` concurrency and durability tests run against an
+ isolated ledger instead of the operator's real spend history.
- The Common-Crawl-only no-score rule is now enforced instead of merely stated. The
`seo-backlinks` skill said not to produce a numeric health score when only Common Crawl
data is available, while `skills/seo/references/free-backlink-sources.md` told the agent
diff --git a/scripts/dataforseo_costs.py b/scripts/dataforseo_costs.py
index 84033fa..2f6f0d1 100644
--- a/scripts/dataforseo_costs.py
+++ b/scripts/dataforseo_costs.py
@@ -10,6 +10,7 @@ Provides cost-aware guardrails for DataForSEO API usage:
Config: ~/.config/claude-seo/dataforseo-costs.json
Ledger: ~/.config/claude-seo/dataforseo-ledger.json
+Set CLAUDE_SEO_CONFIG_DIR to point both somewhere else (used by the tests).
Usage:
python dataforseo_costs.py estimate <endpoint> [--count N]
@@ -27,18 +28,30 @@ Security fixes: config path corrected to ~/.config/claude-seo/
import argparse
import json
import sys
+import tempfile
+from contextlib import contextmanager
from datetime import datetime, timedelta
from pathlib import Path
try:
import fcntl
except ImportError:
- fcntl = None # Windows fallback: no locking
+ fcntl = None
+
+try:
+ import msvcrt
+except ImportError:
+ msvcrt = None
# ----- paths -----
-CONFIG_DIR = Path.home() / ".config" / "claude-seo"
+# CLAUDE_SEO_CONFIG_DIR lets tests (and anyone running an isolated budget) point
+# the ledger somewhere other than the real one. Resolved at import.
+CONFIG_DIR = Path(
+ os.environ.get("CLAUDE_SEO_CONFIG_DIR") or Path.home() / ".config" / "claude-seo"
+)
CONFIG_FILE = CONFIG_DIR / "dataforseo-costs.json"
LEDGER_FILE = CONFIG_DIR / "dataforseo-ledger.json"
+LOCK_FILE = CONFIG_DIR / "dataforseo-ledger.lock"
# ----- cost table (USD per call, standard queue) -----
# Source: https://dataforseo.com/pricing
@@ -140,41 +153,108 @@ def _save_config(cfg):
json.dump(cfg, f, indent=2)
-def _load_ledger():
- """Load spending ledger with file locking."""
+class LedgerError(RuntimeError):
+ """The ledger could not be read or written safely."""
+
+
+def _lock(handle, exclusive):
+ """Take an advisory lock on an open handle, blocking until acquired.
+
+ Never degrades to running unlocked: a spend ledger that silently drops
+ entries is worse than one that refuses to run.
+ """
+ if fcntl is not None:
+ fcntl.flock(handle, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
+ elif msvcrt is not None:
+ # msvcrt has no shared mode, so readers take an exclusive lock too.
+ # LK_LOCK retries 10 times at 1s intervals, then raises OSError.
+ handle.seek(0)
+ msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
+ else:
+ raise LedgerError(
+ "No file-locking primitive available (neither fcntl nor msvcrt). "
+ "Refusing to touch the spend ledger unlocked."
+ )
+
+
+def _unlock(handle):
+ if fcntl is not None:
+ fcntl.flock(handle, fcntl.LOCK_UN)
+ elif msvcrt is not None:
+ handle.seek(0)
+ msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
+
+
+def _read_ledger():
+ """Read the ledger. Caller must hold the lock."""
if not LEDGER_FILE.exists():
return {"entries": []}
- if fcntl:
- lock_path = LEDGER_FILE.with_suffix(".lock")
- with open(lock_path, "w") as lock_file:
- fcntl.flock(lock_file, fcntl.LOCK_SH)
- try:
- with open(LEDGER_FILE) as f:
- return json.load(f)
- except (json.JSONDecodeError, IOError):
- return {"entries": []}
- finally:
- fcntl.flock(lock_file, fcntl.LOCK_UN)
- else:
+ try:
with open(LEDGER_FILE) as f:
- return json.load(f)
+ ledger = json.load(f)
+ except json.JSONDecodeError as exc:
+ raise LedgerError(
+ f"Spend ledger {LEDGER_FILE} is not valid JSON ({exc}). Refusing to "
+ "continue from a zero balance, which would under-report spend. "
+ "Inspect the file, then repair or remove it."
+ ) from exc
+ if not isinstance(ledger, dict) or not isinstance(ledger.get("entries"), list):
+ raise LedgerError(
+ f"Spend ledger {LEDGER_FILE} has an unexpected shape (no 'entries' "
+ "list). Inspect the file, then repair or remove it."
+ )
+ return ledger
-def _save_ledger(ledger):
- """Save spending ledger with file locking."""
- CONFIG_DIR.mkdir(parents=True, exist_ok=True)
- if fcntl:
- lock_path = LEDGER_FILE.with_suffix(".lock")
- with open(lock_path, "w") as lock_file:
- fcntl.flock(lock_file, fcntl.LOCK_EX)
- try:
- with open(LEDGER_FILE, "w") as f:
- json.dump(ledger, f, indent=2)
- finally:
- fcntl.flock(lock_file, fcntl.LOCK_UN)
- else:
- with open(LEDGER_FILE, "w") as f:
+def _write_ledger(ledger):
+ """Write the ledger atomically. Caller must hold the exclusive lock.
+
+ tempfile + os.replace so a crash mid-write can never leave a truncated,
+ unparseable ledger behind (same idiom as scripts/google_auth.py).
+ """
+ fd, tmp_path = tempfile.mkstemp(
+ dir=str(CONFIG_DIR), prefix=".dataforseo-ledger.", suffix=".tmp"
+ )
+ try:
+ with os.fdopen(fd, "w") as f:
json.dump(ledger, f, indent=2)
+ f.flush()
+ os.fsync(f.fileno())
+ os.replace(tmp_path, LEDGER_FILE)
+ except Exception:
+ if os.path.exists(tmp_path):
+ os.unlink(tmp_path)
+ raise
+
+
+@contextmanager
+def _locked_ledger(write=False):
+ """Yield the ledger under ONE lock held across the whole read-modify-write.
+
+ Taking the lock to read, dropping it, then re-taking it to write loses
+ entries: two concurrent writers both read the same ledger and the second
+ overwrites the first. The lock has to span both halves.
+
+ The lock lives in a separate .lock file on purpose. The atomic write
+ replaces the ledger's inode, so a lock held on the ledger itself would not
+ protect the replacement.
+ """
+ CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+ with open(LOCK_FILE, "a+") as lock_handle:
+ _lock(lock_handle, exclusive=write)
+ try:
+ ledger = _read_ledger()
+ yield ledger
+ if write:
+ _write_ledger(ledger)
+ finally:
+ _unlock(lock_handle)
+
+
+def _ledger_snapshot():
+ """Read the ledger under a shared lock. For read-only commands."""
+ with _locked_ledger() as ledger:
+ return ledger
def _today_str():
@@ -229,7 +309,7 @@ def cmd_estimate(args):
def cmd_check(args):
"""Check if an API call should proceed (cost + approval logic)."""
cfg = _load_config()
- ledger = _load_ledger()
+ ledger = _ledger_snapshot()
endpoint = args.endpoint
count = args.count or 1
unit_cost = COST_TABLE.get(endpoint)
@@ -298,7 +378,6 @@ def cmd_check(args):
def cmd_log(args):
"""Log a completed API call cost."""
- ledger = _load_ledger()
entry = {
"timestamp": datetime.now().isoformat(),
"endpoint": args.endpoint,
@@ -306,8 +385,10 @@ def cmd_log(args):
}
if args.note:
entry["note"] = args.note
- ledger["entries"].append(entry)
- _save_ledger(ledger)
+ # Read and write inside one lock, so a concurrent log cannot overwrite this
+ # entry with a ledger it read before this one was appended.
+ with _locked_ledger(write=True) as ledger:
+ ledger["entries"].append(entry)
result = {
"status": "logged",
@@ -319,7 +400,7 @@ def cmd_log(args):
def cmd_summary(args):
"""Show spending summary for recent days."""
- ledger = _load_ledger()
+ ledger = _ledger_snapshot()
days = args.days or 7
cutoff = (datetime.now() - timedelta(days=days)).isoformat()
@@ -350,7 +431,7 @@ def cmd_summary(args):
def cmd_today(args):
"""Show today's spending."""
- ledger = _load_ledger()
+ ledger = _ledger_snapshot()
cfg = _load_config()
today = _today_str()
today_entries = [e for e in ledger["entries"] if e["timestamp"].startswith(today)]
@@ -416,23 +497,23 @@ def cmd_reset(args):
json.dump(result, sys.stdout, indent=2)
return
- ledger = _load_ledger()
today = _today_str()
- today_entries = [e for e in ledger["entries"] if e["timestamp"].startswith(today)]
- removed_total = sum(e["cost"] for e in today_entries)
- removed_count = len(today_entries)
-
- ledger["entries"] = [e for e in ledger["entries"] if not e["timestamp"].startswith(today)]
-
- # Immutable audit entry for the reset itself
- ledger["entries"].append({
- "timestamp": datetime.now().isoformat(),
- "endpoint": "_audit_reset",
- "cost": 0,
- "note": f"Reset cleared {removed_count} entries totaling ${removed_total:.4f}",
- })
-
- _save_ledger(ledger)
+ # One lock across the whole read-modify-write, so a concurrent log is either
+ # cleared by this reset or survives it, never silently discarded.
+ with _locked_ledger(write=True) as ledger:
+ today_entries = [e for e in ledger["entries"] if e["timestamp"].startswith(today)]
+ removed_total = sum(e["cost"] for e in today_entries)
+ removed_count = len(today_entries)
+
+ ledger["entries"] = [e for e in ledger["entries"] if not e["timestamp"].startswith(today)]
+
+ # Immutable audit entry for the reset itself
+ ledger["entries"].append({
+ "timestamp": datetime.now().isoformat(),
+ "endpoint": "_audit_reset",
+ "cost": 0,
+ "note": f"Reset cleared {removed_count} entries totaling ${removed_total:.4f}",
+ })
result = {
"status": "reset",
@@ -492,7 +573,13 @@ def main():
"config": cmd_config,
"reset": cmd_reset,
}
- dispatch[args.command](args)
+ try:
+ dispatch[args.command](args)
+ except LedgerError as exc:
+ # Fail closed and loudly. Callers parse stdout as JSON, so keep the
+ # contract even on the error path.
+ json.dump({"status": "error", "message": str(exc)}, sys.stdout, indent=2)
+ sys.exit(1)
if __name__ == "__main__":
diff --git a/tests/test_dataforseo_costs.py b/tests/test_dataforseo_costs.py
new file mode 100644
index 0000000..481f759
--- /dev/null
+++ b/tests/test_dataforseo_costs.py
@@ -0,0 +1,171 @@
+"""
+Tests for scripts/dataforseo_costs.py — spend-ledger integrity.
+
+Regression cover for the lost-update race: the ledger used to take its file
+lock twice, once to read and once to write, and drop it in between, so two
+concurrent `log` calls both read the same ledger and the second overwrote the
+first. A budget ledger that quietly under-reports spend is the one thing it
+must not do.
+
+Every test runs against an isolated CLAUDE_SEO_CONFIG_DIR, never the real
+~/.config/claude-seo/ ledger.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import pytest
+
+_REPO = Path(__file__).resolve().parents[1]
+_SCRIPT = _REPO / "scripts" / "dataforseo_costs.py"
+
+
+def _run(config_dir: Path, *argv: str) -> subprocess.CompletedProcess:
+ """Invoke the CLI in a subprocess with an isolated ledger."""
+ env = dict(os.environ)
+ env["CLAUDE_SEO_CONFIG_DIR"] = str(config_dir)
+ return subprocess.run(
+ [sys.executable, str(_SCRIPT), *argv],
+ capture_output=True,
+ text=True,
+ env=env,
+ )
+
+
+def _entries(config_dir: Path) -> list[dict]:
+ return json.loads((config_dir / "dataforseo-ledger.json").read_text())["entries"]
+
+
+# ---------------------------------------------------------------------------
+# concurrency — the reported bug
+# ---------------------------------------------------------------------------
+
+
+def test_concurrent_logs_do_not_lose_entries(tmp_path) -> None:
+ """20 concurrent `log` calls must produce exactly 20 entries.
+
+ Fails on the pre-fix code, which loses entries whenever two processes
+ interleave their read-modify-write.
+ """
+ calls = 20
+ unit_cost = 0.01
+
+ with ThreadPoolExecutor(max_workers=calls) as pool:
+ results = list(
+ pool.map(
+ lambda i: _run(
+ tmp_path, "log", "serp_organic_live_advanced", str(unit_cost),
+ "--note", f"call-{i}",
+ ),
+ range(calls),
+ )
+ )
+
+ failed = [r for r in results if r.returncode != 0]
+ assert not failed, f"{len(failed)} log calls failed: {failed[0].stderr}"
+
+ entries = _entries(tmp_path)
+ assert len(entries) == calls, (
+ f"lost {calls - len(entries)} of {calls} concurrent writes"
+ )
+
+ notes = sorted(e["note"] for e in entries)
+ assert notes == sorted(f"call-{i}" for i in range(calls)), "entries were clobbered"
+
+ total = sum(e["cost"] for e in entries)
+ assert total == pytest.approx(calls * unit_cost), "recorded spend under-reports"
+
+
+def test_concurrent_log_and_reset_keep_the_ledger_parseable(tmp_path) -> None:
+ """Interleaved writers must never leave a torn/unparseable ledger."""
+ _run(tmp_path, "log", "serp_organic_live_advanced", "0.01")
+
+ def job(i: int) -> subprocess.CompletedProcess:
+ if i % 5 == 4:
+ return _run(tmp_path, "reset", "--confirm")
+ return _run(tmp_path, "log", "serp_organic_live_advanced", "0.01")
+
+ with ThreadPoolExecutor(max_workers=10) as pool:
+ list(pool.map(job, range(10)))
+
+ # The ledger must still be valid JSON with an entries list. Which entries
+ # survive depends on reset ordering; that the file parses is the invariant.
+ entries = _entries(tmp_path)
+ assert isinstance(entries, list)
+
+ result = json.loads(_run(tmp_path, "today").stdout)
+ assert result["status"] == "today"
+
+
+# ---------------------------------------------------------------------------
+# durability
+# ---------------------------------------------------------------------------
+
+
+def test_corrupt_ledger_is_not_silently_zeroed(tmp_path) -> None:
+ """A truncated ledger must fail closed, not restart the budget at zero.
+
+ The old code caught JSONDecodeError and returned {"entries": []}, so the
+ next write persisted an empty ledger and the entire spend history vanished.
+ """
+ tmp_path.mkdir(parents=True, exist_ok=True)
+ ledger = tmp_path / "dataforseo-ledger.json"
+ ledger.write_text('{"entries": [{"timestamp": "2026-01-01T00:00:00", "cos')
+
+ result = _run(tmp_path, "log", "serp_organic_live_advanced", "0.01")
+ assert result.returncode != 0, "corrupt ledger must not be silently accepted"
+ payload = json.loads(result.stdout)
+ assert payload["status"] == "error"
+ assert "not valid JSON" in payload["message"]
+
+ # The damaged file is left in place for inspection, not overwritten.
+ assert ledger.read_text().endswith('"cos')
+
+
+def test_today_totals_survive_a_write(tmp_path) -> None:
+ """Basic round trip: logged spend is reflected in `today`."""
+ _run(tmp_path, "log", "on_page_lighthouse", "0.02")
+ _run(tmp_path, "log", "on_page_lighthouse", "0.03")
+
+ payload = json.loads(_run(tmp_path, "today").stdout)
+ assert payload["total_usd"] == pytest.approx(0.05)
+ assert payload["calls"] == 2
+
+
+def test_ledger_write_is_atomic_no_temp_files_left(tmp_path) -> None:
+ """The tempfile used for the atomic replace must not accumulate."""
+ for _ in range(5):
+ _run(tmp_path, "log", "on_page_lighthouse", "0.02")
+
+ leftovers = list(tmp_path.glob(".dataforseo-ledger.*.tmp"))
+ assert leftovers == [], f"temp files left behind: {leftovers}"
+
+
+# ---------------------------------------------------------------------------
+# portability guard
+# ---------------------------------------------------------------------------
+
+
+def test_locking_is_never_disabled_unconditionally() -> None:
+ """Guard against the `fcntl = None` regression.
+
+ The module previously ran with no locking at all on Windows. It must now
+ fall back to msvcrt and, failing that, refuse to touch the ledger.
+ """
+ source = _SCRIPT.read_text()
+ assert "import msvcrt" in source, "no Windows locking fallback"
+ assert "Refusing to touch the spend ledger unlocked" in source, (
+ "module must fail closed when no locking primitive is available"
+ )
+
+
+def test_isolated_config_dir_is_honoured(tmp_path) -> None:
+ """Tests must never write to the operator's real ledger."""
+ _run(tmp_path, "log", "on_page_lighthouse", "0.02")
+ assert (tmp_path / "dataforseo-ledger.json").exists()