Python
Stored sign-ins: one lock across processes, and a sign-in is renewed once
Rohit C Prasad61e614eb2465
diff --git a/coworker/cloud.py b/coworker/cloud.pyindex 8e1b5e92..7db5be07 100644--- a/coworker/cloud.py+++ b/coworker/cloud.py@@ -234,21 +234,30 @@ def fresh_access_token(secrets: SecretStore, config: Config) -> Optional[str]:return Noneif float(profile.get("expires") or 0) > _now():return profile["access_token"]- if not profile.get("refresh_token"):- return None- resp = httpx.post(- f"https://{config.cloud_auth_domain}/oauth/token",- data={- "grant_type": "refresh_token",- "client_id": config.cloud_client_id,- "refresh_token": profile["refresh_token"],- },- timeout=15,- )- if resp.status_code != 200:- return None- _store_cloud_tokens(secrets, resp.json())- return (secrets.get(CLOUD_AUTH_PROFILE) or {}).get("access_token")+ # One renewal at a time, across processes (the app and each `openworker run` share+ # this store). Read again inside the lock: whoever got here first has renewed it, and+ # a second renewal with the refresh token it replaced could cancel the session.+ with secrets.exclusive():+ profile = secrets.get(CLOUD_AUTH_PROFILE) or {}+ if not profile.get("access_token"):+ return None+ if float(profile.get("expires") or 0) > _now():+ return profile["access_token"]+ if not profile.get("refresh_token"):+ return None+ resp = httpx.post(+ f"https://{config.cloud_auth_domain}/oauth/token",+ data={+ "grant_type": "refresh_token",+ "client_id": config.cloud_client_id,+ "refresh_token": profile["refresh_token"],+ },+ timeout=15,+ )+ if resp.status_code != 200:+ return None+ _store_cloud_tokens(secrets, resp.json())+ return (secrets.get(CLOUD_AUTH_PROFILE) or {}).get("access_token")def fetch_me(secrets: SecretStore, config: Config) -> Optional[dict]:@@ -505,9 +514,27 @@ def refresh_managed_token(that way. Manual profiles are never touched. `profile_key` targets anaccount-keyed profile (`gmail:account:<email>`); default = `<name>:default`."""key = profile_key or f"{connector}:default"- profile = secrets.get(key) or {}- if not (profile.get("managed") and profile.get("refresh_token")):+ seen = secrets.get(key) or {}+ if not (seen.get("managed") and seen.get("refresh_token")):return None+ # One renewal at a time, across processes; see fresh_access_token. If another process+ # renewed this sign-in while this one waited for the lock, its token is the answer.+ with secrets.exclusive():+ profile = secrets.get(key) or {}+ if not (profile.get("managed") and profile.get("refresh_token")):+ return None+ if (+ profile.get("access_token")+ and profile.get("access_token") != seen.get("access_token")+ and float(profile.get("expires") or 0) > _now()+ ):+ return profile+ return _renew_managed_token(secrets, config, connector, key, profile)+++def _renew_managed_token(+ secrets: SecretStore, config: Config, connector: str, key: str, profile: dict[str, Any]+) -> Optional[dict[str, Any]]:provider = profile.get("provider") or PROVIDER_FOR_CONNECTOR.get(connector)if not provider:return Nonediff --git a/coworker/providers/codex_auth.py b/coworker/providers/codex_auth.pyindex 681a6d9f..7b554c56 100644--- a/coworker/providers/codex_auth.py+++ b/coworker/providers/codex_auth.py@@ -244,7 +244,24 @@ class CodexTokenStore:def refresh(self) -> tuple[str, str]:"""refresh_token grant → fresh (access token, account id). A rejected refreshtoken blanks the profile — the provider reads as cleanly signed out."""- refresh = (self._data().get("tokens") or {}).get("refresh_token") or ""+ seen = (self._data().get("tokens") or {}).get("access_token") or ""+ exclusive = getattr(self._secrets, "exclusive", None)+ if exclusive is None: # a store without the lock (tests' stand-ins)+ return self._refresh_locked(seen)+ # One renewal at a time, across processes: the app and each `openworker run`+ # share this store, and a second renewal with the refresh token the first one+ # replaced would sign the user out.+ with exclusive():+ return self._refresh_locked(seen)++ def _refresh_locked(self, seen: str) -> tuple[str, str]:+ data = self._data()+ tokens = data.get("tokens") or {}+ access = tokens.get("access_token") or ""+ exp = _jwt_claims(access).get("exp")+ if access and access != seen and isinstance(exp, (int, float)) and exp - time.time() >= REFRESH_MARGIN_SECONDS:+ return access, data.get("account_id") or "" # another process just renewed it+ refresh = tokens.get("refresh_token") or ""if not refresh:self.clear()raise CodexSignInRequired(EXPIRED_ERROR)diff --git a/coworker/secrets.py b/coworker/secrets.pyindex 1607cdfe..4504de3b 100644--- a/coworker/secrets.py+++ b/coworker/secrets.py@@ -10,7 +10,9 @@ a Keychain / age-encrypted backend can swap in later without touching them.from __future__ import annotations+import contextlibimport json+import loggingimport osimport reimport subprocess@@ -138,13 +140,111 @@ def write_private_text(path: str | Path, content: str) -> Path:return _atomic_private_write(Path(path).expanduser(), content)+class _StoreLock:+ """One lock per secrets file, across threads AND processes.++ Several OpenWorker processes can share a state folder: the app's server, and each+ `openworker run`. A change to the file is read, modify, write of the WHOLE file, so+ without this two processes can each write and one change is lost. A sign-in renewal+ is worse: two processes renewing the same sign-in at once can leave it cancelled at a+ vendor that hands out a new refresh token each time.++ Inside the process it is a re-entrant lock shared by every SecretStore on the same+ file. Across processes it is an advisory lock on `<file>.lock` (flock; byte-range+ locking on Windows), taken with a deadline: if another process holds it for too long,+ this one goes ahead without it rather than stall a turn on a stuck peer."""++ def __init__(self, path: Path) -> None:+ self.path = path.with_name(path.name + ".lock")+ self._thread = threading.RLock()+ self._depth = 0+ self._fh: Any = None++ def acquire(self, timeout: float) -> None:+ self._thread.acquire()+ if self._depth == 0:+ self._fh = self._lock_file(timeout)+ self._depth += 1++ def release(self) -> None:+ self._depth -= 1+ if self._depth == 0 and self._fh is not None:+ fh, self._fh = self._fh, None+ try:+ if _IS_WINDOWS:+ import msvcrt++ fh.seek(0)+ msvcrt.locking(fh.fileno(), msvcrt.LK_UNLCK, 1)+ else:+ import fcntl++ fcntl.flock(fh.fileno(), fcntl.LOCK_UN)+ except (OSError, ImportError):+ pass+ fh.close()+ self._thread.release()++ def _lock_file(self, timeout: float) -> Any:+ from .statelock import _try_lock++ try:+ self.path.parent.mkdir(parents=True, exist_ok=True)+ # Made private from the first moment, like every file beside the secrets. It+ # is empty and stays: it is the lock, not a temporary file.+ fh = os.fdopen(os.open(self.path, os.O_RDWR | os.O_CREAT, 0o600), "r+")+ except OSError:+ return None # a read-only folder: nothing could be written there anyway+ try:+ _restrict_to_user(self.path, is_dir=False)+ except OSError:+ pass+ deadline = time.monotonic() + max(0.0, timeout)+ while True:+ if _try_lock(fh):+ return fh+ if time.monotonic() >= deadline:+ logging.getLogger(__name__).warning(+ "the stored sign-ins stayed locked by another process for %.0fs; going ahead without the lock",+ timeout,+ )+ fh.close()+ return None+ time.sleep(0.05)+++_STORE_LOCKS: dict[str, _StoreLock] = {}+_STORE_LOCKS_GUARD = threading.Lock()+++def _store_lock(path: Path) -> _StoreLock:+ key = os.path.realpath(str(path))+ with _STORE_LOCKS_GUARD:+ lock = _STORE_LOCKS.get(key)+ if lock is None:+ lock = _STORE_LOCKS[key] = _StoreLock(Path(key))+ return lock++class SecretStore:"""File-backed secret store. Reads resolve `${VAR}` refs; status never leaks values."""def __init__(self, path: Optional[str | Path] = None) -> None:self.path = Path(path).expanduser() if path else state_dir() / "secrets.json"self._dotenv_path = self.path.parent / ".env"- self._lock = threading.Lock()+ self._lock = _store_lock(self.path)++ @contextlib.contextmanager+ def exclusive(self, timeout: float = 30.0):+ """Hold the store for a read-then-write that no other thread or process may+ interleave with. A sign-in renewal takes it, reads the profile AGAIN inside, and+ renews only if nobody else just did: the first process renews, the others read+ its result. Re-entrant: `put` and `delete` inside it are fine."""+ self._lock.acquire(timeout)+ try:+ yield self+ finally:+ self._lock.release()# -- reads ------------------------------------------------------------------def get(self, profile: str) -> Optional[dict[str, Any]]:@@ -193,13 +293,13 @@ class SecretStore:# -- writes -----------------------------------------------------------------def put(self, profile: str, data: dict[str, Any]) -> None:- with self._lock:+ with self.exclusive():store = self._read()store[profile] = dataself._write(store)def delete(self, profile: str) -> bool:- with self._lock:+ with self.exclusive():store = self._read()if profile not in store:return Falsediff --git a/tests/test_secrets_file_mode.py b/tests/test_secrets_file_mode.pyindex ae0c4f5d..846807e1 100644--- a/tests/test_secrets_file_mode.py+++ b/tests/test_secrets_file_mode.py@@ -63,8 +63,10 @@ def test_temp_file_is_never_group_or_world_readable_mid_write(tmp_path, monkeypadef test_no_temp_file_is_left_behind(tmp_path):store = SecretStore(tmp_path / "secrets.json")store.put("openai", {"api_key": "sk-live"})- leftovers = [p.name for p in tmp_path.iterdir() if p.name != "secrets.json"]+ # The lock file beside it is permanent and empty; it is not a temporary file.+ leftovers = [p.name for p in tmp_path.iterdir() if p.name not in ("secrets.json", "secrets.json.lock")]assert leftovers == []+ assert (tmp_path / "secrets.json.lock").read_bytes() == b""def test_content_round_trips(tmp_path):@@ -90,7 +92,7 @@ def test_a_failed_write_leaves_the_previous_file_intact(tmp_path, monkeypatch):store.put("openai", {"api_key": "sk-replacement"})assert json.loads(path.read_text())["openai"]["api_key"] == "sk-original"- assert [p.name for p in tmp_path.iterdir()] == ["secrets.json"], "temp must be cleaned up"+ assert sorted(p.name for p in tmp_path.iterdir()) == ["secrets.json", "secrets.json.lock"], "temp must be cleaned up"def test_a_hostile_preexisting_temp_name_cannot_redirect_the_write(tmp_path):diff --git a/tests/test_secrets_lock.py b/tests/test_secrets_lock.pynew file mode 100644index 00000000..9820f15f--- /dev/null+++ b/tests/test_secrets_lock.py@@ -0,0 +1,183 @@+"""The stored sign-ins are shared by every OpenWorker process on a computer: the app's+server and each `openworker run`. A change must not be lost when two of them write, and a+sign-in must be renewed once: the first process renews, the others read its result."""++from __future__ import annotations++import os+import subprocess+import sys+import textwrap+import threading+import time+from pathlib import Path++import pytest++from coworker import cloud+from coworker.config import Config+from coworker.providers import codex_auth+from coworker.secrets import SecretStore++ROOT = Path(__file__).resolve().parents[1]+++def _python(code: str, *args: str) -> subprocess.Popen:+ env = dict(os.environ)+ env["PYTHONPATH"] = str(ROOT) + os.pathsep + env.get("PYTHONPATH", "")+ return subprocess.Popen([sys.executable, "-c", textwrap.dedent(code), *args], env=env, stdout=subprocess.PIPE, text=True)+++def test_writes_from_several_processes_are_all_kept(tmp_path: Path) -> None:+ path = tmp_path / "secrets.json"+ writer = """+ import sys+ from coworker.secrets import SecretStore+ store = SecretStore(sys.argv[1])+ for i in range(25):+ store.put(f"{sys.argv[2]}:{i}", {"n": i})+ """+ procs = [_python(writer, str(path), name) for name in ("a", "b", "c", "d")]+ for proc in procs:+ assert proc.wait(timeout=120) == 0+ stored = SecretStore(path)+ missing = [f"{name}:{i}" for name in "abcd" for i in range(25) if stored.get(f"{name}:{i}") is None]+ assert missing == []+++def test_a_second_process_waits_for_the_first(tmp_path: Path) -> None:+ path = tmp_path / "secrets.json"+ holder = _python(+ """+ import sys, time+ from coworker.secrets import SecretStore+ with SecretStore(sys.argv[1]).exclusive():+ print("held", flush=True)+ time.sleep(1.5)+ """,+ str(path),+ )+ assert holder.stdout.readline().strip() == "held"+ started = time.monotonic()+ with SecretStore(path).exclusive():+ waited = time.monotonic() - started+ assert holder.wait(timeout=30) == 0+ assert waited > 0.8+++def test_a_stuck_holder_does_not_stall_a_turn_for_ever(tmp_path: Path) -> None:+ path = tmp_path / "secrets.json"+ holder = _python(+ """+ import sys, time+ from coworker.secrets import SecretStore+ with SecretStore(sys.argv[1]).exclusive():+ print("held", flush=True)+ time.sleep(20)+ """,+ str(path),+ )+ try:+ assert holder.stdout.readline().strip() == "held"+ started = time.monotonic()+ store = SecretStore(path)+ with store.exclusive(timeout=0.3):+ store.put("openai", {"api_key": "sk-x"}) # goes ahead without the lock+ assert time.monotonic() - started < 5+ assert store.get("openai") == {"api_key": "sk-x"}+ finally:+ holder.kill()+++def test_the_lock_is_re_entrant_and_shared_by_stores_on_the_same_file(tmp_path: Path) -> None:+ path = tmp_path / "secrets.json"+ one, two = SecretStore(path), SecretStore(path)+ with one.exclusive():+ with one.exclusive():+ two.put("a", {"n": 1}) # the same thread, another store on the same file+ assert one.get("a") == {"n": 1}+++class _Reply:+ def __init__(self, body: dict, status: int = 200) -> None:+ self.status_code = status+ self._body = body++ def json(self) -> dict:+ return self._body+++def test_two_callers_renew_a_connector_sign_in_once(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ path = tmp_path / "secrets.json"+ SecretStore(path).put(cloud.CLOUD_AUTH_PROFILE, {"access_token": "session", "expires": time.time() + 3600})+ SecretStore(path).put(+ "gmail:default",+ {"managed": True, "provider": "google", "access_token": "old", "refresh_token": "r1", "connection_id": "c1", "expires": time.time() - 10},+ )+ calls: list[str] = []++ def post(url, json=None, headers=None, timeout=None, **_):+ calls.append(json["refresh_token"])+ time.sleep(0.4) # both callers are inside their renewal before the first finishes+ return _Reply({"access_token": "new", "refresh_token": "r2", "expires_in": 3600})++ monkeypatch.setattr(cloud.httpx, "post", post)+ results: list = []++ def renew() -> None:+ # Each caller has its own store, as two processes would.+ results.append(cloud.refresh_managed_token(SecretStore(path), Config(), "gmail"))++ threads = [threading.Thread(target=renew) for _ in range(2)]+ for t in threads:+ t.start()+ for t in threads:+ t.join(timeout=30)++ assert calls == ["r1"] # one renewal; the old refresh token was used once+ assert [r["access_token"] for r in results] == ["new", "new"]+ assert SecretStore(path).get("gmail:default")["refresh_token"] == "r2"+++def test_two_callers_renew_the_cloud_session_once(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ path = tmp_path / "secrets.json"+ SecretStore(path).put(cloud.CLOUD_AUTH_PROFILE, {"access_token": "old", "refresh_token": "r1", "expires": time.time() - 10})+ calls: list[str] = []++ def post(url, data=None, timeout=None, **_):+ calls.append(data["refresh_token"])+ time.sleep(0.4)+ return _Reply({"access_token": "new", "refresh_token": "r2", "expires_in": 3600})++ monkeypatch.setattr(cloud.httpx, "post", post)+ results: list = []+ threads = [threading.Thread(target=lambda: results.append(cloud.fresh_access_token(SecretStore(path), Config()))) for _ in range(2)]+ for t in threads:+ t.start()+ for t in threads:+ t.join(timeout=30)+ assert calls == ["r1"] and results == ["new", "new"]+++def test_two_callers_renew_the_model_sign_in_once(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ path = tmp_path / "secrets.json"+ SecretStore(path).put(codex_auth.PROFILE, {"tokens": {"access_token": "old", "refresh_token": "r1"}, "account_id": "acct"})+ monkeypatch.setattr(codex_auth, "_jwt_claims", lambda token: {"exp": time.time() + (3600 if token == "new" else -10)})+ calls: list[str] = []++ def token_post(form):+ calls.append(form["refresh_token"])+ time.sleep(0.4)+ return _Reply({"access_token": "new", "refresh_token": "r2"})++ monkeypatch.setattr(codex_auth, "_token_post", token_post)+ results: list = []+ threads = [+ threading.Thread(target=lambda: results.append(codex_auth.CodexTokenStore(SecretStore(path)).access_token())) for _ in range(2)+ ]+ for t in threads:+ t.start()+ for t in threads:+ t.join(timeout=30)+ assert calls == ["r1"]+ assert [r[0] for r in results] == ["new", "new"]