Skip to content
Python

Stored sign-ins: one lock across processes, and a sign-in is renewed once

Rohit C Prasad61e614eb2465
diff --git a/coworker/cloud.py b/coworker/cloud.py
index 8e1b5e92..7db5be07 100644
--- a/coworker/cloud.py
+++ b/coworker/cloud.py
@@ -234,21 +234,30 @@ def fresh_access_token(secrets: SecretStore, config: Config) -> Optional[str]:
return None
if float(profile.get("expires") or 0) > _now():
return profile["access_token"]
- if not profile.get("refresh_token"):
- return None
- resp = httpx.post(
- f"https://{config.cloud_auth_domain}/oauth/token",
- data={
- "grant_type": "refresh_token",
- "client_id": config.cloud_client_id,
- "refresh_token": profile["refresh_token"],
- },
- timeout=15,
- )
- if resp.status_code != 200:
- return None
- _store_cloud_tokens(secrets, resp.json())
- return (secrets.get(CLOUD_AUTH_PROFILE) or {}).get("access_token")
+ # One renewal at a time, across processes (the app and each `openworker run` share
+ # this store). Read again inside the lock: whoever got here first has renewed it, and
+ # a second renewal with the refresh token it replaced could cancel the session.
+ with secrets.exclusive():
+ profile = secrets.get(CLOUD_AUTH_PROFILE) or {}
+ if not profile.get("access_token"):
+ return None
+ if float(profile.get("expires") or 0) > _now():
+ return profile["access_token"]
+ if not profile.get("refresh_token"):
+ return None
+ resp = httpx.post(
+ f"https://{config.cloud_auth_domain}/oauth/token",
+ data={
+ "grant_type": "refresh_token",
+ "client_id": config.cloud_client_id,
+ "refresh_token": profile["refresh_token"],
+ },
+ timeout=15,
+ )
+ if resp.status_code != 200:
+ return None
+ _store_cloud_tokens(secrets, resp.json())
+ return (secrets.get(CLOUD_AUTH_PROFILE) or {}).get("access_token")
def fetch_me(secrets: SecretStore, config: Config) -> Optional[dict]:
@@ -505,9 +514,27 @@ def refresh_managed_token(
that way. Manual profiles are never touched. `profile_key` targets an
account-keyed profile (`gmail:account:<email>`); default = `<name>:default`."""
key = profile_key or f"{connector}:default"
- profile = secrets.get(key) or {}
- if not (profile.get("managed") and profile.get("refresh_token")):
+ seen = secrets.get(key) or {}
+ if not (seen.get("managed") and seen.get("refresh_token")):
return None
+ # One renewal at a time, across processes; see fresh_access_token. If another process
+ # renewed this sign-in while this one waited for the lock, its token is the answer.
+ with secrets.exclusive():
+ profile = secrets.get(key) or {}
+ if not (profile.get("managed") and profile.get("refresh_token")):
+ return None
+ if (
+ profile.get("access_token")
+ and profile.get("access_token") != seen.get("access_token")
+ and float(profile.get("expires") or 0) > _now()
+ ):
+ return profile
+ return _renew_managed_token(secrets, config, connector, key, profile)
+
+
+def _renew_managed_token(
+ secrets: SecretStore, config: Config, connector: str, key: str, profile: dict[str, Any]
+) -> Optional[dict[str, Any]]:
provider = profile.get("provider") or PROVIDER_FOR_CONNECTOR.get(connector)
if not provider:
return None
diff --git a/coworker/providers/codex_auth.py b/coworker/providers/codex_auth.py
index 681a6d9f..7b554c56 100644
--- a/coworker/providers/codex_auth.py
+++ b/coworker/providers/codex_auth.py
@@ -244,7 +244,24 @@ class CodexTokenStore:
def refresh(self) -> tuple[str, str]:
"""refresh_token grant → fresh (access token, account id). A rejected refresh
token blanks the profile — the provider reads as cleanly signed out."""
- refresh = (self._data().get("tokens") or {}).get("refresh_token") or ""
+ seen = (self._data().get("tokens") or {}).get("access_token") or ""
+ exclusive = getattr(self._secrets, "exclusive", None)
+ if exclusive is None: # a store without the lock (tests' stand-ins)
+ return self._refresh_locked(seen)
+ # One renewal at a time, across processes: the app and each `openworker run`
+ # share this store, and a second renewal with the refresh token the first one
+ # replaced would sign the user out.
+ with exclusive():
+ return self._refresh_locked(seen)
+
+ def _refresh_locked(self, seen: str) -> tuple[str, str]:
+ data = self._data()
+ tokens = data.get("tokens") or {}
+ access = tokens.get("access_token") or ""
+ exp = _jwt_claims(access).get("exp")
+ if access and access != seen and isinstance(exp, (int, float)) and exp - time.time() >= REFRESH_MARGIN_SECONDS:
+ return access, data.get("account_id") or "" # another process just renewed it
+ refresh = tokens.get("refresh_token") or ""
if not refresh:
self.clear()
raise CodexSignInRequired(EXPIRED_ERROR)
diff --git a/coworker/secrets.py b/coworker/secrets.py
index 1607cdfe..4504de3b 100644
--- a/coworker/secrets.py
+++ b/coworker/secrets.py
@@ -10,7 +10,9 @@ a Keychain / age-encrypted backend can swap in later without touching them.
from __future__ import annotations
+import contextlib
import json
+import logging
import os
import re
import subprocess
@@ -138,13 +140,111 @@ def write_private_text(path: str | Path, content: str) -> Path:
return _atomic_private_write(Path(path).expanduser(), content)
+class _StoreLock:
+ """One lock per secrets file, across threads AND processes.
+
+ Several OpenWorker processes can share a state folder: the app's server, and each
+ `openworker run`. A change to the file is read, modify, write of the WHOLE file, so
+ without this two processes can each write and one change is lost. A sign-in renewal
+ is worse: two processes renewing the same sign-in at once can leave it cancelled at a
+ vendor that hands out a new refresh token each time.
+
+ Inside the process it is a re-entrant lock shared by every SecretStore on the same
+ file. Across processes it is an advisory lock on `<file>.lock` (flock; byte-range
+ locking on Windows), taken with a deadline: if another process holds it for too long,
+ this one goes ahead without it rather than stall a turn on a stuck peer."""
+
+ def __init__(self, path: Path) -> None:
+ self.path = path.with_name(path.name + ".lock")
+ self._thread = threading.RLock()
+ self._depth = 0
+ self._fh: Any = None
+
+ def acquire(self, timeout: float) -> None:
+ self._thread.acquire()
+ if self._depth == 0:
+ self._fh = self._lock_file(timeout)
+ self._depth += 1
+
+ def release(self) -> None:
+ self._depth -= 1
+ if self._depth == 0 and self._fh is not None:
+ fh, self._fh = self._fh, None
+ try:
+ if _IS_WINDOWS:
+ import msvcrt
+
+ fh.seek(0)
+ msvcrt.locking(fh.fileno(), msvcrt.LK_UNLCK, 1)
+ else:
+ import fcntl
+
+ fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
+ except (OSError, ImportError):
+ pass
+ fh.close()
+ self._thread.release()
+
+ def _lock_file(self, timeout: float) -> Any:
+ from .statelock import _try_lock
+
+ try:
+ self.path.parent.mkdir(parents=True, exist_ok=True)
+ # Made private from the first moment, like every file beside the secrets. It
+ # is empty and stays: it is the lock, not a temporary file.
+ fh = os.fdopen(os.open(self.path, os.O_RDWR | os.O_CREAT, 0o600), "r+")
+ except OSError:
+ return None # a read-only folder: nothing could be written there anyway
+ try:
+ _restrict_to_user(self.path, is_dir=False)
+ except OSError:
+ pass
+ deadline = time.monotonic() + max(0.0, timeout)
+ while True:
+ if _try_lock(fh):
+ return fh
+ if time.monotonic() >= deadline:
+ logging.getLogger(__name__).warning(
+ "the stored sign-ins stayed locked by another process for %.0fs; going ahead without the lock",
+ timeout,
+ )
+ fh.close()
+ return None
+ time.sleep(0.05)
+
+
+_STORE_LOCKS: dict[str, _StoreLock] = {}
+_STORE_LOCKS_GUARD = threading.Lock()
+
+
+def _store_lock(path: Path) -> _StoreLock:
+ key = os.path.realpath(str(path))
+ with _STORE_LOCKS_GUARD:
+ lock = _STORE_LOCKS.get(key)
+ if lock is None:
+ lock = _STORE_LOCKS[key] = _StoreLock(Path(key))
+ return lock
+
+
class SecretStore:
"""File-backed secret store. Reads resolve `${VAR}` refs; status never leaks values."""
def __init__(self, path: Optional[str | Path] = None) -> None:
self.path = Path(path).expanduser() if path else state_dir() / "secrets.json"
self._dotenv_path = self.path.parent / ".env"
- self._lock = threading.Lock()
+ self._lock = _store_lock(self.path)
+
+ @contextlib.contextmanager
+ def exclusive(self, timeout: float = 30.0):
+ """Hold the store for a read-then-write that no other thread or process may
+ interleave with. A sign-in renewal takes it, reads the profile AGAIN inside, and
+ renews only if nobody else just did: the first process renews, the others read
+ its result. Re-entrant: `put` and `delete` inside it are fine."""
+ self._lock.acquire(timeout)
+ try:
+ yield self
+ finally:
+ self._lock.release()
# -- reads ------------------------------------------------------------------
def get(self, profile: str) -> Optional[dict[str, Any]]:
@@ -193,13 +293,13 @@ class SecretStore:
# -- writes -----------------------------------------------------------------
def put(self, profile: str, data: dict[str, Any]) -> None:
- with self._lock:
+ with self.exclusive():
store = self._read()
store[profile] = data
self._write(store)
def delete(self, profile: str) -> bool:
- with self._lock:
+ with self.exclusive():
store = self._read()
if profile not in store:
return False
diff --git a/tests/test_secrets_file_mode.py b/tests/test_secrets_file_mode.py
index ae0c4f5d..846807e1 100644
--- a/tests/test_secrets_file_mode.py
+++ b/tests/test_secrets_file_mode.py
@@ -63,8 +63,10 @@ def test_temp_file_is_never_group_or_world_readable_mid_write(tmp_path, monkeypa
def test_no_temp_file_is_left_behind(tmp_path):
store = SecretStore(tmp_path / "secrets.json")
store.put("openai", {"api_key": "sk-live"})
- leftovers = [p.name for p in tmp_path.iterdir() if p.name != "secrets.json"]
+ # The lock file beside it is permanent and empty; it is not a temporary file.
+ leftovers = [p.name for p in tmp_path.iterdir() if p.name not in ("secrets.json", "secrets.json.lock")]
assert leftovers == []
+ assert (tmp_path / "secrets.json.lock").read_bytes() == b""
def test_content_round_trips(tmp_path):
@@ -90,7 +92,7 @@ def test_a_failed_write_leaves_the_previous_file_intact(tmp_path, monkeypatch):
store.put("openai", {"api_key": "sk-replacement"})
assert json.loads(path.read_text())["openai"]["api_key"] == "sk-original"
- assert [p.name for p in tmp_path.iterdir()] == ["secrets.json"], "temp must be cleaned up"
+ assert sorted(p.name for p in tmp_path.iterdir()) == ["secrets.json", "secrets.json.lock"], "temp must be cleaned up"
def test_a_hostile_preexisting_temp_name_cannot_redirect_the_write(tmp_path):
diff --git a/tests/test_secrets_lock.py b/tests/test_secrets_lock.py
new file mode 100644
index 00000000..9820f15f
--- /dev/null
+++ b/tests/test_secrets_lock.py
@@ -0,0 +1,183 @@
+"""The stored sign-ins are shared by every OpenWorker process on a computer: the app's
+server and each `openworker run`. A change must not be lost when two of them write, and a
+sign-in must be renewed once: the first process renews, the others read its result."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import textwrap
+import threading
+import time
+from pathlib import Path
+
+import pytest
+
+from coworker import cloud
+from coworker.config import Config
+from coworker.providers import codex_auth
+from coworker.secrets import SecretStore
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def _python(code: str, *args: str) -> subprocess.Popen:
+ env = dict(os.environ)
+ env["PYTHONPATH"] = str(ROOT) + os.pathsep + env.get("PYTHONPATH", "")
+ return subprocess.Popen([sys.executable, "-c", textwrap.dedent(code), *args], env=env, stdout=subprocess.PIPE, text=True)
+
+
+def test_writes_from_several_processes_are_all_kept(tmp_path: Path) -> None:
+ path = tmp_path / "secrets.json"
+ writer = """
+ import sys
+ from coworker.secrets import SecretStore
+ store = SecretStore(sys.argv[1])
+ for i in range(25):
+ store.put(f"{sys.argv[2]}:{i}", {"n": i})
+ """
+ procs = [_python(writer, str(path), name) for name in ("a", "b", "c", "d")]
+ for proc in procs:
+ assert proc.wait(timeout=120) == 0
+ stored = SecretStore(path)
+ missing = [f"{name}:{i}" for name in "abcd" for i in range(25) if stored.get(f"{name}:{i}") is None]
+ assert missing == []
+
+
+def test_a_second_process_waits_for_the_first(tmp_path: Path) -> None:
+ path = tmp_path / "secrets.json"
+ holder = _python(
+ """
+ import sys, time
+ from coworker.secrets import SecretStore
+ with SecretStore(sys.argv[1]).exclusive():
+ print("held", flush=True)
+ time.sleep(1.5)
+ """,
+ str(path),
+ )
+ assert holder.stdout.readline().strip() == "held"
+ started = time.monotonic()
+ with SecretStore(path).exclusive():
+ waited = time.monotonic() - started
+ assert holder.wait(timeout=30) == 0
+ assert waited > 0.8
+
+
+def test_a_stuck_holder_does_not_stall_a_turn_for_ever(tmp_path: Path) -> None:
+ path = tmp_path / "secrets.json"
+ holder = _python(
+ """
+ import sys, time
+ from coworker.secrets import SecretStore
+ with SecretStore(sys.argv[1]).exclusive():
+ print("held", flush=True)
+ time.sleep(20)
+ """,
+ str(path),
+ )
+ try:
+ assert holder.stdout.readline().strip() == "held"
+ started = time.monotonic()
+ store = SecretStore(path)
+ with store.exclusive(timeout=0.3):
+ store.put("openai", {"api_key": "sk-x"}) # goes ahead without the lock
+ assert time.monotonic() - started < 5
+ assert store.get("openai") == {"api_key": "sk-x"}
+ finally:
+ holder.kill()
+
+
+def test_the_lock_is_re_entrant_and_shared_by_stores_on_the_same_file(tmp_path: Path) -> None:
+ path = tmp_path / "secrets.json"
+ one, two = SecretStore(path), SecretStore(path)
+ with one.exclusive():
+ with one.exclusive():
+ two.put("a", {"n": 1}) # the same thread, another store on the same file
+ assert one.get("a") == {"n": 1}
+
+
+class _Reply:
+ def __init__(self, body: dict, status: int = 200) -> None:
+ self.status_code = status
+ self._body = body
+
+ def json(self) -> dict:
+ return self._body
+
+
+def test_two_callers_renew_a_connector_sign_in_once(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ path = tmp_path / "secrets.json"
+ SecretStore(path).put(cloud.CLOUD_AUTH_PROFILE, {"access_token": "session", "expires": time.time() + 3600})
+ SecretStore(path).put(
+ "gmail:default",
+ {"managed": True, "provider": "google", "access_token": "old", "refresh_token": "r1", "connection_id": "c1", "expires": time.time() - 10},
+ )
+ calls: list[str] = []
+
+ def post(url, json=None, headers=None, timeout=None, **_):
+ calls.append(json["refresh_token"])
+ time.sleep(0.4) # both callers are inside their renewal before the first finishes
+ return _Reply({"access_token": "new", "refresh_token": "r2", "expires_in": 3600})
+
+ monkeypatch.setattr(cloud.httpx, "post", post)
+ results: list = []
+
+ def renew() -> None:
+ # Each caller has its own store, as two processes would.
+ results.append(cloud.refresh_managed_token(SecretStore(path), Config(), "gmail"))
+
+ threads = [threading.Thread(target=renew) for _ in range(2)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join(timeout=30)
+
+ assert calls == ["r1"] # one renewal; the old refresh token was used once
+ assert [r["access_token"] for r in results] == ["new", "new"]
+ assert SecretStore(path).get("gmail:default")["refresh_token"] == "r2"
+
+
+def test_two_callers_renew_the_cloud_session_once(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ path = tmp_path / "secrets.json"
+ SecretStore(path).put(cloud.CLOUD_AUTH_PROFILE, {"access_token": "old", "refresh_token": "r1", "expires": time.time() - 10})
+ calls: list[str] = []
+
+ def post(url, data=None, timeout=None, **_):
+ calls.append(data["refresh_token"])
+ time.sleep(0.4)
+ return _Reply({"access_token": "new", "refresh_token": "r2", "expires_in": 3600})
+
+ monkeypatch.setattr(cloud.httpx, "post", post)
+ results: list = []
+ threads = [threading.Thread(target=lambda: results.append(cloud.fresh_access_token(SecretStore(path), Config()))) for _ in range(2)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join(timeout=30)
+ assert calls == ["r1"] and results == ["new", "new"]
+
+
+def test_two_callers_renew_the_model_sign_in_once(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ path = tmp_path / "secrets.json"
+ SecretStore(path).put(codex_auth.PROFILE, {"tokens": {"access_token": "old", "refresh_token": "r1"}, "account_id": "acct"})
+ monkeypatch.setattr(codex_auth, "_jwt_claims", lambda token: {"exp": time.time() + (3600 if token == "new" else -10)})
+ calls: list[str] = []
+
+ def token_post(form):
+ calls.append(form["refresh_token"])
+ time.sleep(0.4)
+ return _Reply({"access_token": "new", "refresh_token": "r2"})
+
+ monkeypatch.setattr(codex_auth, "_token_post", token_post)
+ results: list = []
+ threads = [
+ threading.Thread(target=lambda: results.append(codex_auth.CodexTokenStore(SecretStore(path)).access_token())) for _ in range(2)
+ ]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join(timeout=30)
+ assert calls == ["r1"]
+ assert [r[0] for r in results] == ["new", "new"]