AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): maintain idle OAuth refresh grants daily (#15639)
Nicholas Tindled772be3095b7
diff --git a/autogpt_platform/backend/backend/data/credential_refresh_test.py b/autogpt_platform/backend/backend/data/credential_refresh_test.pyindex 3cb6b3b0a7..4225d0c1a0 100644--- a/autogpt_platform/backend/backend/data/credential_refresh_test.py+++ b/autogpt_platform/backend/backend/data/credential_refresh_test.py@@ -12,15 +12,15 @@ from backend.util.encryption import JSONCryptor"access_expiry,refresh_expiry,eligible",[(None, None, False),- (999, None, False),- (1_000, None, False),- (1_001, None, True),- (1_299, None, True),- (1_300, None, False),- (1_301, None, False),- (1_100, 999, False),- (1_100, 1_000, False),- (1_100, 1_001, True),+ (790_399, None, True),+ (790_400, None, True),+ (790_401, None, False),+ (1_999_999, None, False),+ (2_000_000, None, False),+ (2_000_100, None, False),+ (790_399, 1_999_999, False),+ (790_399, 2_000_000, False),+ (790_399, 2_000_001, True),],)async def test_refresh_candidate_expiry_boundaries(@@ -42,9 +42,9 @@ async def test_refresh_candidate_expiry_boundaries()database = mocker.patch("backend.data.user.prisma")database.integrationcredential.find_many = AsyncMock(return_value=[row])- mocker.patch("time.time", return_value=1_000)+ mocker.patch("time.time", return_value=2_000_000)- result = await get_near_expiry_oauth_credentials(1_300)+ result = await get_near_expiry_oauth_credentials(2_000_000 - 14 * 86_400)assert result == ([("owner", credentials)] if eligible else [])diff --git a/autogpt_platform/backend/backend/data/user.py b/autogpt_platform/backend/backend/data/user.pyindex 8439a53bc6..0b154d8845 100644--- a/autogpt_platform/backend/backend/data/user.py+++ b/autogpt_platform/backend/backend/data/user.py@@ -467,10 +467,10 @@ async def get_user_credentials(user_id: str) -> list[Credentials]:async def get_near_expiry_oauth_credentials(refresh_before: int,) -> list[tuple[str, OAuth2Credentials]]:- """Return still-valid user OAuth credentials approaching expiry.+ """Return refreshable user OAuth credentials expired at or before the cutoff.This cross-user query is for trusted system callers such as the scheduler,- never user-scoped requests. Expired grants stay on the on-demand path.+ never user-scoped requests. Access-token expiry does not invalidate a refresh grant."""rows = await prisma.integrationcredential.find_many(where={"ownerType": "USER", "status": "active", "credentialType": "oauth2"}@@ -497,7 +497,7 @@ async def get_near_expiry_oauth_credentials(and not credential.is_managedand credential.refresh_token is not Noneand credential.access_token_expires_at is not None- and now < credential.access_token_expires_at < refresh_before+ and credential.access_token_expires_at <= refresh_beforeand (credential.refresh_token_expires_at is Noneor credential.refresh_token_expires_at > nowdiff --git a/autogpt_platform/backend/backend/data/user_test.py b/autogpt_platform/backend/backend/data/user_test.pyindex 49504873e6..c140061965 100644--- a/autogpt_platform/backend/backend/data/user_test.py+++ b/autogpt_platform/backend/backend/data/user_test.py@@ -1069,12 +1069,12 @@ class TestTableBackedCredentials:id="due",provider="github",access_token=SecretStr("access"),- access_token_expires_at=1_299,+ access_token_expires_at=790_399,refresh_token=SecretStr("refresh"),scopes=[],)future = due.model_copy(- update={"id": "future", "access_token_expires_at": 1_301}+ update={"id": "future", "access_token_expires_at": 790_401})provider_runtime = due.model_copy(update={"id": "runtime", "refresh_strategy": "provider_runtime"}@@ -1123,11 +1123,11 @@ class TestTableBackedCredentials:])mocker.patch("backend.data.user.prisma", mock_prisma)- mocker.patch("time.time", return_value=1_000)+ mocker.patch("time.time", return_value=2_000_000)- result = await get_near_expiry_oauth_credentials(1_300)+ result = await get_near_expiry_oauth_credentials(790_400)- assert result == [("user-due", due)]+ assert result == [("user-due", due), ("user-expired", expired)]mock_prisma.integrationcredential.find_many.assert_awaited_once_with(where={"ownerType": "USER", "status": "active", "credentialType": "oauth2"})diff --git a/autogpt_platform/backend/backend/executor/credential_sweep_test.py b/autogpt_platform/backend/backend/executor/credential_sweep_test.pynew file mode 100644index 0000000000..167965bf0f--- /dev/null+++ b/autogpt_platform/backend/backend/executor/credential_sweep_test.py@@ -0,0 +1,115 @@+import threading+from datetime import datetime, timedelta, timezone++import pytest+from apscheduler.events import EVENT_JOB_EXECUTED+from apscheduler.job import Job+from apscheduler.schedulers.background import BackgroundScheduler+from apscheduler.triggers.cron import CronTrigger+from apscheduler.triggers.interval import IntervalTrigger++from backend.executor.jobstore import ResilientSQLAlchemyJobStore+from backend.executor.scheduler import Scheduler, refresh_near_expiry_credentials++JOB_ID = "refresh_near_expiry_credentials"+++def _scheduler(database_url):+ return BackgroundScheduler(+ jobstores={"execution": ResilientSQLAlchemyJobStore(url=database_url)},+ job_defaults={"coalesce": True, "misfire_grace_time": None, "max_instances": 1},+ timezone=timezone.utc,+ )++[email protected]("paused", [False, True])+def test_persisted_idle_sweep_catches_up_once_after_restart(+ tmp_path, monkeypatch, paused+):+ database_url = f"sqlite:///{(tmp_path / 'jobs.sqlite').as_posix()}"+ due = datetime.now(timezone.utc).replace(hour=3, minute=0, second=0, microsecond=0)+ due -= timedelta(days=2)+ first = ResilientSQLAlchemyJobStore(url=database_url)+ seed_scheduler = BackgroundScheduler(timezone=timezone.utc)+ first.start(seed_scheduler, "execution")+ try:+ first.add_job(+ Job(+ seed_scheduler,+ func=refresh_near_expiry_credentials,+ id=JOB_ID,+ trigger=CronTrigger(hour=3, minute=0, timezone=timezone.utc),+ next_run_time=None if paused else due,+ executor="default",+ args=(),+ kwargs={},+ coalesce=True,+ misfire_grace_time=None,+ max_instances=1,+ )+ )+ finally:+ first.shutdown()++ restarted = _scheduler(database_url)+ restarted.start(paused=True)+ service = Scheduler(register_system_tasks=False)+ service.scheduler = restarted+ executed = threading.Event()+ calls = []++ def complete(coro, timeout):+ coro.close()+ calls.append(timeout)++ monkeypatch.setattr("backend.executor.scheduler.run_async", complete)+ restarted.add_listener(lambda event: executed.set(), EVENT_JOB_EXECUTED)+ try:+ service._register_idle_credential_sweep()+ job = restarted.get_job(JOB_ID, jobstore="execution")+ assert job is not None+ assert job.next_run_time == (None if paused else due)+ assert len(restarted.get_jobs(jobstore="execution")) == 1+ if not paused:+ restarted.resume()+ assert executed.wait(5)+ assert calls == [None]+ updated = restarted.get_job(JOB_ID, jobstore="execution")+ assert updated is not None+ assert updated.next_run_time > datetime.now(timezone.utc)+ finally:+ restarted.shutdown()++[email protected]("previous", ["missing", "interval", "different_timezone"])+def test_idle_sweep_installs_or_migrates_the_daily_job(tmp_path, previous):+ scheduler = _scheduler(f"sqlite:///{(tmp_path / 'jobs.sqlite').as_posix()}")+ scheduler.start(paused=True)+ service = Scheduler(register_system_tasks=False)+ service.scheduler = scheduler+ try:+ if previous != "missing":+ scheduler.add_job(+ refresh_near_expiry_credentials,+ id=JOB_ID,+ trigger=(+ IntervalTrigger(seconds=60)+ if previous == "interval"+ else CronTrigger(hour=3, minute=0, timezone="America/Chicago")+ ),+ jobstore="execution",+ )+ service._register_idle_credential_sweep()+ job = scheduler.get_job(JOB_ID, jobstore="execution")+ assert job is not None+ expected = CronTrigger(hour=3, minute=0, timezone=timezone.utc)+ assert str(job.trigger) == str(expected)+ assert job.next_run_time == expected.get_next_fire_time(+ None, datetime.now(timezone.utc)+ )+ assert job.max_instances == 1+ assert job.coalesce is True+ assert job.misfire_grace_time is None+ assert len(scheduler.get_jobs(jobstore="execution")) == 1+ finally:+ scheduler.shutdown()diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex a8adbf1530..ce2f33dfed 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -1855,9 +1855,7 @@ def _clear_dream_registration_marker(user_id: str, key_prefix: str) -> None:def refresh_near_expiry_credentials() -> None:"""Keep max_instances guarding the sweep until every refresh finishes."""- run_async(- IntegrationCredentialsManager().refresh_near_expiry_credentials(), timeout=None- )+ run_async(IntegrationCredentialsManager().refresh_idle_credentials(), timeout=None)def cleanup_oauth_tokens():@@ -2583,17 +2581,6 @@ class Scheduler(AppService):jobstore=Jobstores.EXECUTION.value,)- # Credential Refresh - keep third-party OAuth credentials valid before use.- self.scheduler.add_job(- refresh_near_expiry_credentials,- id="refresh_near_expiry_credentials",- trigger="interval",- replace_existing=True,- max_instances=1,- seconds=60,- jobstore=Jobstores.EXECUTION.value,- )-# OAuth Token Cleanup - configurable intervalself.scheduler.add_job(cleanup_oauth_tokens,@@ -2704,6 +2691,7 @@ class Scheduler(AppService):if self.register_system_tasks:# After start: until then get_job only sees pending jobs, not the# jobstore, so the "leave an unchanged job alone" check can't work.+ self._register_idle_credential_sweep()self._register_posthog_lifecycle_sweep()self._report_parked_jobs()@@ -2714,6 +2702,28 @@ class Scheduler(AppService):# Keep the service running since BackgroundScheduler doesn't blocksuper().run_service()+ def _register_idle_credential_sweep(self) -> None:+ """Keep an existing daily job's overdue run or explicit pause intact."""+ job_id = "refresh_near_expiry_credentials"+ trigger = CronTrigger(hour=3, minute=0, timezone=timezone.utc)+ existing = self.scheduler.get_job(job_id, jobstore=Jobstores.EXECUTION.value)+ if (+ existing is not None+ and str(existing.trigger) == str(trigger)+ and _job_timezone_name(existing) == "UTC"+ ):+ return+ self.scheduler.add_job(+ refresh_near_expiry_credentials,+ id=job_id,+ trigger=trigger,+ replace_existing=True,+ max_instances=1,+ coalesce=True,+ misfire_grace_time=None,+ jobstore=Jobstores.EXECUTION.value,+ )+# Paused and fired-once rows are never deleted, so this scan is bounded# rather than left to grow with the backlog — startup precedes the RPC port.def _report_parked_jobs(self) -> None:diff --git a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py b/autogpt_platform/backend/backend/executor/scheduler_unit_test.pyindex 61834f9907..056a337798 100644--- a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py@@ -1682,7 +1682,10 @@ class _StartupRun(NamedTuple):def _registered_jobs(- monkeypatch, interval_hours: int, startup_embedding_backfill: bool = True+ monkeypatch,+ interval_hours: int,+ startup_embedding_backfill: bool = True,+ existing_idle_job=None,) -> _StartupRun:"""Drive ``Scheduler.run_service`` with every heavy dependency stubbed anda mock APScheduler, recording what it registered and what it ran before@@ -1696,6 +1699,11 @@ def _registered_jobs(startup_embedding_backfill,)mock_scheduler = MagicMock()+ mock_scheduler.get_job.side_effect = lambda job_id, **_: (+ existing_idle_job+ if mock_scheduler.start.called and job_id == "refresh_near_expiry_credentials"+ else None+ )embedding_backfill = MagicMock(return_value=None)at_readiness = []with (@@ -1743,7 +1751,30 @@ def test_reconcile_stripe_tiers_job_registered_with_interval_and_single_instanceassert call.kwargs["seconds"] == 6 * 3600-def test_refresh_near_expiry_credentials_job_is_registered_every_minute(monkeypatch):[email protected]("offset_days", [-2, 1, None])+def test_idle_credentials_startup_preserves_existing_daily_job(+ monkeypatch, offset_days+):+ existing = MagicMock(+ trigger=CronTrigger(hour=3, minute=0, timezone=timezone.utc),+ next_run_time=(+ datetime.now(timezone.utc) + timedelta(days=offset_days)+ if offset_days is not None+ else None+ ),+ )+ calls = _registered_jobs(+ monkeypatch, interval_hours=6, existing_idle_job=existing+ ).add_job_calls++ assert not [+ call+ for call in calls+ if call.args and call.args[0] is refresh_near_expiry_credentials+ ]+++def test_idle_credentials_job_is_registered_daily(monkeypatch):calls = _registered_jobs(monkeypatch, interval_hours=6).add_job_callsmatches = [@@ -1755,8 +1786,9 @@ def test_refresh_near_expiry_credentials_job_is_registered_every_minute(monkeypaassert len(matches) == 1call = matches[0]assert call.kwargs["id"] == "refresh_near_expiry_credentials"- assert call.kwargs["trigger"] == "interval"- assert call.kwargs["seconds"] == 60+ assert str(call.kwargs["trigger"]) == str(+ CronTrigger(hour=3, minute=0, timezone=timezone.utc)+ )assert call.kwargs["replace_existing"] is Trueassert call.kwargs["max_instances"] == 1assert call.kwargs["jobstore"] == "execution"@@ -1781,7 +1813,7 @@ def test_refresh_job_stays_active_until_the_coroutine_finishes(monkeypatch):monkeypatch.setattr(scheduler_module,"IntegrationCredentialsManager",- lambda: MagicMock(refresh_near_expiry_credentials=refresh),+ lambda: MagicMock(refresh_idle_credentials=refresh),)thread.start()try:diff --git a/autogpt_platform/backend/backend/integrations/creds_manager.py b/autogpt_platform/backend/backend/integrations/creds_manager.pyindex 19182443bf..7a6604117a 100644--- a/autogpt_platform/backend/backend/integrations/creds_manager.py+++ b/autogpt_platform/backend/backend/integrations/creds_manager.py@@ -239,9 +239,9 @@ class IntegrationCredentialsManager:return get_credentials- async def refresh_near_expiry_credentials(self) -> None:- """Refresh eligible OAuth credentials that enter the five-minute window."""- refresh_before = int(time.time()) + 300+ async def refresh_idle_credentials(self) -> None:+ """Maintain unused refresh grants after 14 days of access-token expiry."""+ refresh_before = int(time.time()) - 14 * 24 * 60 * 60candidates = await self.store.db_manager.get_near_expiry_oauth_credentials(refresh_before)@@ -251,7 +251,7 @@ class IntegrationCredentialsManager:await self.refresh_if_needed(user_id, credentials)except Exception as error:logger.warning(- f"Proactive refresh failed for credential #{credentials.id} "+ f"Idle refresh failed for credential #{credentials.id} "f"belonging to user {user_id[:12]} ({type(error).__name__})")diff --git a/autogpt_platform/backend/backend/integrations/creds_manager_test.py b/autogpt_platform/backend/backend/integrations/creds_manager_test.pyindex d01697d2d5..64fbe59346 100644--- a/autogpt_platform/backend/backend/integrations/creds_manager_test.py+++ b/autogpt_platform/backend/backend/integrations/creds_manager_test.py@@ -6,6 +6,7 @@ import pytestfrom pydantic import SecretStrfrom backend.data.model import APIKeyCredentials, OAuth2Credentials+from backend.data.user import get_near_expiry_oauth_credentialsfrom backend.integrations.creds_manager import (IntegrationCredentialsManager,_invoke_creds_changed_hook,@@ -13,6 +14,7 @@ from backend.integrations.creds_manager import (unregister_creds_changed_hook,)from backend.integrations.oauth.github import GitHubOAuthHandler+from backend.util.encryption import JSONCryptor@pytest.fixture(autouse=True)@@ -88,7 +90,7 @@ class TestInvokeCredsChangedHook:@pytest.mark.asyncio-async def test_refresh_near_expiry_credentials_refreshes_each_candidate(mocker):+async def test_refresh_idle_credentials_refreshes_each_candidate(mocker):manager = IntegrationCredentialsManager()first = _provider_runtime_credentials().model_copy(update={"id": "cred-1", "refresh_strategy": "oauth_handler"}@@ -102,9 +104,9 @@ async def test_refresh_near_expiry_credentials_refreshes_each_candidate(mocker):refresh = mocker.patch.object(manager, "refresh_if_needed", AsyncMock())with patch("backend.integrations.creds_manager.time.time", return_value=1_000):- await manager.refresh_near_expiry_credentials()+ await manager.refresh_idle_credentials()- get_candidates.assert_awaited_once_with(1_300)+ get_candidates.assert_awaited_once_with(1_000 - 14 * 86_400)assert refresh.await_args_list == [(("user-1", first), {}),(("user-2", second), {}),@@ -112,9 +114,7 @@ async def test_refresh_near_expiry_credentials_refreshes_each_candidate(mocker):@pytest.mark.asyncio-async def test_refresh_near_expiry_credentials_continues_after_a_failure(- mocker, caplog-):+async def test_refresh_idle_credentials_continues_after_a_failure(mocker, caplog):manager = IntegrationCredentialsManager()first = _provider_runtime_credentials().model_copy(update={"id": "cred-1", "refresh_strategy": "oauth_handler"}@@ -131,7 +131,7 @@ async def test_refresh_near_expiry_credentials_continues_after_a_failure(AsyncMock(side_effect=[RuntimeError("provider rejected secret-token"), None]),)- await manager.refresh_near_expiry_credentials()+ await manager.refresh_idle_credentials()assert refresh.await_args_list == [(("user-1", first), {}),@@ -141,22 +141,39 @@ async def test_refresh_near_expiry_credentials_continues_after_a_failure(@pytest.mark.asyncio-async def test_sweep_persists_rotated_tokens_and_does_not_refresh_them_twice(mocker):[email protected]("idle_days", [15, 30])[email protected]("fail_first", [False, True])+async def test_idle_sweep_maintains_unused_credentials_and_recovers_after_failure(+ mocker, idle_days, fail_first+):manager = IntegrationCredentialsManager()+ now = 1_750_000_000original = _provider_runtime_credentials().model_copy(update={"provider": "github","refresh_strategy": "oauth_handler",- "access_token_expires_at": 1_100,+ "access_token_expires_at": now - idle_days * 86_400,+ "refresh_token_expires_at": now + 30 * 86_400,})stored = {original.id: original}handler = GitHubOAuthHandler("client", "secret", "https://example.com/callback")- mocker.patch("time.time", return_value=1_000)+ clock = mocker.patch("time.time", return_value=now)+ database = mocker.patch("backend.data.user.prisma")+ database.integrationcredential.find_many = AsyncMock(+ side_effect=lambda **_: [+ MagicMock(+ id=c.id,+ ownerId="user-a",+ encryptedPayload=JSONCryptor().encrypt(c.model_dump()),+ )+ for c in stored.values()+ ]+ )mocker.patch.object(manager.store.db_manager,"get_near_expiry_oauth_credentials",- AsyncMock(return_value=[("user-a", original)]),+ AsyncMock(side_effect=get_near_expiry_oauth_credentials),)mocker.patch.object(manager, "_get_oauth_handler", AsyncMock(return_value=handler))mocker.patch.object(manager, "_locked", return_value=_noop_lock_context())@@ -168,6 +185,7 @@ async def test_sweep_persists_rotated_tokens_and_does_not_refresh_them_twice(moc)async def persist(_user, credentials):+ assert _user == "user-a"stored[credentials.id] = credentialsmocker.patch.object(manager.store, "update_creds", AsyncMock(side_effect=persist))@@ -176,23 +194,39 @@ async def test_sweep_persists_rotated_tokens_and_does_not_refresh_them_twice(moc"access_token": "rotated-access","refresh_token": "rotated-refresh","expires_in": 3_600,- "refresh_token_expires_in": 7_200,+ "refresh_token_expires_in": 60 * 86_400,"scope": "repo",}- post = AsyncMock(return_value=response)+ post = AsyncMock(+ side_effect=([RuntimeError("temporary failure")] if fail_first else [])+ + [response, response]+ )mocker.patch("backend.integrations.oauth.github.Requests", return_value=MagicMock(post=post))mocker.patch.object(handler, "_request_username", AsyncMock(return_value="user"))- await manager.refresh_near_expiry_credentials()- await manager.refresh_near_expiry_credentials()+ await manager.refresh_idle_credentials()+ if fail_first:+ assert stored[original.id] == original+ clock.return_value += 86_400+ await manager.refresh_idle_credentials()- post.assert_awaited_once()- assert post.call_args.kwargs["data"]["refresh_token"] == "refresh"+ assert post.await_count == 1 + fail_first+ assert post.await_args.kwargs["data"]["refresh_token"] == "refresh"assert stored[original.id].access_token.get_secret_value() == "rotated-access"assert stored[original.id].refresh_token == SecretStr("rotated-refresh")- assert stored[original.id].access_token_expires_at == 4_600+ assert stored[original.id].access_token_expires_at == clock.return_value + 3_600++ for _ in range(14):+ clock.return_value += 86_400+ await manager.refresh_idle_credentials()+ assert post.await_count == 1 + fail_first++ clock.return_value += 86_400+ await manager.refresh_idle_credentials()+ assert post.await_count == 2 + fail_first+ assert post.await_args.kwargs["data"]["refresh_token"] == "rotated-refresh"@pytest.mark.asyncio