AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): leave a skill review that another run may still be making alone (#15596)
Nicholas Tindle34e4e2375239
diff --git a/autogpt_platform/backend/backend/copilot/learning/_fake_store.py b/autogpt_platform/backend/backend/copilot/learning/_fake_store.pyindex 61a1f00d33..fcd176728f 100644--- a/autogpt_platform/backend/backend/copilot/learning/_fake_store.py+++ b/autogpt_platform/backend/backend/copilot/learning/_fake_store.py@@ -35,6 +35,7 @@ from backend.data.skill_publication import (VersionDraft,)from backend.data.skill_reviews import (+ LIVE_REVIEW_SECONDS,TERMINAL_REVIEW_DISPOSITIONS,LearningReviewRecord,LearningRunSummary,@@ -286,6 +287,7 @@ class FakeLearningStore:cost_microdollars: int | None = None,completed: bool = True,expected_run_id: str | None = None,+ live_seconds: int = LIVE_REVIEW_SECONDS,) -> LearningReviewRecord:source = await self.require_owned_source(user_id, source_id)key = self._review_key(source_id, source_revision, policy_version)@@ -298,6 +300,7 @@ class FakeLearningStore:existing.disposition in TERMINAL_REVIEW_DISPOSITIONSor existing.disposition == APPLIED_PENDING_DISPOSITIONor (expected_run_id is not None and existing.run_id != expected_run_id)+ or existing.held_by_another_run(run_id, live_seconds)):return existingif existing is not None and existing.replay is not None and not completed:@@ -343,6 +346,7 @@ class FakeLearningStore:output_tokens=output_tokens,cost_microdollars=cost_microdollars,created_at=existing.created_at if existing else _now(),+ updated_at=_now(),completed_at=_now() if completed else None,)self.reviews[key] = record@@ -358,6 +362,7 @@ class FakeLearningStore:run_id: str,max_attempts: int,expected_epoch: int,+ live_seconds: int = LIVE_REVIEW_SECONDS,) -> LearningReviewRecord | None:source = await self.get_source(user_id, source_id)if (@@ -375,6 +380,7 @@ class FakeLearningStore:or existing.disposition in TERMINAL_REVIEW_DISPOSITIONSor existing.disposition == "applied_pending"or existing.replay is not None+ or existing.held_by_another_run(run_id, live_seconds)):return Noneif existing is None:@@ -395,6 +401,7 @@ class FakeLearningStore:"disposition": "reviewing","reason": "review started","completed_at": None,+ "updated_at": _now(),})self.reviews[self._review_key(source_id, source_revision, policy_version)] = (@@ -430,6 +437,7 @@ class FakeLearningStore:"input_tokens": input_tokens,"output_tokens": output_tokens,"cost_microdollars": cost_microdollars,+ "updated_at": _now(),})return True@@ -447,7 +455,9 @@ class FakeLearningStore:or row.completed_at is not None):return False- self.reviews[key] = row.model_copy(update={"replay": replay})+ self.reviews[key] = row.model_copy(+ update={"replay": replay, "updated_at": _now()}+ )return Trueasync def get_review_for_revision(@@ -798,6 +808,7 @@ class FakeLearningStore:"disposition": "applied","reason": reason,"completed_at": _now(),+ "updated_at": _now(),"replay": None,})@@ -834,6 +845,7 @@ class FakeLearningStore:"disposition": disposition,"reason": reason,"completed_at": _now(),+ "updated_at": _now(),"replay": None,})diff --git a/autogpt_platform/backend/backend/copilot/learning/lease_test.py b/autogpt_platform/backend/backend/copilot/learning/lease_test.pyindex 7cc185def3..2878a7bce4 100644--- a/autogpt_platform/backend/backend/copilot/learning/lease_test.py+++ b/autogpt_platform/backend/backend/copilot/learning/lease_test.py@@ -1,8 +1,15 @@+from datetime import timedeltafrom unittest.mock import AsyncMockimport pytest-from backend.copilot.learning import nightly+from backend.copilot.learning import nightly, nightly_test+from backend.copilot.learning.nightly_test import USER, _run, _source+from backend.copilot.learning.reviewer import REVIEW_TIMEOUT_SECONDS+from backend.data.skill_reviews import LIVE_REVIEW_SECONDS++adapter = nightly_test.adapter+boundaries = nightly_test.boundaries@pytest.mark.asyncio@@ -33,3 +40,121 @@ async def test_only_confirmed_lease_contention_is_reported_as_held(else:assert not result.skipped and result.skip_reason != "lease_held"assert result.error == "RuntimeError: skill learning lease unavailable"+++async def _reviewed_by_another_run(+ store, source, attempts: int = nightly.MAX_MODEL_ATTEMPTS+):+ """The revision after ``attempts`` model attempts, the last one still+ reviewing under another run's id."""+ for _ in range(attempts):+ claim = await store.reserve_model_review(+ USER,+ source_id=source.id,+ source_revision=source.revision,+ policy_version=nightly.POLICY_VERSION,+ run_id="another-run",+ max_attempts=nightly.MAX_MODEL_ATTEMPTS,+ expected_epoch=source.epoch,+ )+ assert claim is not None+ return await store.get_review_for_revision(+ USER, source.id, source.revision, nightly.POLICY_VERSION+ )+++def _left_by_a_run_that_died(store, review):+ """Age the claim past any model call it could still be waiting on."""+ key = store._review_key(+ review.source_id, review.source_revision, review.policy_version+ )+ store.reviews[key] = review.model_copy(+ update={+ "updated_at": review.updated_at - timedelta(seconds=LIVE_REVIEW_SECONDS + 1)+ }+ )++[email protected]("attempts", range(1, nightly.MAX_MODEL_ATTEMPTS + 1))+async def test_a_review_another_run_may_still_be_making_is_left_alone(+ fake_store, adapter, boundaries, attempts+):+ """A run that lost its lease can still be inside its model call. Taking+ over or settling the revision here would overwrite its row and waste+ that paid review."""+ source = await _source(fake_store)+ live = await _reviewed_by_another_run(fake_store, source, attempts)++ result = await _run()++ assert result.dispositions == {"deferred": 1}+ boundaries["review"].assert_not_called()+ row = await fake_store.get_review_for_revision(+ USER, source.id, source.revision, nightly.POLICY_VERSION+ )+ assert row == live++[email protected]("attempts", range(1, nightly.MAX_MODEL_ATTEMPTS))+async def test_a_review_left_by_a_run_that_died_is_taken_over(+ fake_store, adapter, boundaries, attempts+):+ """Below the attempt limit, the revision is reviewed again."""+ source = await _source(fake_store)+ stale = await _reviewed_by_another_run(fake_store, source, attempts)+ _left_by_a_run_that_died(fake_store, stale)++ result = await _run()++ assert result.dispositions == {"applied": 1}+ boundaries["review"].assert_awaited_once()+ row = await fake_store.get_review_for_revision(+ USER, source.id, source.revision, nightly.POLICY_VERSION+ )+ assert (row.run_id, row.model_attempts) == (result.run_id, attempts + 1)+++async def test_a_review_left_by_a_run_that_died_is_settled(+ fake_store, adapter, boundaries+):+ """At the attempt limit, the revision settles as before."""+ source = await _source(fake_store)+ stale = await _reviewed_by_another_run(fake_store, source)+ _left_by_a_run_that_died(fake_store, stale)++ result = await _run()++ assert result.dispositions == {"skipped": 1}+ boundaries["review"].assert_not_called()+ row = await fake_store.get_review_for_revision(+ USER, source.id, source.revision, nightly.POLICY_VERSION+ )+ assert row.disposition == "skipped" and "attempt" in row.reason+++def test_a_claim_stays_live_for_longer_than_a_model_review_can_run():+ assert LIVE_REVIEW_SECONDS >= 2 * REVIEW_TIMEOUT_SECONDS+++async def test_an_exit_before_the_reservation_leaves_a_live_review_alone(+ fake_store, adapter, boundaries+):+ """The pass can stop on a source before reserving it, here because the+ budget ran out after the pass began; recording why must not overwrite+ another run's review either."""+ source = await _source(fake_store)+ live = await _reviewed_by_another_run(fake_store, source, 1)+ boundaries["budget"].side_effect = [(True, None), (False, "insufficient_credits")]++ result = await _run()++ assert result.dispositions == {"deferred": 1}+ boundaries["review"].assert_not_called()+ row = await fake_store.get_review_for_revision(+ USER, source.id, source.revision, nightly.POLICY_VERSION+ )+ assert row == livediff --git a/autogpt_platform/backend/backend/copilot/learning/nightly.py b/autogpt_platform/backend/backend/copilot/learning/nightly.pyindex c2a77d818f..f01665608b 100644--- a/autogpt_platform/backend/backend/copilot/learning/nightly.py+++ b/autogpt_platform/backend/backend/copilot/learning/nightly.py@@ -298,6 +298,14 @@ async def _process_source(expected_epoch=source.epoch,)if claim is None:+ previous = await reviews.get_review_for_revision(+ user_id, source.source_id, source.revision, POLICY_VERSION+ )+ if previous is not None and previous.held_by_another_run(result.run_id):+ # Every exit below writes this revision's row, which would cut+ # off that run's paid review mid-call; it settles the row itself.+ result.reviewed += 1+ return "deferred"current = await adapter.revalidate(source_id=source.source_id,revision=source.revision,@@ -313,9 +321,6 @@ async def _process_source(return await settle_ineligible(user_id, source, result, current.state, problem)- previous = await reviews.get_review_for_revision(- user_id, source.source_id, source.revision, POLICY_VERSION- )if previous is not None and previous.model_attempts >= MAX_MODEL_ATTEMPTS:return await settle(user_id,diff --git a/autogpt_platform/backend/backend/data/skill_review_claim_test.py b/autogpt_platform/backend/backend/data/skill_review_claim_test.pyindex bb56d76bd7..4e92733d19 100644--- a/autogpt_platform/backend/backend/data/skill_review_claim_test.py+++ b/autogpt_platform/backend/backend/data/skill_review_claim_test.py@@ -1,6 +1,8 @@import asyncio+from datetime import datetime, timedelta, timezonefrom uuid import uuid4+import prisma.modelsimport pytestfrom backend.data import skill_learning as learning@@ -29,6 +31,7 @@ async def test_replaced_worker_cannot_publish_after_newer_reservation(source_revision=source.revision,policy_version=1,max_attempts=3,+ live_seconds=0,expected_epoch=source.epoch,)await reviews.reserve_model_review(user, run_id="old", **params)@@ -73,6 +76,89 @@ async def test_replaced_worker_cannot_publish_after_newer_reservation(await _cleanup(user)[email protected](loop_scope="session")+async def test_a_live_claim_is_left_to_its_run_and_a_dead_one_is_taken_over(server):+ """A run that lost its lease can still be inside its paid model call;+ only a claim older than any call can run is taken over."""+ user = str(uuid4())+ await _create_user(user)+ try:+ source = await _source(user)+ params = dict(+ source_id=source.id,+ source_revision=source.revision,+ policy_version=1,+ max_attempts=3,+ expected_epoch=source.epoch,+ )+ claims = await asyncio.gather(+ *(+ reviews.reserve_model_review(user, run_id=f"run-{i}", **params)+ for i in range(3)+ )+ )+ [live] = [claim for claim in claims if claim is not None]+ assert (+ await reviews.get_review_for_revision(user, source.id, source.revision, 1)+ == live+ )+ died_at = datetime.now(timezone.utc) - timedelta(+ seconds=reviews.LIVE_REVIEW_SECONDS + 1+ )+ await prisma.models.SkillLearningReview.prisma().update(+ where={"id": live.id}, data={"updatedAt": died_at}+ )+ taken_over = await reviews.reserve_model_review(user, run_id="next", **params)+ assert taken_over is not None+ assert (taken_over.run_id, taken_over.model_attempts) == ("next", 2)+ assert taken_over.updated_at > died_at+ stored = await reviews.get_review_for_revision(+ user, source.id, source.revision, 1+ )+ assert stored.updated_at == taken_over.updated_at+ finally:+ await _cleanup(user)++[email protected](loop_scope="session")+async def test_another_runs_write_leaves_a_live_claim_alone(server):+ """Every ledger write, not only a reservation, leaves a claim another+ run may still be using as it is; once the claim is older than any+ call can run, the write applies."""+ user = str(uuid4())+ await _create_user(user)+ try:+ source = await _source(user)+ params = dict(+ source_id=source.id, source_revision=source.revision, policy_version=1+ )+ await reviews.reserve_model_review(+ user, **params, run_id="paid", max_attempts=3, expected_epoch=source.epoch+ )+ live = await reviews.get_review_for_revision(+ user, source.id, source.revision, 1+ )+ stopped = dict(+ **params, run_id="other", disposition="budget_exhausted", completed=False+ )+ assert await reviews.upsert_review(user, **stopped) == live+ assert (+ await reviews.get_review_for_revision(user, source.id, source.revision, 1)+ == live+ )+ await prisma.models.SkillLearningReview.prisma().update(+ where={"id": live.id},+ data={+ "updatedAt": datetime.now(timezone.utc)+ - timedelta(seconds=reviews.LIVE_REVIEW_SECONDS + 1)+ },+ )+ applied = await reviews.upsert_review(user, **stopped)+ assert (applied.run_id, applied.disposition) == ("other", "budget_exhausted")+ finally:+ await _cleanup(user)++@pytest.mark.asyncio(loop_scope="session")async def test_late_standalone_write_cannot_reopen_a_terminal_review(server):user = str(uuid4())@@ -106,7 +192,9 @@ async def test_lost_claim_cannot_settle_or_charge_the_replacement(server):params = dict(source_id=source.id, source_revision=source.revision, policy_version=1)- reservation = dict(**params, max_attempts=3, expected_epoch=source.epoch)+ reservation = dict(+ **params, max_attempts=3, live_seconds=0, expected_epoch=source.epoch+ )await reviews.reserve_model_review(user, run_id="old", **reservation)current = await reviews.reserve_model_review(user, run_id="new", **reservation)late = await reviews.upsert_review(@@ -115,7 +203,7 @@ async def test_lost_claim_cannot_settle_or_charge_the_replacement(server):run_id="old",expected_run_id="old",disposition="skipped",- cost_microdollars=2000+ cost_microdollars=2000,)assert late == currenthead = await versions.ensure_head(user, None, "claim-check")@@ -228,10 +316,15 @@ async def test_deferral_keeps_the_paid_replay_claim_and_missing_claim_cannot_be_run_id="missing",expected_run_id="missing",disposition="provider_error",- completed=False+ completed=False,)paid = await reviews.reserve_model_review(- user, **params, run_id="paid", max_attempts=3, expected_epoch=source.epoch+ user,+ **params,+ run_id="paid",+ max_attempts=3,+ live_seconds=0,+ expected_epoch=source.epoch,)assert await reviews.set_review_replay(user, paid.id, "paid", {"version": 1})deferred = await reviews.upsert_review(diff --git a/autogpt_platform/backend/backend/data/skill_review_replay_test.py b/autogpt_platform/backend/backend/data/skill_review_replay_test.pyindex b1ea02b983..b97638ebef 100644--- a/autogpt_platform/backend/backend/data/skill_review_replay_test.py+++ b/autogpt_platform/backend/backend/data/skill_review_replay_test.py@@ -19,6 +19,7 @@ async def test_model_attempt_reservation_is_atomic_and_owned(server):source_revision=source.revision,policy_version=1,max_attempts=3,+ live_seconds=0,expected_epoch=source.epoch,)assert (@@ -70,6 +71,7 @@ async def test_model_attempt_reservation_is_atomic_and_owned(server):run_id="settled",disposition="skipped",reason="attempt limit reached",+ live_seconds=0,)assert settled.cost_microdollars == 1000 and settled.input_tokens == 10assert settled.model_attempts == 3 and settled.replay is None@@ -93,6 +95,7 @@ async def test_checkpoint_replay_is_bounded_and_does_not_create_another_paid_attsource_revision=source.revision,policy_version=1,max_attempts=3,+ live_seconds=0,expected_epoch=source.epoch,)first = await reviews.reserve_model_review(user, run_id="paid", **params)diff --git a/autogpt_platform/backend/backend/data/skill_reviews.py b/autogpt_platform/backend/backend/data/skill_reviews.pyindex e6d5994236..67c0d099b8 100644--- a/autogpt_platform/backend/backend/data/skill_reviews.py+++ b/autogpt_platform/backend/backend/data/skill_reviews.py@@ -7,7 +7,7 @@ change. Every mutation enforces the owning ``user_id``.from __future__ import annotations-from datetime import datetime, timezone+from datetime import datetime, timedelta, timezonefrom typing import Anyimport prisma@@ -25,6 +25,12 @@ from backend.data.skill_learning import (from backend.data.skill_review_lock import lock_review_sourcefrom backend.util.json import SafeJson+# A model review is cut off after the reviewer's timeout (240s). A reviewing+# row another run reserved or checkpointed more recently than twice that may+# still be waiting on its paid call, so no other run may take it over or+# overwrite it; an older one was left by a run that died.+LIVE_REVIEW_SECONDS = 480+TERMINAL_REVIEW_DISPOSITIONS = frozenset({"applied",@@ -61,6 +67,7 @@ class LearningReviewRecord(BaseModel):output_tokens: intcost_microdollars: int | Nonecreated_at: datetime+ updated_at: datetimecompleted_at: datetime | None@classmethod@@ -89,9 +96,22 @@ class LearningReviewRecord(BaseModel):int(row.costMicrodollars) if row.costMicrodollars is not None else None),created_at=row.createdAt,+ updated_at=row.updatedAt,completed_at=row.completedAt,)+ def held_by_another_run(+ self, run_id: str, live_seconds: int = LIVE_REVIEW_SECONDS+ ) -> bool:+ """Another run reserved or checkpointed this review within+ ``live_seconds``, so its paid model call may still be running."""+ return (+ self.disposition == "reviewing"+ and self.run_id != run_id+ and datetime.now(timezone.utc) - self.updated_at+ < timedelta(seconds=live_seconds)+ )+def review_data(*,@@ -143,10 +163,12 @@ async def upsert_review_in(policy_version: int,data: dict[str, Any],expected_run_id: str | None = None,+ live_seconds: int = LIVE_REVIEW_SECONDS,) -> prisma.models.SkillLearningReview:"""Ledger upsert on ``client`` (a transaction or the default client).The caller must hold the owned source row lock throughout the update.+ A review another run is still making is returned unchanged."""rev = canonical_revision(source_revision)existing = await client.skilllearningreview.find_unique(@@ -178,6 +200,9 @@ async def upsert_review_in(existing.disposition in TERMINAL_REVIEW_DISPOSITIONSor existing.disposition == "applied_pending"or (expected_run_id is not None and existing.runId != expected_run_id)+ or LearningReviewRecord.from_db(existing).held_by_another_run(+ data["runId"], live_seconds+ )):return existingif existing.reviewReplay is not None and not data.get("completedAt"):@@ -207,6 +232,7 @@ async def upsert_review(cost_microdollars: int | None = None,completed: bool = True,expected_run_id: str | None = None,+ live_seconds: int = LIVE_REVIEW_SECONDS,) -> LearningReviewRecord:"""Write the ledger row for one review attempt (update in place on retry)."""async with transaction() as tx:@@ -220,6 +246,7 @@ async def upsert_review(source_revision=source_revision,policy_version=policy_version,expected_run_id=expected_run_id,+ live_seconds=live_seconds,data=review_data(run_id=run_id,disposition=disposition,@@ -316,7 +343,15 @@ async def reserve_model_review(run_id: str,max_attempts: int,expected_epoch: int,+ live_seconds: int = LIVE_REVIEW_SECONDS,) -> LearningReviewRecord | None:+ """Claim one paid model review of the revision for ``run_id``.++ Declines (``None``) a claim another run made within ``live_seconds``:+ taking it over would make that run's checkpoint fail after its call+ was paid for. An older ``reviewing`` row was left by a run that died,+ and is taken over.+ """async with transaction() as tx:source = await lock_review_source(tx, user_id, source_id)if source is None:@@ -342,6 +377,9 @@ async def reserve_model_review(or existing.disposition in TERMINAL_REVIEW_DISPOSITIONSor existing.disposition == "applied_pending"or existing.reviewReplay is not None+ or LearningReviewRecord.from_db(existing).held_by_another_run(+ run_id, live_seconds+ )):return Noneif existing is None: