AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(platform): pause a scheduled chat that keeps failing the same way, and say why (#15375)
Nicholas Tindle19845cf6b08e
diff --git a/autogpt_platform/backend/backend/api/features/experts/routines.py b/autogpt_platform/backend/backend/api/features/experts/routines.pyindex 83103a7b9a..365c5a38cf 100644--- a/autogpt_platform/backend/backend/api/features/experts/routines.py+++ b/autogpt_platform/backend/backend/api/features/experts/routines.py@@ -46,6 +46,7 @@ from backend.api.features.experts.routine_jobs import (spread_cron,)from backend.copilot.credential_selection import CredentialPin, CredentialPins+from backend.copilot.scheduled_chats.forget import forget_routinefrom backend.data.user import get_user_by_idfrom backend.util.clients import get_scheduler_clientfrom backend.util.timezone_utils import get_user_timezone_or_utc@@ -428,6 +429,17 @@ async def disable_routine()if updated is None:raise RoutineNotFoundError(routine_id)+ # Switched off is no longer paused for failing: its jobs are gone. The+ # routine is off whatever this does, so a failure is logged, not raised;+ # a streak left behind ends on its first run once switched back on.+ try:+ await forget_routine(user_id, row.id)+ except Exception:+ logger.warning(+ "Could not clear the failure streak of switched-off routine %s",+ row.id,+ exc_info=True,+ )return to_model(updated)diff --git a/autogpt_platform/backend/backend/api/features/schedules/model.py b/autogpt_platform/backend/backend/api/features/schedules/model.pyindex 10e347c619..eecb4719f8 100644--- a/autogpt_platform/backend/backend/api/features/schedules/model.py+++ b/autogpt_platform/backend/backend/api/features/schedules/model.py@@ -22,6 +22,10 @@ class ScheduleCreationRequest(pydantic.BaseModel):)+class ResumeScheduledChatResponse(pydantic.BaseModel):+ resumed: bool++class ScheduleUpdateRequest(pydantic.BaseModel):"""Fields to change on an existing schedule; omitted fields keep their value."""diff --git a/autogpt_platform/backend/backend/api/features/schedules/routes.py b/autogpt_platform/backend/backend/api/features/schedules/routes.pyindex dcea041fb1..3eddfe4361 100644--- a/autogpt_platform/backend/backend/api/features/schedules/routes.py+++ b/autogpt_platform/backend/backend/api/features/schedules/routes.py@@ -1,17 +1,20 @@+import loggingfrom typing import Annotated, Anyfrom autogpt_libs.auth import get_request_context, get_user_id, requires_userfrom autogpt_libs.auth.models import RequestContextfrom fastapi import APIRouter, Body, Depends, HTTPException, Path, Security-from starlette.status import HTTP_404_NOT_FOUND+from starlette.status import HTTP_404_NOT_FOUND, HTTP_503_SERVICE_UNAVAILABLEfrom backend.api.features.experts import experts_dbfrom backend.api.features.schedule_visibility import visible_graph_schedulesfrom backend.api.features.schedules.model import (+ ResumeScheduledChatResponse,ScheduleCreationRequest,ScheduleUpdateRequest,)from backend.copilot.rate_limit import enforce_payment_paywall+from backend.copilot.scheduled_chats import service as scheduled_chatsfrom backend.data import graph as graph_dbfrom backend.data.onboarding import OnboardingStep, complete_onboarding_stepfrom backend.data.tenancy import get_user_team_ids@@ -24,7 +27,9 @@ from backend.util.timezone_utils import (get_user_timezone_or_utc,)-# This router mounts at /api and keeps the section's full paths: its six routes+logger = logging.getLogger(__name__)++# This router mounts at /api and keeps the section's full paths: its routes# span two prefixes (/schedules and /graphs/{graph_id}/schedules), so a prefixed# mount would have to move one of them.router = APIRouter(dependencies=[Security(requires_user)])@@ -167,6 +172,50 @@ async def list_copilot_turn_schedules(return [s for s in schedules if isinstance(s, scheduler.CopilotTurnJobInfo)]+ path="/schedules/followups/paused",+ summary="List scheduled chats paused after failing the same way",+ operation_id="listPausedScheduledChats",+)+async def list_paused_scheduled_chats(+ user_id: Annotated[str, Security(get_user_id)],+) -> list[scheduled_chats.PausedScheduledChat]:+ """A paused job drops out of the follow-ups list above, so the chats a+ failure streak paused are listed here, each once however many fire times+ it has, with why and what Resume takes."""+ return await scheduled_chats.list_paused_scheduled_chats(user_id)+++ dependencies=[Depends(enforce_payment_paywall)],+ path="/schedules/followups/{schedule_key}/resume",+ summary="Resume a scheduled chat paused after failing the same way",+ operation_id="resumeScheduledChat",+)+async def resume_scheduled_chat(+ user_id: Annotated[str, Security(get_user_id)],+ schedule_key: str = Path(+ ..., description="The routine's id, or the follow-up's schedule id"+ ),+) -> ResumeScheduledChatResponse:+ try:+ resumed = await scheduled_chats.resume_scheduled_chat(user_id, schedule_key)+ except Exception:+ # The scheduler refused or the database blinked: the chat stays+ # paused, so Resume can simply be tried again.+ logger.exception("Could not resume scheduled chat %s", schedule_key)+ raise HTTPException(+ status_code=HTTP_503_SERVICE_UNAVAILABLE,+ detail="This scheduled chat could not be resumed just now. Try again.",+ )+ if not resumed:+ raise HTTPException(+ status_code=HTTP_404_NOT_FOUND,+ detail="This scheduled chat is not paused.",+ )+ return ResumeScheduledChatResponse(resumed=True)++@router.delete(path="/schedules/{schedule_id}",summary="Delete execution schedule",@@ -179,13 +228,32 @@ async def delete_graph_execution_schedule(try:await get_scheduler_client().delete_schedule(schedule_id, user_id=user_id)except NotFoundError:+ # Gone already, perhaps by a delete whose streak cleanup failed: a+ # retry still clears the streak, or the chat stays listed as paused.+ await _forget_scheduled_chat(user_id, schedule_id)raise HTTPException(status_code=HTTP_404_NOT_FOUND,detail=f"Schedule #{schedule_id} not found",)+ # A follow-up's failure streak is keyed by its schedule id.+ await _forget_scheduled_chat(user_id, schedule_id)return {"id": schedule_id}+async def _forget_scheduled_chat(user_id: str, schedule_id: str) -> None:+ """The schedule is gone whatever this does, so a failure is logged, not+ answered: a 500 would say the delete failed. A streak left behind keeps+ the chat listed as paused, and deleting it from there again clears it."""+ try:+ await scheduled_chats.forget_scheduled_chat(user_id, schedule_id)+ except Exception:+ logger.warning(+ "Could not clear the failure streak of deleted schedule %s",+ schedule_id,+ exc_info=True,+ )++@router.patch(dependencies=[Depends(enforce_payment_paywall)],path="/schedules/{schedule_id}",diff --git a/autogpt_platform/backend/backend/api/features/schedules/routes_test.py b/autogpt_platform/backend/backend/api/features/schedules/routes_test.pyindex e6e2cc77fe..e147e94ee6 100644--- a/autogpt_platform/backend/backend/api/features/schedules/routes_test.py+++ b/autogpt_platform/backend/backend/api/features/schedules/routes_test.py@@ -1,5 +1,6 @@"""Tests for the schedule routes."""+from datetime import datetime, timezonefrom unittest.mock import AsyncMock, Mockimport fastapi@@ -10,6 +11,7 @@ from fastapi.routing import APIRoutefrom backend.api.features.schedules.routes import routerfrom backend.api.rest_api import app as real_app+from backend.copilot.scheduled_chats.service import PausedScheduledChatfrom backend.executor import schedulerfrom backend.util.exceptions import NotAuthorizedError, NotFoundError@@ -53,6 +55,11 @@ EXPECTED_OPERATIONS = {),("get", "/api/schedules"): "getV1List execution schedules for a user",("get", "/api/schedules/followups"): "listCopilotFollowupSchedules",+ ("get", "/api/schedules/followups/paused"): "listPausedScheduledChats",+ (+ "post",+ "/api/schedules/followups/{schedule_key}/resume",+ ): "resumeScheduledChat",("delete", "/api/schedules/{schedule_id}"): "deleteV1Delete execution schedule",("patch", "/api/schedules/{schedule_id}"): "patchV1Update execution schedule",}@@ -225,6 +232,10 @@ def test_list_all_schedules_does_not_scope_to_a_graph(def test_delete_schedule_returns_the_id(mocker: pytest_mock.MockFixture) -> None:sched = _patch_scheduler(mocker, delete_schedule=AsyncMock())+ mocker.patch(+ "backend.api.features.schedules.routes.scheduled_chats.forget_scheduled_chat",+ AsyncMock(),+ )response = client.delete("/schedules/sched-1")@@ -233,12 +244,135 @@ def test_delete_schedule_returns_the_id(mocker: pytest_mock.MockFixture) -> Noneassert sched.delete_schedule.await_args.args[0] == "sched-1"+def test_deleting_a_schedule_forgets_its_failure_streak(+ mocker: pytest_mock.MockFixture, test_user_id: str+) -> None:+ """A paused chat that is deleted must not stay listed as paused, raise+ its alert, or answer Resume."""+ _patch_scheduler(mocker, delete_schedule=AsyncMock())+ forget = mocker.patch(+ "backend.api.features.schedules.routes.scheduled_chats.forget_scheduled_chat",+ AsyncMock(),+ )++ response = client.delete("/schedules/sched-1")++ assert response.status_code == 200+ forget.assert_awaited_once_with(test_user_id, "sched-1")+++def test_a_retried_delete_still_forgets_the_streak(+ mocker: pytest_mock.MockFixture, test_user_id: str+) -> None:+ """The job went, then the streak cleanup failed: the retry finds no job,+ and must still clear the streak, or the chat stays listed as paused."""+ _patch_scheduler(+ mocker, delete_schedule=AsyncMock(side_effect=NotFoundError("gone"))+ )+ forget = mocker.patch(+ "backend.api.features.schedules.routes.scheduled_chats.forget_scheduled_chat",+ AsyncMock(),+ )++ assert client.delete("/schedules/sched-1").status_code == 404+ forget.assert_awaited_once_with(test_user_id, "sched-1")+++def test_a_delete_whose_streak_cleanup_fails_still_reports_the_delete(+ mocker: pytest_mock.MockFixture,+) -> None:+ """The job is gone by then. A 500 would tell the owner the delete+ failed; the paused row it leaves is cleared by deleting it again."""+ _patch_scheduler(mocker, delete_schedule=AsyncMock())+ mocker.patch(+ "backend.api.features.schedules.routes.scheduled_chats.forget_scheduled_chat",+ AsyncMock(side_effect=RuntimeError("database down")),+ )++ response = client.delete("/schedules/sched-1")++ assert response.status_code == 200+ assert response.json() == {"id": "sched-1"}+++def test_paused_scheduled_chats_are_listed_for_the_caller(+ mocker: pytest_mock.MockFixture, test_user_id: str+) -> None:+ paused = PausedScheduledChat(+ schedule_key="routine-1",+ title="Discord monitor",+ session_id="session-1",+ reason="chat_link",+ provider="discord",+ summary="Paused: can't post to Discord",+ paused_at=datetime(2026, 10, 11, tzinfo=timezone.utc),+ )+ listing = mocker.patch(+ "backend.api.features.schedules.routes.scheduled_chats"+ ".list_paused_scheduled_chats",+ AsyncMock(return_value=[paused]),+ )++ response = client.get("/schedules/followups/paused")++ assert response.status_code == 200+ assert response.json()[0]["schedule_key"] == "routine-1"+ assert response.json()[0]["summary"] == "Paused: can't post to Discord"+ listing.assert_awaited_once_with(test_user_id)+++def test_resume_scheduled_chat_resumes_the_callers_chat(+ mocker: pytest_mock.MockFixture, test_user_id: str+) -> None:+ resume = mocker.patch(+ "backend.api.features.schedules.routes.scheduled_chats.resume_scheduled_chat",+ AsyncMock(return_value=True),+ )++ response = client.post("/schedules/followups/routine-1/resume")++ assert response.status_code == 200+ assert response.json() == {"resumed": True}+ resume.assert_awaited_once_with(test_user_id, "routine-1")+++def test_resume_scheduled_chat_maps_nothing_paused_to_404(+ mocker: pytest_mock.MockFixture,+) -> None:+ """A second click on an old card, or a chat resumed some other way."""+ mocker.patch(+ "backend.api.features.schedules.routes.scheduled_chats.resume_scheduled_chat",+ AsyncMock(return_value=False),+ )++ response = client.post("/schedules/followups/routine-1/resume")++ assert response.status_code == 404+++def test_a_resume_the_scheduler_refuses_can_be_tried_again(+ mocker: pytest_mock.MockFixture,+) -> None:+ mocker.patch(+ "backend.api.features.schedules.routes.scheduled_chats.resume_scheduled_chat",+ AsyncMock(side_effect=RuntimeError("scheduler down")),+ )++ response = client.post("/schedules/followups/routine-1/resume")++ assert response.status_code == 503++def test_delete_schedule_maps_not_found_to_404(mocker: pytest_mock.MockFixture,) -> None:_patch_scheduler(mocker, delete_schedule=AsyncMock(side_effect=NotFoundError("gone")))+ mocker.patch(+ "backend.api.features.schedules.routes.scheduled_chats.forget_scheduled_chat",+ AsyncMock(),+ )response = client.delete("/schedules/missing")diff --git a/autogpt_platform/backend/backend/api/paywall_routes_test.py b/autogpt_platform/backend/backend/api/paywall_routes_test.pyindex 55f8b262aa..a60f3d277e 100644--- a/autogpt_platform/backend/backend/api/paywall_routes_test.py+++ b/autogpt_platform/backend/backend/api/paywall_routes_test.py@@ -219,6 +219,7 @@ OPEN_WHILE_PAYWALLED = {"GET /api/review/session/{chat_session_id}","GET /api/schedules","GET /api/schedules/followups",+ "GET /api/schedules/followups/paused","GET /api/skill-learning/decisions","GET /api/skill-learning/history","GET /api/skill-learning/skills/{name}",diff --git a/autogpt_platform/backend/backend/copilot/executor/processor.py b/autogpt_platform/backend/backend/copilot/executor/processor.pyindex 19080682c3..c464c227f6 100644--- a/autogpt_platform/backend/backend/copilot/executor/processor.py+++ b/autogpt_platform/backend/backend/copilot/executor/processor.py@@ -269,13 +269,18 @@ async def _wait_for_expert_setup(session: "ChatSession") -> None:async def _refuse_turn(- entry: CoPilotExecutionEntry, session: "ChatSession", pause: TurnPause+ entry: CoPilotExecutionEntry,+ session: "ChatSession",+ pause: TurnPause,+ scheduled_watch: ScheduledTurnWatch | None,) -> None:"""End a turn the allowance can't cover without running it.The chat keeps the turn's message, as a turn that ran would, followed bya not-started marker; the live stream carries the same pause part the- engines end a landed turn with.+ engines end a landed turn with. A scheduled turn's watch sees it too, so+ its streak counts the run as not run rather than as one that tried+ nothing."""maybe_append_user_message(session, entry.message or None, entry.is_user_message, entry.message_metadata@@ -288,6 +293,8 @@ async def _refuse_turn(await stream_registry.publish_chunk(entry.turn_id, chunk, session_id=entry.session_id)+ if scheduled_watch is not None:+ scheduled_watch.observe(chunk)def execute_copilot_turn(@@ -686,7 +693,7 @@ class CoPilotProcessor:returnif pause is not None and credit_turn is None:log.info(f"Turn not started: the {pause.reason} limit is used up")- await _refuse_turn(entry, session, pause)+ await _refuse_turn(entry, session, pause, scheduled_watch)returnif credit_turn is not None:log.info(@@ -957,9 +964,18 @@ class CoPilotProcessor:except Exception as mark_err:log.error(f"Failed to mark session completed: {mark_err}")finally:- await cost_context_stack.aclose()- # A browser turn leaves a Chromium tree on this pod that- # nothing else ever stops; state is already persisted, so- # the next turn restores it wherever it lands.- if await close_browser_daemon(entry.session_id):- log.info("Browser daemon teardown attempted for this turn")+ try:+ # After the turn is marked done, so the owner never+ # waits on it: counts the run toward its schedule's+ # failure streak. It can be cancelled while it waits,+ # so the cleanups below still run.+ if scheduled_watch is not None:+ await scheduled_watch.record(error_msg)+ finally:+ await cost_context_stack.aclose()+ # A browser turn leaves a Chromium tree on this pod+ # that nothing else ever stops; state is already+ # persisted, so the next turn restores it wherever it+ # lands.+ if await close_browser_daemon(entry.session_id):+ log.info("Browser daemon teardown attempted for this turn")diff --git a/autogpt_platform/backend/backend/copilot/executor/scheduled_turn_alert.py b/autogpt_platform/backend/backend/copilot/executor/scheduled_turn_alert.pyindex 1c5040f014..9957f58ed3 100644--- a/autogpt_platform/backend/backend/copilot/executor/scheduled_turn_alert.py+++ b/autogpt_platform/backend/backend/copilot/executor/scheduled_turn_alert.py@@ -4,9 +4,13 @@ The scheduler only learns whether it got a turn onto the queue. What happensnext, such as the turn erroring or a tool it called returning an error(``run_agent`` in SECRT-2799, every Monday for ten weeks), is only visiblehere in the copilot executor, and nobody is watching an unattended turn. So a-failed scheduled turn raises one Sentry event and a metric.+failed scheduled turn raises one Sentry event and a metric, and every+scheduled turn's outcome counts toward its schedule's failure streak, which+pauses a chat that keeps failing the same way and tells its owner+(SECRT-2879)."""+import asyncioimport loggingimport re@@ -15,24 +19,49 @@ import sentry_sdkfrom pydantic import BaseModelfrom backend.copilot import stream_registry+from backend.copilot.constants import COMPACTION_TOOL_NAMEfrom backend.copilot.permissions import SDK_BUILTIN_TOOL_NAMESfrom backend.copilot.response_model import (StreamBaseResponse,StreamError,StreamToolOutputAvailable,+ StreamTurnPaused,)+from backend.copilot.scheduled_chats.notices import schedule_title+from backend.copilot.scheduled_chats.outcomes import (+ PAUSE_ERROR_CODES,+ RunOutcome,+ ToolFailureDetail,+ classify_run,+ text_hash,+ tool_failure_detail,+)+from backend.copilot.scheduled_chats.service import ScheduledRunfrom backend.copilot.tools.models import ResponseTypefrom backend.monitoring.instrumentation import COPILOT_SCHEDULED_TURN_FAILURES+from backend.util.clients import get_database_manager_async_clientfrom .utils import CoPilotExecutionEntry, ScheduledTurnOriginlogger = logging.getLogger(__name__)+# How long a finished turn may spend recording its outcome. The turn is+# already marked done; this only holds the worker.+_RECORD_TIMEOUT_SECONDS = 20+# A tool's error text can quote the user's data, and neither the log sinks nor# Sentry are redacted, so a failure is reported by its error code alone._ERROR_CODE = re.compile(r"[a-z][a-z0-9_]{0,63}")UNCLASSIFIED = "unclassified"+# Finding a block or an agent and reading its schema is how a run gets ready+# to work, not work done: a run whose one real step failed after them failed+# outright, like a weekly report that finds its agent and then fails to run it+# (SECRT-2799).+_LOOKUP_TOOLS = frozenset(+ {"find_capability", "describe_capability", "find_agent", "find_library_agent"}+)+class ToolFailure(BaseModel):tool: str@@ -48,6 +77,12 @@ class ScheduledTurnWatch:self._origin = originself.tool_failures: list[ToolFailure] = []self._turn_error_code: str | None = None+ # For the failure streak: what each failure said, and how many+ # platform tool calls worked.+ self._failure_details: list[tuple[str, ToolFailureDetail]] = []+ self._tool_successes = 0+ # Set when the turn landed at a limit, or was refused one at pickup.+ self._pause: dict | None = None@classmethoddef for_entry(cls, entry: CoPilotExecutionEntry) -> "ScheduledTurnWatch | None":@@ -59,32 +94,99 @@ class ScheduledTurnWatch:if isinstance(chunk, StreamError):self._turn_error_code = chunk.codereturn+ if isinstance(chunk, StreamTurnPaused):+ self._pause = chunk.pause+ returnif not isinstance(chunk, StreamToolOutputAvailable):returnerror_type = tool_error_type(chunk.output)# A platform tool can also fail without an ErrorResponse payload (a# login prompt, a held read, a plain-text error). The CLI's own tools# stay out: a missed Read or Grep is the model probing, not a failure.- if (- error_type is None- and not chunk.success- and chunk.toolName not in SDK_BUILTIN_TOOL_NAMES- ):+ if error_type is None and not chunk.success and _ours(chunk.toolName):error_type = UNCLASSIFIED+ tool = chunk.toolName or "unknown"if error_type is not None:- self.tool_failures.append(- ToolFailure(tool=chunk.toolName or "unknown", error_type=error_type)+ self.tool_failures.append(ToolFailure(tool=tool, error_type=error_type))+ detail = tool_failure_detail(chunk.output) or ToolFailureDetail(+ error_type=UNCLASSIFIED, text_hash=text_hash(chunk.output)+ )+ self._failure_details.append((tool, detail))+ elif _ours(chunk.toolName) and chunk.toolName not in _LOOKUP_TOOLS:+ self._tool_successes += 1++ def outcome(self, turn_error: str | None) -> RunOutcome:+ landed = self._pause is not None+ return classify_run(+ cancelled=turn_error == stream_registry.CANCELLED_MESSAGE,+ turn_error_code=self._landed_error_code() or self._turn_error_code,+ # A pause decides the run as a turn error does: its tools were+ # refused while it wrote a checkpoint, so they are no failures.+ turn_errored=turn_error is not None or landed,+ tool_failures=[] if landed else self._failure_details,+ tool_successes=self._tool_successes,+ )++ def _landed_error_code(self) -> str | None:+ """The error a pause at the run's own budget or step limit counts as;+ None for the account's windows, and for a run refused at pickup, which+ never ran."""+ if self._pause is None or self._pause.get("started") is False:+ return None+ reason = self._pause.get("reason")+ return PAUSE_ERROR_CODES.get(reason) if isinstance(reason, str) else None++ async def record(self, turn_error: str | None) -> None:+ """Count this run toward its schedule's failure streak.++ Never raises, and bounded: it runs in the turn's cleanup, after the+ turn has been marked done.+ """+ if self._origin.purpose != "followup":+ # A chat's own turn, resumed or sent at its budget reset: the+ # chat is the user's, not a schedule that can be paused.+ return+ outcome = self.outcome(turn_error)+ schedule_key = self._origin.routine_id or self._origin.schedule_id+ if outcome.neutral or schedule_key is None or self._entry.user_id is None:+ return+ run = ScheduledRun(+ user_id=self._entry.user_id,+ schedule_key=schedule_key,+ schedule_id=self._origin.schedule_id,+ routine_id=self._origin.routine_id,+ session_id=self._entry.session_id,+ title=schedule_title(routine_title=None, message=self._entry.message),+ outcome=outcome,+ )+ try:+ await asyncio.wait_for(+ get_database_manager_async_client().record_scheduled_run(run),+ timeout=_RECORD_TIMEOUT_SECONDS,+ )+ except Exception:+ logger.warning(+ "Could not record scheduled copilot turn %s toward its schedule's "+ "failure streak",+ self._entry.turn_id,+ exc_info=True,)def report(self, turn_error: str | None) -> None:"""Never raises: it runs in the turn's cleanup."""if turn_error == stream_registry.CANCELLED_MESSAGE:turn_error = None- if turn_error is None and not self.tool_failures:+ failures = self.tool_failures+ if self._pause is not None:+ # The tools a landing refused are no failures; the run's own+ # limits report as the errors they replaced.+ failures = []+ turn_error = turn_error or self._landed_error_code()+ if turn_error is None and not failures:returntry:- self._count(turn_error)- self._alert(turn_error)+ self._count(turn_error, failures)+ self._alert(turn_error, failures)except Exception:logger.warning("Could not report failed scheduled copilot turn %s",@@ -92,20 +194,20 @@ class ScheduledTurnWatch:exc_info=True,)- def _count(self, turn_error: str | None) -> None:+ def _count(self, turn_error: str | None, failures: list[ToolFailure]) -> None:if turn_error is not None:COPILOT_SCHEDULED_TURN_FAILURES.labels(reason="turn_error", tool="").inc()- for failure in self.tool_failures:+ for failure in failures:COPILOT_SCHEDULED_TURN_FAILURES.labels(reason="tool_error", tool=failure.tool).inc()- def _alert(self, turn_error: str | None) -> None:+ def _alert(self, turn_error: str | None, failures: list[ToolFailure]) -> None:if turn_error is not None:reason, summary = "turn_error", "the turn errored"else:reason = "tool_error"- summary = f"{self.tool_failures[0].tool} returned an error"+ summary = f"{failures[0].tool} returned an error"details = {"schedule_id": self._origin.schedule_id,"routine_id": self._origin.routine_id,@@ -115,11 +217,9 @@ class ScheduledTurnWatch:"turn_error_type": (error_type(self._turn_error_code or turn_error) if turn_error else None),- "tool_failures": [f.model_dump() for f in self.tool_failures],+ "tool_failures": [f.model_dump() for f in failures],}- failed_tools = ", ".join(- f"{f.tool} ({f.error_type})" for f in self.tool_failures- )+ failed_tools = ", ".join(f"{f.tool} ({f.error_type})" for f in failures)logger.warning(f"Scheduled copilot turn failed: {summary} "f"(schedule {self._origin.schedule_id}, session "@@ -157,3 +257,15 @@ def error_type(error: object) -> str:if isinstance(error, str) and _ERROR_CODE.fullmatch(error):return errorreturn UNCLASSIFIED+++def _ours(tool_name: str | None) -> bool:+ """A platform tool call. The CLI's own tools stay out (a missed Read or+ Grep is the model probing), and so does the compaction notice, which no+ tool returned: counting it as work done would keep a run that failed+ everywhere else from ever reading as failed."""+ return (+ tool_name is not None+ and tool_name not in SDK_BUILTIN_TOOL_NAMES+ and tool_name != COMPACTION_TOOL_NAME+ )diff --git a/autogpt_platform/backend/backend/copilot/executor/scheduled_turn_alert_test.py b/autogpt_platform/backend/backend/copilot/executor/scheduled_turn_alert_test.pyindex f403f43ce2..a04fa94b4b 100644--- a/autogpt_platform/backend/backend/copilot/executor/scheduled_turn_alert_test.py+++ b/autogpt_platform/backend/backend/copilot/executor/scheduled_turn_alert_test.py@@ -1,11 +1,14 @@+import asyncioimport loggingimport threading+from collections.abc import Iteratorfrom types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom backend.copilot import stream_registry, usage_activation_fixtures+from backend.copilot.constants import COMPACTION_TOOL_NAMEfrom backend.copilot.executor.processor import CoPilotProcessorfrom backend.copilot.executor.scheduled_turn_alert import (UNCLASSIFIED,@@ -22,8 +25,12 @@ from backend.copilot.response_model import (StreamError,StreamTextDelta,StreamToolOutputAvailable,+ StreamTurnPaused,)+from backend.copilot.scheduled_chats.outcomes import RunOutcome+from backend.copilot.scheduled_chats.streaks import advancefrom backend.copilot.tools.models import ErrorResponse, ExecutionStartedResponse+from backend.copilot.turn_pause import PauseReason, TurnPausefrom backend.integrations.codex.transport import CodexCredentialIntegrityErrorfrom backend.monitoring.instrumentation import COPILOT_SCHEDULED_TURN_FAILURES@@ -100,10 +107,223 @@ def _failures(reason: str, tool: str) -> float:return COPILOT_SCHEDULED_TURN_FAILURES.labels(reason=reason, tool=tool)._value.get()[email protected](autouse=True)+def database_manager() -> Iterator[MagicMock]:+ """Where a scheduled turn records its outcome. Patched for every test so+ none waits on a DatabaseManager that is not running."""+ client = MagicMock(record_scheduled_run=AsyncMock())+ with patch(f"{_ALERT}.get_database_manager_async_client", return_value=client):+ yield client+++def _twitter_refused() -> StreamToolOutputAvailable:+ return StreamToolOutputAvailable(+ toolCallId="call-2",+ toolName="run_block",+ output=ErrorResponse(+ message="Twitter rejected the saved credential (HTTP 401).",+ error="credential_rejected",+ details={"provider": "twitter"},+ session_id="sess-weekly-report",+ ).model_dump_json(),+ )+++def test_the_watch_sees_a_refused_connection_as_one_failing_step():+ """The briefing case: one step refused, the rest worked."""+ watch = ScheduledTurnWatch.for_entry(_entry(_weekly()))+ assert watch is not None+ watch.observe(_run_agent_success())+ watch.observe(_twitter_refused())++ outcome = watch.outcome(None)++ assert outcome.failure is not None+ assert outcome.failure.key == "connection:twitter"+ assert not outcome.whole_run+++def test_a_compacted_history_is_not_work_done():+ """The every-minute monitor has a long history, so its runs compact. A+ run whose only other output is the failure still failed outright."""+ watch = ScheduledTurnWatch.for_entry(_entry(_weekly()))+ assert watch is not None+ watch.observe(+ StreamToolOutputAvailable(+ toolCallId="compaction", toolName=COMPACTION_TOOL_NAME, output="done"+ )+ )+ watch.observe(_twitter_refused())++ outcome = watch.outcome(None)++ assert outcome.failure is not None and outcome.whole_run+++def _looked_up(tool: str) -> StreamToolOutputAvailable:+ return StreamToolOutputAvailable(+ toolCallId=f"call-{tool}", toolName=tool, output='{"type": "capabilities"}'+ )+++def _discord_not_connected() -> StreamToolOutputAvailable:+ return StreamToolOutputAvailable(+ toolCallId="call-run",+ toolName="run_capability",+ output=ErrorResponse(+ message="This chat has no Discord credential connected.",+ error="missing_credentials",+ details={"provider": "discord"},+ session_id="sess-weekly-report",+ ).model_dump_json(),+ )+++def _discord_monitor_run(*, sent: bool = False) -> RunOutcome:+ """The every-minute Discord monitor: the model finds the block and reads+ its schema before it calls it, then the call fails or works."""+ watch = ScheduledTurnWatch.for_entry(_entry(_weekly()))+ assert watch is not None+ watch.observe(_looked_up("find_capability"))+ watch.observe(_looked_up("describe_capability"))+ watch.observe(_run_agent_success() if sent else _discord_not_connected())+ return watch.outcome(None)+++def test_looking_up_the_step_that_failed_is_not_work_done():+ """Finding the Discord block and reading its schema did none of the+ work, so a run whose only real step failed failed outright."""+ outcome = _discord_monitor_run()++ assert outcome.failure is not None+ assert outcome.failure.key == "connection:discord"+ assert outcome.whole_run+++def test_finding_the_agent_that_then_failed_is_not_work_done():+ """The weekly report finds its agent, then run_agent fails (SECRT-2799):+ the run did none of its work, so it failed outright."""+ watch = ScheduledTurnWatch.for_entry(_entry(_weekly()))+ assert watch is not None+ watch.observe(_looked_up("find_library_agent"))+ watch.observe(_run_agent_failure())++ outcome = watch.outcome(None)++ assert outcome.failure is not None and outcome.whole_run+++def test_the_discord_monitor_pauses_on_its_third_run():+ streak = None+ steps = []+ for _ in range(3):+ step = advance(streak, _discord_monitor_run())+ streak = step.streak+ steps.append(step)++ assert [s.pause for s in steps] == [False, False, True]+++def test_a_monitor_run_that_sends_clears_its_streak():+ step = advance(None, _discord_monitor_run())+ step = advance(step.streak, _discord_monitor_run())++ assert advance(step.streak, _discord_monitor_run(sent=True)).streak is None+++def test_a_run_that_only_looked_things_up_tried_nothing():+ watch = ScheduledTurnWatch.for_entry(_entry(_weekly()))+ assert watch is not None+ watch.observe(_looked_up("find_capability"))+ watch.observe(_looked_up("describe_capability"))++ assert watch.outcome(None).idle+++def test_the_watch_counts_only_platform_tools_as_trying():+ """A run that only read files with the CLI's own tools tried nothing."""+ watch = ScheduledTurnWatch.for_entry(_entry(_weekly()))+ assert watch is not None+ watch.observe(+ StreamToolOutputAvailable(toolCallId="c", toolName="Read", output="contents")+ )++ assert watch.outcome(None).idle++def test_interactive_turns_are_not_watched():assert ScheduledTurnWatch.for_entry(_entry(None)) is None+def _paused(reason: PauseReason, *, started: bool = True) -> StreamTurnPaused:+ return StreamTurnPaused(+ pause=TurnPause(+ reason=reason, checkpoint_saved=started, started=started+ ).as_part()+ )+++def _refused_while_landing() -> StreamToolOutputAvailable:+ """A tool call refused once the run had to land: the refusal reaches the+ model as a failed tool result."""+ return StreamToolOutputAvailable(+ toolCallId="call-refused",+ toolName="run_agent",+ output="Tool calls are paused: this task has used the budget it has for now.",+ success=False,+ )+++def _landed_run(reason: PauseReason) -> ScheduledTurnWatch:+ watch = ScheduledTurnWatch.for_entry(_entry(_weekly()))+ assert watch is not None+ watch.observe(_refused_while_landing())+ watch.observe(_paused(reason))+ return watch+++ ("reason", "key"),+ [("task", "turn:max_budget_exhausted"), ("steps", "turn:max_turns_exhausted")],+)+def test_a_run_that_lands_at_its_own_limit_ran_out_as_before(reason, key):+ """Its own budget and step limit pause a run now, where they used to end+ it with an error. Three in a row still pause the schedule."""+ outcome = _landed_run(reason).outcome(None)++ assert outcome.failure is not None+ assert outcome.failure.key == key and outcome.failure.kind == "budget"+ assert outcome.whole_run++[email protected]("reason", ["daily", "weekly", "trial"])+def test_a_run_paused_by_the_accounts_allowance_counts_for_nothing(reason):+ """The owner's allowance ran out mid-run: their state, not the+ schedule's, and the runs after it say so as skips. The tool calls the+ landing refused are no failure either."""+ watch = _landed_run(reason)++ assert watch.outcome(None).neutral+ with patch(f"{_ALERT}.sentry_sdk") as sentry:+ watch.report(None)+ sentry.capture_message.assert_not_called()+++def test_a_run_at_its_own_limit_alerts_as_that_limit_not_as_a_tool():+ watch = _landed_run("task")++ with patch(f"{_ALERT}.sentry_sdk") as sentry:+ watch.report(None)++ sentry.capture_message.assert_called_once_with(+ "Scheduled copilot turn failed: the turn errored", level="error"+ )+ scope = sentry.new_scope.return_value.__enter__.return_value+ _, context = scope.set_context.call_args.args+ assert context["turn_error_type"] == "max_budget_exhausted"+ assert context["tool_failures"] == []++def test_tool_error_type_reads_error_payloads_only():assert tool_error_type(_run_agent_failure().output) == UNCLASSIFIEDassert (@@ -266,7 +486,9 @@ class _Stream:pass-async def _run_turn(entry: CoPilotExecutionEntry, events) -> MagicMock:+async def _run_turn(+ entry: CoPilotExecutionEntry, events, mark_done: AsyncMock | None = None+) -> MagicMock:with (patch("backend.copilot.executor.processor.ChatConfig",@@ -282,7 +504,7 @@ async def _run_turn(entry: CoPilotExecutionEntry, events) -> MagicMock:),patch("backend.copilot.executor.processor.stream_registry.mark_session_completed",- new=AsyncMock(),+ new=mark_done or AsyncMock(),),patch("backend.copilot.model.get_chat_session",@@ -330,6 +552,107 @@ async def test_executor_does_not_alert_for_an_interactive_turn():sentry.capture_message.assert_not_called()+async def test_executor_records_a_scheduled_turns_outcome(database_manager):+ """Every scheduled run counts toward its schedule's failure streak; the+ error's text never leaves the executor, only a hash of it."""+ await _run_turn(_entry(_weekly()), [_run_agent_failure()])++ database_manager.record_scheduled_run.assert_awaited_once()+ run = database_manager.record_scheduled_run.await_args.args[0]+ assert run.user_id == "user-1"+ assert run.schedule_key == "sched-weekly"+ assert run.session_id == "sess-weekly-report"+ assert run.title == "Run the weekly report agent"+ assert run.outcome.whole_run+ assert run.outcome.failure.key.startswith("tool:run_agent:")+ assert "query engine" not in run.model_dump_json()+++async def test_executor_records_after_the_turn_is_marked_done(database_manager):+ """The owner never waits on the streak: the turn is done first."""+ order = MagicMock()+ database_manager.record_scheduled_run.side_effect = lambda *_: order.record()+ mark_done = AsyncMock(side_effect=lambda *_, **__: order.mark_done())++ await _run_turn(_entry(_weekly()), [_run_agent_failure()], mark_done=mark_done)++ assert [c[0] for c in order.mock_calls] == ["mark_done", "record"]+++async def test_executor_records_nothing_for_an_interactive_turn(database_manager):+ await _run_turn(_entry(None), [_run_agent_failure()])++ database_manager.record_scheduled_run.assert_not_awaited()+++async def test_a_broken_record_never_breaks_the_turn_cleanup(database_manager, caplog):+ database_manager.record_scheduled_run.side_effect = RuntimeError("db manager down")++ with caplog.at_level(logging.WARNING, logger=_ALERT):+ await _run_turn(_entry(_weekly()), [_run_agent_success()])++ assert "toward its schedule's failure streak" in caplog.text++[email protected]("purpose", ["resume", "send"])+async def test_a_chats_own_turn_at_its_reset_alerts_but_has_no_streak(+ database_manager, purpose+):+ """A turn resumed or sent at the budget reset is the user's own chat, not+ a schedule: it can fail unwatched, so it alerts, but there is nothing to+ pause and no streak to count toward."""+ entry = _entry(+ ScheduledTurnOrigin(schedule_id=f"at-reset-{purpose}-1", purpose=purpose)+ )++ sentry = await _run_turn(entry, [_run_agent_failure()])++ sentry.capture_message.assert_called_once()+ database_manager.record_scheduled_run.assert_not_awaited()+++async def test_a_cancelled_record_still_closes_the_turns_browser(database_manager):+ """Recording waits on the DatabaseManager and can be cancelled meanwhile,+ as a shutdown cancels the turn. The cleanups after it still run, or the+ turn's Chromium tree outlives it on the pod."""+ database_manager.record_scheduled_run.side_effect = asyncio.CancelledError()++ with patch(+ "backend.copilot.executor.processor.close_browser_daemon",+ new=AsyncMock(return_value=False),+ ) as close_browser:+ with pytest.raises(asyncio.CancelledError):+ await _run_turn(_entry(_weekly()), [_run_agent_success()])++ close_browser.assert_awaited_once()+++async def test_a_scheduled_turn_refused_at_pickup_counts_for_nothing(database_manager):+ """The allowance ran out between the schedule firing and the turn being+ picked up, so the turn never ran. Read as a run that tried nothing, it+ would continue an open connection failure and could finish pausing a+ monitor for a cause it never reached."""+ not_started = TurnPause(reason="daily", checkpoint_saved=False, started=False)++ with (+ patch(+ "backend.copilot.executor.processor.refusal_for",+ AsyncMock(return_value=not_started),+ ),+ patch("backend.copilot.budget_gate.upsert_chat_session", AsyncMock()),+ ):+ await _run_turn(_entry(_weekly()), [])++ database_manager.record_scheduled_run.assert_not_awaited()++@pytest.mark.asyncioasync def test_executor_alerts_when_the_codex_checkpoint_fails_after_a_scheduled_turn():session = ChatSession.new(diff --git a/autogpt_platform/backend/backend/copilot/executor/utils.py b/autogpt_platform/backend/backend/copilot/executor/utils.pyindex 59fe8f96a4..43b69ce576 100644--- a/autogpt_platform/backend/backend/copilot/executor/utils.py+++ b/autogpt_platform/backend/backend/copilot/executor/utils.py@@ -9,7 +9,7 @@import loggingimport timefrom collections.abc import Mapping-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, Literalfrom uuid import uuid4from pydantic import BaseModel, Field@@ -273,6 +273,10 @@ class ScheduledTurnOrigin(BaseModel):schedule_id: str | None = Noneroutine_id: str | None = Nonecron: str | None = None+ # A scheduled chat its owner set up, or a chat's own turn resumed or+ # sent at its budget reset. Both alert; only a scheduled chat has a+ # failure streak to count toward.+ purpose: Literal["followup", "resume", "send"] = "followup"class CoPilotExecutionEntry(BaseModel):diff --git a/autogpt_platform/backend/backend/copilot/scheduled_chats/failure_notices.py b/autogpt_platform/backend/backend/copilot/scheduled_chats/failure_notices.pynew file mode 100644index 0000000000..cc44805d4b--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/scheduled_chats/failure_notices.py@@ -0,0 +1,195 @@+"""What a failing scheduled chat tells its owner (SECRT-2879).++Either it was paused, after three runs in a row failed outright the same way,+or it is still running but the same step has failed three times in a row.+Either way the note says what broke and offers the fix: reconnect the+provider, link the chat platform, resume. One that could not be paused says+the same without that claim. Like every note here, it is for the owner and+hidden from the model.+"""++from backend.copilot.scheduled_chats.notices import (+ BILLING_PATH,+ INTEGRATIONS_PATH,+ NoticeAction,+ NoticeLink,+ NoticeReconnect,+ NoticeResume,+ ScheduledChatNotice,+)+from backend.copilot.scheduled_chats.outcomes import RunFailure++BOTS_PATH = "/settings/bots"++# Where ``str.title()`` gets a provider's name wrong.+_PROVIDER_NAMES = {+ "github": "GitHub",+ "gitlab": "GitLab",+ "hubspot": "HubSpot",+ "linkedin": "LinkedIn",+ "openai": "OpenAI",+ "youtube": "YouTube",+}+++def provider_name(provider: str | None) -> str:+ if not provider:+ return "a connected service"+ return _PROVIDER_NAMES.get(provider, provider.replace("_", " ").title())+++def paused_notice(+ failure: RunFailure,+ *,+ title: str,+ failures: int,+ streak_id: str,+ paused: bool = True,+) -> ScheduledChatNotice:+ """Three runs in a row failed outright the same way. *paused* is False+ when the schedule could not be paused, so it is still firing."""+ name = provider_name(failure.provider)+ runs = f"Its last {failures} runs"+ if failure.kind == "connection" and failure.provider:+ headline = f"“{title}” can't use {name}"+ cause = (+ f"{runs} failed because the {name} connection is missing or was " "refused."+ )+ fix = "Reconnect it and the chat starts again." if paused else "Reconnect it."+ elif failure.kind == "connection":+ # Its tool didn't say which account: the owner finds it in Settings,+ # and reconnecting there resumes nothing by itself.+ headline = f"“{title}” can't use one of its connections"+ cause = (+ f"{runs} failed because a connection it needs is missing or was refused."+ )+ fix = (+ "Reconnect it in Settings, then resume."+ if paused+ else "Reconnect it in Settings."+ )+ elif failure.kind == "chat_link":+ headline = f"“{title}” can't post to {_platform(failure)}"+ cause = (+ f"{runs} couldn't post, because {_no_link(failure)} is linked to "+ "your account."+ )+ fix = "Link one, then resume." if paused else "Link one."+ elif failure.kind == "budget":+ limit = "step" if failure.key == "turn:max_turns_exhausted" else "spending"+ headline = f"“{title}” keeps hitting its {limit} limit"+ cause = (+ f"{runs} stopped at the {limit} limit one run may use, before finishing."+ )+ fix = (+ "Resume it to try again, or give it a smaller job."+ if paused+ else "Give it a smaller job."+ )+ else:+ headline = f"“{title}” keeps failing"+ cause = f"{runs} failed the same way."+ fix = (+ "Open the chat to see the error, then resume."+ if paused+ else "Open the chat to see the error."+ )+ if not paused:+ return ScheduledChatNotice(+ reason=failure.kind,+ headline=headline,+ body=f"{cause} {fix}",+ actions=_fixes(failure),+ window_key=f"failing:{streak_id}",+ )+ return ScheduledChatNotice(+ reason=failure.kind,+ headline=f"Paused: {headline}",+ body=f"{cause} {fix}",+ actions=[*_fixes(failure), NoticeResume()],+ window_key=f"paused:{streak_id}",+ )+++def failing_notice(+ failure: RunFailure, *, title: str, failures: int, streak_id: str+) -> ScheduledChatNotice:+ """The schedule keeps running: the rest of what it does still works."""+ name = provider_name(failure.provider)+ runs = f"The last {failures} runs"+ if failure.kind == "connection" and failure.provider:+ headline = f"“{title}” can't use {name}"+ body = (+ f"{runs} went ahead without {name}, because the connection is "+ "missing or was refused. Reconnect it to bring it back."+ )+ elif failure.kind == "connection":+ headline = f"“{title}” can't use one of its connections"+ body = (+ f"{runs} went ahead without a connection they need, which is "+ "missing or was refused. Reconnect it in Settings to bring it back."+ )+ elif failure.kind == "chat_link":+ headline = f"“{title}” can't post to {_platform(failure)}"+ body = (+ f"{runs} went ahead without posting, because {_no_link(failure)} "+ "is linked to your account."+ )+ else:+ headline = f"“{title}” keeps failing at one step"+ body = f"{runs} went ahead without it. Open the chat to see the error."+ return ScheduledChatNotice(+ reason=failure.kind,+ headline=headline,+ body=body,+ actions=_fixes(failure),+ window_key=f"failing:{streak_id}",+ )+++def pause_summary(failure: RunFailure) -> str:+ """Why a scheduled chat is paused, in a few words, for the lists."""+ if failure.kind == "connection" and failure.provider:+ return f"Paused: can't use {provider_name(failure.provider)}"+ if failure.kind == "connection":+ return "Paused: can't use one of its connections"+ if failure.kind == "chat_link":+ return f"Paused: can't post to {_platform(failure)}"+ if failure.key == "turn:max_turns_exhausted":+ return "Paused: keeps hitting its step limit"+ if failure.kind == "budget":+ return "Paused: keeps hitting its spending limit"+ return "Paused: keeps failing"+++def resumed_notice(*, title: str, streak_id: str) -> ScheduledChatNotice:+ return ScheduledChatNotice(+ reason="resumed",+ headline=f"Resumed “{title}”",+ body="It runs again at its next scheduled time.",+ actions=[],+ window_key=f"resumed:{streak_id}",+ )+++def _fixes(failure: RunFailure) -> list[NoticeAction]:+ if failure.kind == "connection" and failure.provider:+ label = f"Reconnect {provider_name(failure.provider)}"+ return [NoticeReconnect(label=label, provider=failure.provider)]+ if failure.kind == "connection":+ return [NoticeLink(label="Open settings", href=INTEGRATIONS_PATH)]+ if failure.kind == "chat_link":+ return [NoticeLink(label="Open bot settings", href=BOTS_PATH)]+ if failure.key == "turn:max_budget_exhausted":+ return [NoticeLink(label="Upgrade", href=BILLING_PATH)]+ return []+++def _platform(failure: RunFailure) -> str:+ return provider_name(failure.provider) if failure.provider else "a chat platform"+++def _no_link(failure: RunFailure) -> str:+ if failure.provider:+ return f"no {provider_name(failure.provider)} server or chat"+ return "no server or chat"diff --git a/autogpt_platform/backend/backend/copilot/scheduled_chats/failure_notices_test.py b/autogpt_platform/backend/backend/copilot/scheduled_chats/failure_notices_test.pynew file mode 100644index 0000000000..8e20d8d179--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/scheduled_chats/failure_notices_test.py@@ -0,0 +1,160 @@+from backend.copilot.hidden_rows import is_hidden_from_model+from backend.copilot.scheduled_chats.failure_notices import (+ BOTS_PATH,+ failing_notice,+ pause_summary,+ paused_notice,+ provider_name,+ resumed_notice,+)+from backend.copilot.scheduled_chats.notices import BILLING_PATH, INTEGRATIONS_PATH+from backend.copilot.scheduled_chats.outcomes import RunFailure++_TWITTER = RunFailure(key="connection:twitter", kind="connection", provider="twitter")+_DISCORD_LINK = RunFailure(+ key="chat_link:discord", kind="chat_link", provider="discord"+)+_BUDGET = RunFailure(key="turn:max_budget_exhausted", kind="budget")+_STEPS = RunFailure(key="turn:max_turns_exhausted", kind="budget")+_ERROR = RunFailure(key="tool:run_agent:abc", kind="error", tool="run_agent")+++def _actions(notice):+ return [action.model_dump() for action in notice.actions]+++def test_a_paused_connection_offers_the_reconnect_and_resume():+ notice = paused_notice(_TWITTER, title="Briefing", failures=3, streak_id="s1")++ assert notice.headline == "Paused: “Briefing” can't use Twitter"+ assert notice.body.startswith("Its last 3 runs failed")+ assert _actions(notice) == [+ {"kind": "reconnect", "label": "Reconnect Twitter", "provider": "twitter"},+ {"kind": "resume", "label": "Resume"},+ ]+ assert notice.window_key == "paused:s1"+++def test_a_paused_chat_platform_points_at_the_bot_settings():+ notice = paused_notice(_DISCORD_LINK, title="Monitor", failures=3, streak_id="s1")++ assert notice.headline == "Paused: “Monitor” can't post to Discord"+ assert _actions(notice) == [+ {"kind": "link", "label": "Open bot settings", "href": BOTS_PATH},+ {"kind": "resume", "label": "Resume"},+ ]+++def test_a_chat_over_its_spending_limit_offers_an_upgrade():+ notice = paused_notice(_BUDGET, title="Digest", failures=3, streak_id="s1")++ assert notice.headline == "Paused: “Digest” keeps hitting its spending limit"+ assert _actions(notice)[0] == {+ "kind": "link",+ "label": "Upgrade",+ "href": BILLING_PATH,+ }+++def test_a_chat_out_of_steps_says_step_limit_and_offers_only_resume():+ notice = paused_notice(_STEPS, title="Digest", failures=3, streak_id="s1")++ assert "step limit" in notice.headline+ assert _actions(notice) == [{"kind": "resume", "label": "Resume"}]+++def test_any_other_error_points_at_the_chat():+ notice = paused_notice(_ERROR, title="Weekly", failures=3, streak_id="s1")++ assert notice.headline == "Paused: “Weekly” keeps failing"+ assert "Open the chat to see the error" in notice.body+++def test_one_that_could_not_be_paused_says_so_without_claiming_it():+ notice = paused_notice(+ _TWITTER, title="Briefing", failures=3, streak_id="s1", paused=False+ )++ assert notice.headline == "“Briefing” can't use Twitter"+ assert notice.body == (+ "Its last 3 runs failed because the Twitter connection is missing or "+ "was refused. Reconnect it."+ )+ # Nothing to resume: it is still firing.+ assert _actions(notice) == [+ {"kind": "reconnect", "label": "Reconnect Twitter", "provider": "twitter"}+ ]+ assert notice.window_key == "failing:s1"+++def test_a_failing_step_is_told_without_a_resume():+ """Nothing was paused, so there is nothing to resume."""+ notice = failing_notice(_TWITTER, title="Briefing", failures=3, streak_id="s1")++ assert notice.headline == "“Briefing” can't use Twitter"+ assert "went ahead without Twitter" in notice.body+ assert [a["kind"] for a in _actions(notice)] == ["reconnect"]+ assert notice.window_key == "failing:s1"+++def test_every_note_is_hidden_from_the_model():+ for notice in (+ paused_notice(_TWITTER, title="B", failures=3, streak_id="s"),+ failing_notice(_TWITTER, title="B", failures=3, streak_id="s"),+ resumed_notice(title="B", streak_id="s"),+ ):+ assert is_hidden_from_model(notice.metadata(schedule_key="k", title="B"))+++def test_provider_names_read_as_people_write_them():+ assert provider_name("twitter") == "Twitter"+ assert provider_name("github") == "GitHub"+ assert provider_name("google_drive") == "Google Drive"+ assert provider_name(None) == "a connected service"+++def test_a_connection_its_tool_did_not_name_points_at_settings():+ """A tool that needs several accounts doesn't say which one failed: no+ Reconnect button to offer, and no provider to put in the sentence."""+ unnamed = RunFailure(key="connection:run_block", kind="connection")++ paused = paused_notice(unnamed, title="Briefing", failures=3, streak_id="s1")+ failing = failing_notice(unnamed, title="Briefing", failures=3, streak_id="s1")++ assert paused.headline == "Paused: “Briefing” can't use one of its connections"+ assert paused.body == (+ "Its last 3 runs failed because a connection it needs is missing or was "+ "refused. Reconnect it in Settings, then resume."+ )+ assert _actions(paused) == [+ {"kind": "link", "label": "Open settings", "href": INTEGRATIONS_PATH},+ {"kind": "resume", "label": "Resume"},+ ]+ assert failing.body == (+ "The last 3 runs went ahead without a connection they need, which is "+ "missing or was refused. Reconnect it in Settings to bring it back."+ )+ assert _actions(failing) == [+ {"kind": "link", "label": "Open settings", "href": INTEGRATIONS_PATH}+ ]+ assert pause_summary(unnamed) == "Paused: can't use one of its connections"+++def test_a_chat_platform_its_tool_did_not_name_reads_without_one():+ unnamed = RunFailure(key="chat_link:post", kind="chat_link")++ paused = paused_notice(unnamed, title="Monitor", failures=3, streak_id="s1")+ failing = failing_notice(unnamed, title="Monitor", failures=3, streak_id="s1")++ assert paused.headline == "Paused: “Monitor” can't post to a chat platform"+ assert "because no server or chat is linked" in paused.body+ assert "because no server or chat is linked" in failing.body+ assert pause_summary(unnamed) == "Paused: can't post to a chat platform"+++def test_pause_summary():+ assert pause_summary(_TWITTER) == "Paused: can't use Twitter"+ assert pause_summary(_DISCORD_LINK) == "Paused: can't post to Discord"+ assert pause_summary(_BUDGET) == "Paused: keeps hitting its spending limit"+ assert pause_summary(_STEPS) == "Paused: keeps hitting its step limit"+ assert pause_summary(_ERROR) == "Paused: keeps failing"diff --git a/autogpt_platform/backend/backend/copilot/scheduled_chats/forget.py b/autogpt_platform/backend/backend/copilot/scheduled_chats/forget.pynew file mode 100644index 0000000000..3c0c0b5e77--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/scheduled_chats/forget.py@@ -0,0 +1,16 @@+"""Dropping a routine's failure streak once its owner switches it off.++Apart from ``service`` so the routines layer can call it: ``service`` reaches+the copilot's tools, and the tools reach the routines layer.+"""++from prisma.models import ScheduledChatStreak+++async def forget_routine(user_id: str, routine_id: str) -> None:+ """Its jobs are deleted as it is switched off, and switching it on again+ makes new ones that fire. Kept, a paused one would stay listed as paused,+ with a Resume that says it runs again."""+ await ScheduledChatStreak.prisma().delete_many(+ where={"userId": user_id, "scheduleKey": routine_id}+ )diff --git a/autogpt_platform/backend/backend/copilot/scheduled_chats/jobs.py b/autogpt_platform/backend/backend/copilot/scheduled_chats/jobs.pynew file mode 100644index 0000000000..8b4ed72070--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/scheduled_chats/jobs.py@@ -0,0 +1,103 @@+"""Pausing and resuming a scheduled chat's jobs, all of them or none+(SECRT-2879).++A chat that reads as paused must not fire, and one that reads as running must+not be stuck: a routine with two fire times that paused only one would be+both. So a pause or a resume that fails partway puts back what it changed.+"""++import logging+from typing import Any, Awaitable, Callable++from backend.util.clients import get_scheduler_client+from backend.util.exceptions import NotFoundError++logger = logging.getLogger(__name__)+++async def pause_jobs(user_id: str, schedule_ids: list[str]) -> list[str] | None:+ """Pause every job that is due to fire. Returns the jobs this call paused,+ the only ones a resume may restart: a job its owner paused, or one deleted+ since, is left as it is. None when the scheduler refused partway, after+ resuming what this call had paused, and the job it failed on if that one+ was running: a pause whose answer was lost may still have landed."""+ if not schedule_ids:+ return []+ scheduler = get_scheduler_client()+ try:+ running = {+ job.id+ for job in await scheduler.get_execution_schedules(+ user_id=user_id, kind="copilot_turn"+ )+ }+ except Exception:+ logger.warning("Could not read the scheduled chat jobs", exc_info=True)+ return None+ paused: list[str] = []+ for schedule_id in schedule_ids:+ try:+ # False: already paused by its owner, and not ours to restart.+ if await scheduler.pause_schedule(schedule_id, user_id=user_id):+ paused.append(schedule_id)+ except NotFoundError:+ continue+ except Exception:+ logger.warning(+ "Could not pause scheduled chat job %s", schedule_id, exc_info=True+ )+ # Resuming one whose pause never landed changes nothing; one its+ # owner had paused was not running, and stays as it is.+ uncertain = [schedule_id] if schedule_id in running else []+ await unpause(user_id, paused + uncertain)+ return None+ return paused+++async def unpause(user_id: str, schedule_ids: list[str]) -> None:+ """Resume jobs that ``pause_jobs`` paused, best effort."""+ await _put_back(get_scheduler_client().resume_schedule, user_id, schedule_ids)+++async def resume_jobs(user_id: str, schedule_ids: list[str]) -> None:+ """Resume every job, or raise with the ones this call resumed paused+ again, the one it failed on included: a resume whose answer was lost may+ have landed. A job deleted since the pause has nothing to resume."""+ scheduler = get_scheduler_client()+ resumed: list[str] = []+ for schedule_id in schedule_ids:+ try:+ if await scheduler.resume_schedule(schedule_id, user_id=user_id):+ resumed.append(schedule_id)+ except NotFoundError:+ continue+ except Exception:+ # Pausing one whose resume never landed changes nothing: it is+ # still paused.+ await _put_back(scheduler.pause_schedule, user_id, [*resumed, schedule_id])+ raise+++async def jobs_paused(user_id: str, schedule_ids: list[str]) -> bool:+ """Whether none of these jobs is due to fire. Paused jobs are left out of+ the scheduler's list. Unknown counts as paused."""+ try:+ active = await get_scheduler_client().get_execution_schedules(+ user_id=user_id, kind="copilot_turn"+ )+ except Exception:+ logger.warning("Could not read the scheduled chat jobs", exc_info=True)+ return True+ return not {job.id for job in active} & set(schedule_ids)+++async def _put_back(+ undo: Callable[..., Awaitable[Any]], user_id: str, schedule_ids: list[str]+) -> None:+ for schedule_id in schedule_ids:+ try:+ await undo(schedule_id, user_id=user_id)+ except Exception:+ logger.warning(+ "Could not put scheduled chat job %s back", schedule_id, exc_info=True+ )diff --git a/autogpt_platform/backend/backend/copilot/scheduled_chats/jobs_test.py b/autogpt_platform/backend/backend/copilot/scheduled_chats/jobs_test.pynew file mode 100644index 0000000000..ee06d54e06--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/scheduled_chats/jobs_test.py@@ -0,0 +1,142 @@+from unittest.mock import AsyncMock, MagicMock++import pytest++from backend.copilot.scheduled_chats import jobs+from backend.util.exceptions import NotFoundError++_USER = "user-1"+++def scheduler(mocker) -> MagicMock:+ client = MagicMock(+ pause_schedule=AsyncMock(return_value=True),+ resume_schedule=AsyncMock(return_value=True),+ get_execution_schedules=AsyncMock(return_value=[]),+ )+ mocker.patch.object(jobs, "get_scheduler_client", return_value=client)+ return client+++def _fail_on(failing: str, error: Exception):+ async def act(schedule_id: str, *, user_id: str) -> bool:+ if schedule_id == failing:+ raise error+ return True++ return act+++async def test_every_job_is_paused(scheduler: MagicMock):+ assert await jobs.pause_jobs(_USER, ["a", "b"]) == ["a", "b"]+++async def test_a_job_its_owner_paused_is_not_ours_to_restart(scheduler):+ scheduler.pause_schedule.side_effect = lambda schedule_id, *, user_id: (+ schedule_id != "a"+ )++ assert await jobs.pause_jobs(_USER, ["a", "b"]) == ["b"]+++async def test_a_job_deleted_since_is_skipped_and_the_rest_pause(scheduler):+ scheduler.pause_schedule.side_effect = _fail_on("b", NotFoundError("gone"))++ assert await jobs.pause_jobs(_USER, ["a", "b"]) == ["a"]+ scheduler.resume_schedule.assert_not_awaited()+++async def test_with_no_live_job_there_is_nothing_to_pause(scheduler):+ scheduler.pause_schedule.side_effect = NotFoundError("gone")++ assert await jobs.pause_jobs(_USER, ["a"]) == []+ assert await jobs.pause_jobs(_USER, []) == []+++async def test_a_pause_that_fails_resumes_only_what_it_paused(scheduler):+ async def pause(schedule_id: str, *, user_id: str) -> bool:+ if schedule_id == "c":+ raise RuntimeError("scheduler refused")+ # "b" was already paused by its owner, and stays so.+ return schedule_id != "b"++ scheduler.pause_schedule.side_effect = pause++ assert await jobs.pause_jobs(_USER, ["a", "b", "c"]) is None+ scheduler.resume_schedule.assert_awaited_once_with("a", user_id=_USER)+++async def test_a_pause_whose_answer_was_lost_is_put_back_too(scheduler):+ """The scheduler paused "b" and the answer never came back. Left out of+ the put-back, it would stay paused with no streak saying so, and nothing+ would ever resume it."""+ scheduler.get_execution_schedules.return_value = [+ MagicMock(id="a"),+ MagicMock(id="b"),+ ]+ scheduler.pause_schedule.side_effect = _fail_on("b", TimeoutError("no answer"))++ assert await jobs.pause_jobs(_USER, ["a", "b"]) is None+ resumed = [c.args[0] for c in scheduler.resume_schedule.await_args_list]+ assert resumed == ["a", "b"]+++async def test_a_lost_answer_for_a_job_its_owner_paused_leaves_it_paused(scheduler):+ scheduler.get_execution_schedules.return_value = [MagicMock(id="a")]+ scheduler.pause_schedule.side_effect = _fail_on("b", TimeoutError("no answer"))++ assert await jobs.pause_jobs(_USER, ["a", "b"]) is None+ scheduler.resume_schedule.assert_awaited_once_with("a", user_id=_USER)+++async def test_unknown_jobs_are_not_paused(scheduler):+ scheduler.get_execution_schedules.side_effect = RuntimeError("scheduler down")++ assert await jobs.pause_jobs(_USER, ["a"]) is None+ scheduler.pause_schedule.assert_not_awaited()+++async def test_a_resume_that_fails_pauses_again_what_it_resumed(scheduler):+ """And the job it failed on, whose resume may have landed with its answer+ lost: the chat stays listed as paused, so none of its jobs may fire."""+ scheduler.resume_schedule.side_effect = _fail_on("b", RuntimeError("down"))++ with pytest.raises(RuntimeError):+ await jobs.resume_jobs(_USER, ["a", "b", "c"])++ paused = [call.args[0] for call in scheduler.pause_schedule.await_args_list]+ assert paused == ["a", "b"]+++async def test_a_resume_skips_a_job_deleted_since(scheduler):+ scheduler.resume_schedule.side_effect = _fail_on("a", NotFoundError("gone"))++ await jobs.resume_jobs(_USER, ["a", "b"])++ scheduler.pause_schedule.assert_not_awaited()+++async def test_jobs_are_paused_while_none_of_them_is_listed(scheduler):+ assert await jobs.jobs_paused(_USER, ["a"]) is True++ scheduler.get_execution_schedules.return_value = [MagicMock(id="a")]+ assert await jobs.jobs_paused(_USER, ["a", "b"]) is False+++async def test_an_unreadable_scheduler_counts_as_paused(scheduler):+ scheduler.get_execution_schedules.side_effect = RuntimeError("down")++ assert await jobs.jobs_paused(_USER, ["a"]) is Truediff --git a/autogpt_platform/backend/backend/copilot/scheduled_chats/notices.py b/autogpt_platform/backend/backend/copilot/scheduled_chats/notices.pyindex 6856a55bcc..a6823e7a1e 100644--- a/autogpt_platform/backend/backend/copilot/scheduled_chats/notices.py+++ b/autogpt_platform/backend/backend/copilot/scheduled_chats/notices.py@@ -12,10 +12,10 @@ keeps it out of everything the model reads (``hidden_rows``).import uuidfrom datetime import datetime, timezone-from typing import Any, Literal+from typing import Annotated, Any, Literal, Unionfrom zoneinfo import ZoneInfo, ZoneInfoNotFoundError-from pydantic import BaseModel+from pydantic import BaseModel, Fieldfrom backend.copilot.hidden_rows import HIDDEN_FROM_MODEL_KEYfrom backend.copilot.turn_queue import TurnRefusal@@ -30,7 +30,20 @@ INTEGRATIONS_PATH = "/settings/integrations"_MESSAGE_NAMESPACE = uuid.UUID("5d0a3a4e-2f3b-5c8e-9a51-6e7c0b1d2f43")_TITLE_MAX = 60-NoticeReason = Literal["usage", "no_plan", "codex_access"]+# Skipped before it started: a used-up limit, no plan, a lost ChatGPT plan.+# Failed after it started (``failure_notices``): a connection, an unlinked chat+# platform, the run's own spending or step limit, any other error, and the+# note that a paused chat was resumed.+NoticeReason = Literal[+ "usage",+ "no_plan",+ "codex_access",+ "connection",+ "chat_link",+ "budget",+ "error",+ "resumed",+]# What became of the skipped run: a repeating schedule keeps its next time, a# one-shot either waits for its usage window to reset or is gone.SkippedRun = Literal["recurring", "waits_for_reset", "dropped"]@@ -44,13 +57,34 @@ class NoticeLink(BaseModel):href: str+class NoticeResume(BaseModel):+ """A button that resumes the paused scheduled chat the note is about."""++ kind: Literal["resume"] = "resume"+ label: str = "Resume"+++class NoticeReconnect(BaseModel):+ """A button that opens the connect dialog for one provider, and resumes+ the scheduled chat once the connection is made."""++ kind: Literal["reconnect"] = "reconnect"+ label: str+ provider: str+++NoticeAction = Annotated[+ Union[NoticeLink, NoticeResume, NoticeReconnect], Field(discriminator="kind")+]++class ScheduledChatNotice(BaseModel):reason: NoticeReasonheadline: strbody: str# When the schedule runs again by itself, where it does.resumes_at: datetime | None = None- actions: list[NoticeLink]+ actions: list[NoticeAction]# Names the window this note covers, so a schedule says it once per# window: part of the message id.window_key: strdiff --git a/autogpt_platform/backend/backend/copilot/scheduled_chats/outcomes.py b/autogpt_platform/backend/backend/copilot/scheduled_chats/outcomes.pynew file mode 100644index 0000000000..aeb349d003--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/scheduled_chats/outcomes.py@@ -0,0 +1,178 @@+"""What one scheduled run came to, in the terms a failure streak counts.++A scheduled chat that keeps failing the same way is paused on the third run+and its owner told once (SECRT-2879). "The same way" has to be precise enough+to pick the fix: which connection, the run's spending limit, or one tool's+error. It has to be stable without storing anything a tool's error text says,+because that can quote the user's data, so an uncoded error is compared by a+hash of its text.++Two distinctions decide what the streak does with a failure:++* A failed step is not a failed run. A morning briefing whose Twitter step is+ refused still goes out without Twitter; pausing it would stop the rest too.+ A run fails outright when it ends in an error, or nothing it tried worked.+* A run that calls no tool proves nothing either way. The every-minute monitor+ that could not reach Discord spent its later runs repeating "Discord still+ needs connecting" without trying: those runs must not clear the streak.+"""++import hashlib+import re+from typing import Literal++import orjson+from pydantic import BaseModel++from backend.copilot.tools.models import ResponseType++# An error code such as ``credential_rejected``; anything else is free text.+# The same rule as the scheduled-turn alert's.+_ERROR_CODE = re.compile(r"[a-z][a-z0-9_]{0,63}")+UNCLASSIFIED = "unclassified"+# A word with a digit in it: a count, an id, a time, a duration. Two runs that+# fail the same way rarely agree on one.+_VOLATILE_WORD = re.compile(r"\w*\d\w*")++# A connection the owner fixes on the integrations page.+CONNECTION_ERRORS = frozenset(+ {+ "missing_credentials",+ "credential_rejected",+ "pinned_credential_missing",+ "pinned_credential_unusable",+ }+)+# A chat platform (Discord, Slack, ...) not linked to the account yet.+CHAT_LINK_ERRORS = frozenset({"no_linked_servers", "no_dm_link"})+# Turn errors that are this schedule's own doing: it asks for more than one+# run may spend or take. Every other turn error (an API blip, an idle timeout)+# is the platform's weather, which our own alerts cover, and neither counts+# toward a pause nor clears one.+COUNTED_TURN_ERRORS = frozenset({"max_budget_exhausted", "max_turns_exhausted"})+# A run that reaches either limit lands as a pause (``turn_pause``), and a+# pause at either counts as that limit's error. A pause because the account's+# daily, weekly or trial allowance ran out is the owner's state, not this+# schedule's, and counts as neither.+PAUSE_ERROR_CODES = {"task": "max_budget_exhausted", "steps": "max_turns_exhausted"}++FailureKind = Literal["connection", "chat_link", "budget", "error"]+++class RunFailure(BaseModel):+ # What "failed the same way" compares across runs.+ key: str+ kind: FailureKind+ provider: str | None = None+ # The tool that failed, or None when the run itself errored.+ tool: str | None = None+++class RunOutcome(BaseModel):+ # None when the run was fine.+ failure: RunFailure | None = None+ # The run failed outright, rather than one of its steps.+ whole_run: bool = False+ # Called no platform tool: it neither counts nor clears a failure of its+ # own, but continues a connection failure that is still open.+ idle: bool = False+ # Cancelled, or our outage: neither counts nor clears anything.+ neutral: bool = False+++class ToolFailureDetail(BaseModel):+ error_type: str+ provider: str | None = None+ # For an uncoded error: a short hash of its text, never the text.+ text_hash: str | None = None+++def classify_run(+ *,+ cancelled: bool,+ turn_error_code: str | None,+ turn_errored: bool,+ tool_failures: list[tuple[str, ToolFailureDetail]],+ tool_successes: int,+) -> RunOutcome:+ """*tool_failures* pairs each failing platform tool with what it said;+ *tool_successes* counts the platform tool calls that worked, leaving out+ the lookups that only got a run ready to work."""+ if cancelled:+ return RunOutcome(neutral=True)+ if turn_errored:+ if turn_error_code not in COUNTED_TURN_ERRORS:+ return RunOutcome(neutral=True)+ return RunOutcome(+ failure=RunFailure(key=f"turn:{turn_error_code}", kind="budget"),+ whole_run=True,+ )+ if tool_failures:+ tool, detail = _leading_failure(tool_failures)+ return RunOutcome(+ failure=_failure_of(tool, detail), whole_run=tool_successes == 0+ )+ return RunOutcome(idle=tool_successes == 0)+++def _leading_failure(+ failures: list[tuple[str, ToolFailureDetail]],+) -> tuple[str, ToolFailureDetail]:+ """A connection the owner can fix explains a run better than whatever+ else failed. Of several, the least key: tools called together finish in+ any order, and a run failing the same ways must fail the same way."""+ fixable = CONNECTION_ERRORS | CHAT_LINK_ERRORS+ candidates = [f for f in failures if f[1].error_type in fixable] or failures+ return min(candidates, key=lambda failure: _failure_of(*failure).key)+++def _failure_of(tool: str, detail: ToolFailureDetail) -> RunFailure:+ if detail.error_type in CONNECTION_ERRORS:+ provider = detail.provider or tool+ return RunFailure(+ key=f"connection:{provider}",+ kind="connection",+ provider=detail.provider,+ tool=tool,+ )+ if detail.error_type in CHAT_LINK_ERRORS:+ return RunFailure(+ key=f"chat_link:{detail.provider or tool}",+ kind="chat_link",+ provider=detail.provider,+ tool=tool,+ )+ suffix = detail.text_hash or detail.error_type+ return RunFailure(key=f"tool:{tool}:{suffix}", kind="error", tool=tool)+++def tool_failure_detail(output: str | dict) -> ToolFailureDetail | None:+ """What a copilot tool's ``ErrorResponse`` said, or None when the output+ is not one. Tools report failures as a payload rather than by raising."""+ payload = output+ if isinstance(output, str):+ try:+ payload = orjson.loads(output)+ except orjson.JSONDecodeError:+ return None+ if not isinstance(payload, dict) or payload.get("type") != ResponseType.ERROR:+ return None+ error = payload.get("error")+ details = payload.get("details")+ provider = details.get("provider") if isinstance(details, dict) else None+ provider = provider if isinstance(provider, str) and provider else None+ if isinstance(error, str) and _ERROR_CODE.fullmatch(error):+ return ToolFailureDetail(error_type=error, provider=provider)+ return ToolFailureDetail(+ error_type=UNCLASSIFIED,+ provider=provider,+ text_hash=text_hash(error or payload.get("message")),+ )+++def text_hash(text: object) -> str:+ """Compares two uncoded errors without keeping what either says. Words+ with a digit in them don't count, so a timeout after 30 s and one after+ 31 s, each with its own request id, fail the same way."""+ raw = text if isinstance(text, str) else repr(text)+ return hashlib.sha256(_VOLATILE_WORD.sub("#", raw).encode()).hexdigest()[:12]diff --git a/autogpt_platform/backend/backend/copilot/scheduled_chats/outcomes_test.py b/autogpt_platform/backend/backend/copilot/scheduled_chats/outcomes_test.pynew file mode 100644index 0000000000..63f8b64a81--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/scheduled_chats/outcomes_test.py@@ -0,0 +1,186 @@+import pytest++from backend.copilot.scheduled_chats.outcomes import (+ UNCLASSIFIED,+ ToolFailureDetail,+ classify_run,+ tool_failure_detail,+)+from backend.copilot.tools.models import ErrorResponse+++def _run(**overrides):+ base = dict(+ cancelled=False,+ turn_error_code=None,+ turn_errored=False,+ tool_failures=[],+ tool_successes=0,+ )+ base.update(overrides)+ return classify_run(**base)+++def _rejected(provider: str = "twitter") -> ToolFailureDetail:+ return ToolFailureDetail(error_type="credential_rejected", provider=provider)+++def test_a_run_whose_tools_worked_is_fine():+ outcome = _run(tool_successes=3)++ assert outcome.failure is None+ assert not outcome.idle+ assert not outcome.neutral+++def test_a_run_that_called_no_tool_is_idle():+ assert _run().idle+++def test_a_refused_connection_with_other_steps_working_fails_one_step():+ """The Twitter briefing: it still went out, without Twitter."""+ outcome = _run(tool_failures=[("run_block", _rejected())], tool_successes=4)++ assert outcome.failure is not None+ assert outcome.failure.key == "connection:twitter"+ assert outcome.failure.kind == "connection"+ assert outcome.failure.provider == "twitter"+ assert not outcome.whole_run+++def test_a_run_whose_only_step_failed_fails_outright():+ outcome = _run(tool_failures=[("run_block", _rejected("discord"))])++ assert outcome.whole_run+ assert outcome.failure is not None+ assert outcome.failure.key == "connection:discord"+++def test_a_connection_the_owner_can_fix_explains_the_run_first():+ other = ToolFailureDetail(error_type=UNCLASSIFIED, text_hash="abc")+ outcome = _run(+ tool_failures=[("web_fetch", other), ("run_block", _rejected())],+ tool_successes=1,+ )++ assert outcome.failure is not None+ assert outcome.failure.kind == "connection"+++ "failures",+ [+ [("run_block", _rejected("twitter")), ("run_block", _rejected("discord"))],+ [+ ("web_fetch", ToolFailureDetail(error_type=UNCLASSIFIED, text_hash="b")),+ ("run_agent", ToolFailureDetail(error_type=UNCLASSIFIED, text_hash="a")),+ ],+ ],+ ids=["two-connections", "two-errors"],+)+def test_a_run_failing_the_same_ways_fails_the_same_way_in_any_order(failures):+ """Tools called together finish in any order: whichever came back first+ must not decide, or the streak would start over every run."""+ first = _run(tool_failures=failures)+ reversed_ = _run(tool_failures=failures[::-1])++ assert first.failure and reversed_.failure+ assert first.failure.key == reversed_.failure.key+++def test_an_unlinked_chat_platform_is_its_own_kind():+ detail = ToolFailureDetail(error_type="no_linked_servers", provider="discord")+ outcome = _run(tool_failures=[("post_to_chat_platform", detail)])++ assert outcome.failure is not None+ assert outcome.failure.key == "chat_link:discord"+ assert outcome.failure.kind == "chat_link"+++def test_an_uncoded_error_is_compared_by_its_hash():+ a = ToolFailureDetail(error_type=UNCLASSIFIED, text_hash="aaa")+ b = ToolFailureDetail(error_type=UNCLASSIFIED, text_hash="bbb")++ first = _run(tool_failures=[("run_agent", a)])+ same = _run(tool_failures=[("run_agent", a)])+ other = _run(tool_failures=[("run_agent", b)])++ assert first.failure and same.failure and other.failure+ assert first.failure.key == same.failure.key == "tool:run_agent:aaa"+ assert first.failure.key != other.failure.key++[email protected]("code", ["max_budget_exhausted", "max_turns_exhausted"])+def test_a_run_over_its_own_limit_fails_outright(code):+ outcome = _run(turn_errored=True, turn_error_code=code, tool_successes=2)++ assert outcome.whole_run+ assert outcome.failure is not None+ assert outcome.failure.key == f"turn:{code}"+ assert outcome.failure.kind == "budget"+++ "code", ["transient_api_error", "idle_timeout", "sdk_stream_error", None]+)+def test_the_platforms_weather_neither_counts_nor_clears(code):+ assert _run(turn_errored=True, turn_error_code=code).neutral+++def test_a_cancelled_run_neither_counts_nor_clears():+ outcome = _run(+ cancelled=True, turn_errored=True, tool_failures=[("x", _rejected())]+ )++ assert outcome.neutral+++def test_tool_failure_detail_reads_the_code_and_provider():+ payload = ErrorResponse(+ message="Twitter rejected the saved credential (HTTP 401).",+ error="credential_rejected",+ details={"provider": "twitter"},+ ).model_dump_json()++ detail = tool_failure_detail(payload)++ assert detail == ToolFailureDetail(+ error_type="credential_rejected", provider="twitter"+ )+++def test_tool_failure_detail_hashes_free_text_instead_of_keeping_it():+ payload = ErrorResponse(+ message="Failed to execute block",+ error="Unauthorized (401) for user [email protected]",+ ).model_dump()++ detail = tool_failure_detail(payload)++ assert detail is not None+ assert detail.error_type == UNCLASSIFIED+ assert detail.text_hash is not None+ assert "nick" not in detail.model_dump_json()+++def test_an_uncoded_error_differing_only_in_its_numbers_fails_the_same_way():+ """Ids, times and durations change every run; the failure doesn't."""++ def hashed(error: str) -> str | None:+ detail = tool_failure_detail(+ ErrorResponse(message="x", error=error).model_dump()+ )+ assert detail is not None+ return detail.text_hash++ first = hashed("Timed out after 30.2s (request 8f3a9c, 2026-10-19T08:00:00Z)")+ again = hashed("Timed out after 31.7s (request 77b1e0, 2026-10-20T08:00:00Z)")+ other = hashed("Connection refused (request 8f3a9c)")++ assert first == again+ assert first != other+++def test_tool_failure_detail_ignores_anything_but_an_error_payload():+ assert tool_failure_detail('{"type": "block_output"}') is None+ assert tool_failure_detail("not json") is Nonediff --git a/autogpt_platform/backend/backend/copilot/scheduled_chats/service.py b/autogpt_platform/backend/backend/copilot/scheduled_chats/service.pynew file mode 100644index 0000000000..f276339d95--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/scheduled_chats/service.py@@ -0,0 +1,380 @@+"""A scheduled chat's failure streak, kept in the database (SECRT-2879).++Runs in the DatabaseManager: the copilot executor records each scheduled run's+outcome through one RPC (``record_scheduled_run``), and the REST API resumes a+paused chat (``resume_scheduled_chat``). This module loads the streak, asks+``streaks.advance`` what the run changes, and carries it out: pause the+schedule's jobs, persist the streak, and tell the owner in the chat.+"""++import logging+from datetime import datetime, timezone++from prisma.errors import UniqueViolationError+from prisma.models import ExpertRoutine, ScheduledChatStreak+from prisma.types import (+ ExpertRoutineWhereInput,+ ScheduledChatStreakWhereInput,+ ScheduledChatStreakWhereUniqueInput,+)+from pydantic import BaseModel, ValidationError++from backend.copilot.scheduled_chats.failure_notices import (+ failing_notice,+ pause_summary,+ paused_notice,+ resumed_notice,+)+from backend.copilot.scheduled_chats.jobs import (+ jobs_paused,+ pause_jobs,+ resume_jobs,+ unpause,+)+from backend.copilot.scheduled_chats.notices import ScheduledChatNotice, post_notice+from backend.copilot.scheduled_chats.outcomes import RunFailure, RunOutcome+from backend.copilot.scheduled_chats.streaks import Streak, StreakStep, advance++logger = logging.getLogger(__name__)+++class ScheduledRun(BaseModel):+ """One scheduled run's outcome, as the copilot executor saw it."""++ user_id: str+ # The routine's id, or the follow-up's schedule id.+ schedule_key: str+ schedule_id: str | None = None+ routine_id: str | None = None+ # The chat the run went to.+ session_id: str+ # A follow-up's name; a routine's comes from its row.+ title: str+ outcome: RunOutcome+++class PausedScheduledChat(BaseModel):+ """A scheduled chat its failure streak paused, as the lists show it."""++ # What Resume takes: the routine's id, or the follow-up's schedule id.+ schedule_key: str+ title: str+ # The chat its last run went to, where the note with the fix is.+ session_id: str | None+ reason: str+ provider: str | None+ # "Paused: can't use Twitter", and so on.+ summary: str+ paused_at: datetime+ # A routine is deleted with its expert, not from this list.+ routine: bool = False+++async def record_scheduled_run(run: ScheduledRun) -> None:+ where = _key(run.user_id, run.schedule_key)+ row = await ScheduledChatStreak.prisma().find_unique(where=where)+ current = _streak_of(row)+ if current is not None and current.paused and await _still_paused(run):+ # It fired before the pause and finished after it, so it changes+ # nothing. One that fired after was resumed some other way.+ return+ step = advance(current, run.outcome)+ if step.streak is None:+ if row is not None:+ await _drop(row, saw_pause=current is not None and current.paused)+ return+ if step.streak == current:+ return+ streak = step.streak+ schedule_ids, title = await _schedule_of(run)+ changed = await pause_jobs(run.user_id, schedule_ids) if step.pause else None+ # Only a pause that stopped something counts, and only what it+ # stopped comes back on Resume.+ paused = bool(changed)+ if step.pause and not paused:+ # Nothing was paused, so nothing may say it was. Each later run tries+ # again; meanwhile the owner is told once, without that claim.+ streak = streak.model_copy(update={"paused": False})+ told = streak.notified and not (current is not None and current.notified)+ try:+ row = await _save(+ run,+ streak,+ changed or [],+ title,+ restarted=row is None or streak.failures == 1,+ told=told,+ replaces_pause=current is not None and current.paused,+ )+ except Exception:+ # A pause with no row behind it is never listed or resumed: the chat+ # would stop without a word. Put the jobs back; the next run retries.+ if changed:+ await unpause(run.user_id, changed)+ raise+ if row is None:+ # Another run of this chat paused it after this one read the streak,+ # and told the owner. That pause stands.+ return+ notice = _notice(step, streak, paused=paused, told=told, title=title, row=row)+ if notice is not None:+ await _tell(run, row, notice, told=told)+++async def resume_scheduled_chat(user_id: str, schedule_key: str) -> bool:+ """Resume a scheduled chat its streak paused, and say whether it was.++ Its jobs are resumed before the streak is cleared, all of them or none:+ a failure keeps the row, so the chat still reads as paused and Resume can+ be tried again.+ """+ row = await ScheduledChatStreak.prisma().find_unique(+ where=_key(user_id, schedule_key)+ )+ if row is None or row.pausedAt is None:+ return False+ await resume_jobs(user_id, row.scheduleIds)+ await ScheduledChatStreak.prisma().delete(where={"id": row.id})+ if row.sessionId is not None:+ try:+ await post_notice(+ resumed_notice(title=row.title, streak_id=row.id),+ user_id=user_id,+ schedule_key=schedule_key,+ title=row.title,+ session_id=row.sessionId,+ expert_id=None,+ )+ except Exception:+ logger.warning(+ "Could not post the resumed note for scheduled chat %s",+ schedule_key,+ exc_info=True,+ )+ return True+++async def forget_scheduled_chat(user_id: str, schedule_key: str) -> None:+ """Drop the streak of a follow-up its owner deleted. Kept, a paused one+ would stay listed as paused, with a Resume that says it runs again.++ A routine's streak is left alone: its paused jobs outlive any one+ schedule id, and a streak dropped from under them would stop the routine+ without a word."""+ if await ExpertRoutine.prisma().find_first(+ where={"id": schedule_key, **_owned_by(user_id)}+ ):+ return+ await ScheduledChatStreak.prisma().delete_many(+ where={"userId": user_id, "scheduleKey": schedule_key}+ )+++async def list_paused_scheduled_chats(user_id: str) -> list[PausedScheduledChat]:+ rows = await ScheduledChatStreak.prisma().find_many(+ where={"userId": user_id, "NOT": [{"pausedAt": None}]},+ order={"pausedAt": "desc"},+ )+ routines = await ExpertRoutine.prisma().find_many(+ where={"id": {"in": [row.scheduleKey for row in rows]}, **_owned_by(user_id)}+ )+ routine_ids = {routine.id for routine in routines}+ paused: list[PausedScheduledChat] = []+ for row in rows:+ streak = _streak_of(row)+ if streak is None or row.pausedAt is None:+ continue+ paused.append(+ PausedScheduledChat(+ schedule_key=row.scheduleKey,+ title=row.title,+ session_id=row.sessionId,+ reason=streak.failure.kind,+ provider=streak.failure.provider,+ summary=pause_summary(streak.failure),+ paused_at=row.pausedAt,+ routine=row.scheduleKey in routine_ids,+ )+ )+ return paused+++async def _still_paused(run: ScheduledRun) -> bool:+ """Whether the chat's jobs are paused yet. A routine's are its current+ ones: editing it while paused replaces them with jobs that fire."""+ schedule_ids, _ = await _schedule_of(run)+ return await jobs_paused(run.user_id, schedule_ids)+++def _notice(+ step: StreakStep,+ streak: Streak,+ *,+ paused: bool,+ told: bool,+ title: str,+ row: ScheduledChatStreak,+) -> ScheduledChatNotice | None:+ failure, failures = streak.failure, streak.failures+ if paused:+ return paused_notice(failure, title=title, failures=failures, streak_id=row.id)+ if step.pause and told:+ return paused_notice(+ failure, title=title, failures=failures, streak_id=row.id, paused=False+ )+ if step.tell:+ return failing_notice(failure, title=title, failures=failures, streak_id=row.id)+ return None+++async def _tell(+ run: ScheduledRun,+ row: ScheduledChatStreak,+ notice: ScheduledChatNotice,+ *,+ told: bool,+) -> None:+ try:+ await post_notice(+ notice,+ user_id=run.user_id,+ schedule_key=run.schedule_key,+ title=row.title,+ session_id=run.session_id,+ expert_id=None,+ )+ except Exception:+ logger.warning(+ "Could not post the note for scheduled chat %s",+ run.schedule_key,+ exc_info=True,+ )+ if told:+ # Not told after all: a later run that fails this way tells+ # again. A paused chat has no later run, but the Scheduled page+ # still lists it, with Resume.+ await ScheduledChatStreak.prisma().update(+ where={"id": row.id}, data={"notifiedAt": None}+ )+++async def _schedule_of(run: ScheduledRun) -> tuple[list[str], str]:+ """The jobs to pause and the name to use: a routine's from its row, which+ knows every fire time, a follow-up's from the run."""+ fallback = [run.schedule_id] if run.schedule_id else []+ if run.routine_id is None:+ return fallback, run.title+ routine = await ExpertRoutine.prisma().find_unique(where={"id": run.routine_id})+ if routine is None:+ return fallback, run.title+ return list(routine.scheduleIds) or fallback, routine.title+++async def _save(+ run: ScheduledRun,+ streak: Streak,+ paused_ids: list[str],+ title: str,+ *,+ restarted: bool,+ told: bool,+ replaces_pause: bool,+) -> ScheduledChatStreak | None:+ """A streak that restarted gets a new row, so its id names this streak+ alone: the notes it posts are deduplicated on it. *told* stamps when the+ owner was told, once.++ Two runs of an every-minute chat can finish together. A view of the+ streak from before a pause never overwrites that pause: its jobs are+ paused, and a row that says otherwise could never be resumed. Only a run+ that saw the pause, and found the jobs firing again, replaces it+ (*replaces_pause*). Returns None, having written nothing, when the pause+ stands."""+ now = datetime.now(tz=timezone.utc)+ fields = {+ "scheduleIds": paused_ids,+ "title": title,+ "sessionId": run.session_id,+ "failureKey": streak.failure.key,+ "failureKind": streak.failure.kind,+ "provider": streak.failure.provider,+ "tool": streak.failure.tool,+ "failures": streak.failures,+ "wholeRunFailures": streak.whole_run_failures,+ # Only the run that pauses a schedule writes here: a paused one fires+ # no runs until it is resumed, and resuming removes the row.+ "pausedAt": now if streak.paused else None,+ }+ where: ScheduledChatStreakWhereInput = {+ "userId": run.user_id,+ "scheduleKey": run.schedule_key,+ }+ if not (streak.paused or replaces_pause):+ where["pausedAt"] = None+ if not restarted:+ updated = await ScheduledChatStreak.prisma().update_many(+ where=where, data={**fields, **({"notifiedAt": now} if told else {})}+ )+ if updated:+ return await ScheduledChatStreak.prisma().find_unique(+ where=_key(run.user_id, run.schedule_key)+ )+ await ScheduledChatStreak.prisma().delete_many(where=where)+ try:+ return await ScheduledChatStreak.prisma().create(+ data={+ **fields,+ "userId": run.user_id,+ "scheduleKey": run.schedule_key,+ "notifiedAt": now if told else None,+ }+ )+ except UniqueViolationError:+ # The paused row the delete left in place.+ return None+++async def _drop(row: ScheduledChatStreak, *, saw_pause: bool) -> bool:+ """End the streak a run read, and say whether it did. As in ``_save``, a+ view from before a pause never ends that pause: its jobs are paused, and+ without the row nothing lists or resumes them. Only a run that saw the+ pause, and found the jobs firing again, ends one."""+ where: ScheduledChatStreakWhereInput = {"id": row.id}+ if not saw_pause:+ where["pausedAt"] = None+ return bool(await ScheduledChatStreak.prisma().delete_many(where=where))+++def _streak_of(row: ScheduledChatStreak | None) -> Streak | None:+ if row is None:+ return None+ try:+ failure = RunFailure.model_validate(+ {+ "key": row.failureKey,+ "kind": row.failureKind,+ "provider": row.provider,+ "tool": row.tool,+ }+ )+ except ValidationError:+ # A kind this code no longer knows: start the count over.+ logger.warning("Dropping unreadable streak %s", row.id)+ return None+ return Streak(+ failure=failure,+ failures=row.failures,+ whole_run_failures=row.wholeRunFailures,+ notified=row.notifiedAt is not None,+ paused=row.pausedAt is not None,+ )+++def _key(user_id: str, schedule_key: str) -> ScheduledChatStreakWhereUniqueInput:+ return {"userId_scheduleKey": {"userId": user_id, "scheduleKey": schedule_key}}+++def _owned_by(user_id: str) -> ExpertRoutineWhereInput:+ """The account's own routines and its experts'. A streak's key is the+ owner's routine only if it names one of these."""+ return {"OR": [{"userId": user_id}, {"Expert": {"is": {"ownerUserId": user_id}}}]}diff --git a/autogpt_platform/backend/backend/copilot/scheduled_chats/service_test.py b/autogpt_platform/backend/backend/copilot/scheduled_chats/service_test.pynew file mode 100644index 0000000000..a07507ea60--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/scheduled_chats/service_test.py@@ -0,0 +1,749 @@+"""The failure streak against a real database (SECRT-2879).++The scheduler is mocked; the notes are really written, into a real chat, the+way the DatabaseManager writes them.+"""++import asyncio+import logging+from unittest.mock import AsyncMock, MagicMock, patch+from uuid import uuid4++import pytest+from prisma.errors import UniqueViolationError+from prisma.models import ChatMessage, ExpertRoutine, ScheduledChatStreak, User++from backend.api.features.experts import routines+from backend.copilot import db as copilot_db+from backend.copilot.scheduled_chats import service+from backend.copilot.scheduled_chats.outcomes import RunFailure, RunOutcome+from backend.copilot.scheduled_chats.service import (+ ScheduledRun,+ list_paused_scheduled_chats,+ record_scheduled_run,+ resume_scheduled_chat,+)+from backend.util.exceptions import NotFoundError+from backend.util.test import SpinTestServer++logger = logging.getLogger(__name__)++_SERVICE = "backend.copilot.scheduled_chats.service"+_JOBS = "backend.copilot.scheduled_chats.jobs"++_DISCORD = RunFailure(+ key="connection:discord", kind="connection", provider="discord", tool="run_block"+)+_TWITTER = RunFailure(+ key="connection:twitter", kind="connection", provider="twitter", tool="run_block"+)+_TWITTER_OUTRIGHT = RunOutcome(failure=_TWITTER, whole_run=True)+++def scheduler(mocker) -> MagicMock:+ client = MagicMock(+ pause_schedule=AsyncMock(return_value=True),+ resume_schedule=AsyncMock(return_value=True),+ # Paused jobs drop out of this list, so empty means none fires.+ get_execution_schedules=AsyncMock(return_value=[]),+ )+ mocker.patch(f"{_JOBS}.get_scheduler_client", return_value=client)+ return client+++async def _owner() -> tuple[str, str]:+ user_id = f"streak-{uuid4()}"+ try:+ await User.prisma().create(+ data={"id": user_id, "email": f"{user_id}@example.com", "name": "Streak"}+ )+ except UniqueViolationError:+ pass+ session = await copilot_db.create_chat_session(+ session_id=str(uuid4()), user_id=user_id+ )+ return user_id, session.session_id+++async def _cleanup(user_id: str) -> None:+ try:+ await User.prisma().delete_many(where={"id": user_id})+ except Exception as exc:+ logger.warning("cleanup for %s failed: %s", user_id, exc)+++def _run(user_id: str, session_id: str, outcome: RunOutcome, **overrides):+ base = dict(+ user_id=user_id,+ schedule_key="sched-1",+ schedule_id="sched-1",+ routine_id=None,+ session_id=session_id,+ title="Discord monitor",+ outcome=outcome,+ )+ base.update(overrides)+ return ScheduledRun(**base)+++async def _notes(session_id: str) -> list[ChatMessage]:+ rows = await ChatMessage.prisma().find_many(+ where={"sessionId": session_id}, order={"sequence": "asc"}+ )+ return [+ r+ for r in rows+ if isinstance(r.metadata, dict)+ and r.metadata.get("kind") == "scheduled_chat_notice"+ ]++[email protected](loop_scope="session")+async def test_the_monitor_is_paused_and_told_on_the_third_run(+ server: SpinTestServer, scheduler: MagicMock+):+ user_id, session_id = await _owner()+ try:+ for outcome in (+ RunOutcome(failure=_DISCORD, whole_run=True),+ RunOutcome(idle=True),+ RunOutcome(idle=True),+ ):+ await record_scheduled_run(_run(user_id, session_id, outcome))++ scheduler.pause_schedule.assert_awaited_once_with("sched-1", user_id=user_id)+ row = await ScheduledChatStreak.prisma().find_first(where={"userId": user_id})+ assert row is not None+ assert row.pausedAt is not None and row.notifiedAt is not None+ assert row.failures == 3+ notes = await _notes(session_id)+ assert [n.metadata["headline"] for n in notes] == [+ "Paused: “Discord monitor” can't use Discord"+ ]+ assert notes[0].metadata["hidden_from_model"] is True+ assert notes[0].metadata["schedule_key"] == "sched-1"++ paused = await list_paused_scheduled_chats(user_id)+ assert [p.summary for p in paused] == ["Paused: can't use Discord"]+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_failing_step_is_told_once_and_never_paused(+ server: SpinTestServer, scheduler: MagicMock+):+ user_id, session_id = await _owner()+ try:+ for _ in range(4):+ await record_scheduled_run(+ _run(+ user_id,+ session_id,+ RunOutcome(failure=_TWITTER, whole_run=False),+ title="Daily briefing",+ )+ )++ scheduler.pause_schedule.assert_not_awaited()+ notes = await _notes(session_id)+ assert [n.metadata["headline"] for n in notes] == [+ "“Daily briefing” can't use Twitter"+ ]+ row = await ScheduledChatStreak.prisma().find_first(where={"userId": user_id})+ assert row is not None and row.failures == 4 and row.pausedAt is None+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_good_run_clears_the_streak(+ server: SpinTestServer, scheduler: MagicMock+):+ user_id, session_id = await _owner()+ try:+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ await record_scheduled_run(_run(user_id, session_id, failed))+ await record_scheduled_run(_run(user_id, session_id, RunOutcome()))++ assert await ScheduledChatStreak.prisma().count(where={"userId": user_id}) == 0+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_routine_pauses_every_fire_time_and_is_named_by_its_title(+ server: SpinTestServer, scheduler: MagicMock+):+ user_id, session_id = await _owner()+ try:+ routine = await ExpertRoutine.prisma().create(+ data={+ "userId": user_id,+ "title": "Morning and noon sweep",+ "prompt": "Check the inbox.",+ "crons": ["30 8 * * *", "0 13 * * *"],+ "asks": [],+ "scheduleIds": ["job-a", "job-b"],+ }+ )+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for job in ("job-a", "job-b", "job-a"):+ await record_scheduled_run(+ _run(+ user_id,+ session_id,+ failed,+ schedule_key=routine.id,+ schedule_id=job,+ routine_id=routine.id,+ title="check the inbox",+ )+ )++ paused = {c.args[0] for c in scheduler.pause_schedule.await_args_list}+ assert paused == {"job-a", "job-b"}+ notes = await _notes(session_id)+ assert notes[0].metadata["title"] == "Morning and noon sweep"+ # Deleted with its expert, so the paused list offers no Delete.+ assert [p.routine for p in await list_paused_scheduled_chats(user_id)] == [True]+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_switching_a_paused_routine_off_forgets_its_streak(+ server: SpinTestServer, scheduler: MagicMock, mocker+):+ """Its jobs are deleted as it is switched off, and switching it on again+ makes new ones that fire. Kept, the streak would still list it as+ paused, with a Resume that says it runs again."""+ scheduler.delete_schedule = AsyncMock()+ mocker.patch(+ "backend.api.features.experts.routine_jobs.get_scheduler_client",+ return_value=scheduler,+ )+ user_id, session_id = await _owner()+ try:+ routine = await ExpertRoutine.prisma().create(+ data={+ "userId": user_id,+ "title": "Morning sweep",+ "prompt": "Check the inbox.",+ "crons": ["30 8 * * *"],+ "asks": [],+ "scheduleIds": ["job-a"],+ }+ )+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(3):+ await record_scheduled_run(+ _run(+ user_id,+ session_id,+ failed,+ schedule_key=routine.id,+ schedule_id="job-a",+ routine_id=routine.id,+ )+ )+ assert len(await list_paused_scheduled_chats(user_id)) == 1++ await routines.disable_routine(user_id, None, routine.id)++ assert await list_paused_scheduled_chats(user_id) == []+ assert not await resume_scheduled_chat(user_id, routine.id)+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_routine_is_switched_off_even_when_its_streak_cannot_be_cleared(+ server: SpinTestServer, scheduler: MagicMock, mocker+):+ """Its jobs are deleted and its row says off by then: a 500 would say the+ switch failed."""+ scheduler.delete_schedule = AsyncMock()+ mocker.patch(+ "backend.api.features.experts.routine_jobs.get_scheduler_client",+ return_value=scheduler,+ )+ mocker.patch.object(+ routines, "forget_routine", AsyncMock(side_effect=RuntimeError("db blip"))+ )+ user_id, _ = await _owner()+ try:+ routine = await _routine(user_id)++ await routines.disable_routine(user_id, None, routine.id)++ row = await ExpertRoutine.prisma().find_unique(where={"id": routine.id})+ assert row is not None and row.scheduleIds == [] and row.enabledAt is None+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_schedule_that_could_not_be_paused_does_not_say_it_was(+ server: SpinTestServer, scheduler: MagicMock+):+ user_id, session_id = await _owner()+ scheduler.pause_schedule.side_effect = RuntimeError("scheduler down")+ try:+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(4):+ await record_scheduled_run(_run(user_id, session_id, failed))++ # Every run from the third tries the pause again.+ assert scheduler.pause_schedule.await_count == 2+ row = await ScheduledChatStreak.prisma().find_first(where={"userId": user_id})+ assert row is not None and row.pausedAt is None+ # The owner is still told, once, without being told it is paused.+ assert [n.metadata["headline"] for n in await _notes(session_id)] == [+ "“Discord monitor” can't use Discord"+ ]+ finally:+ await _cleanup(user_id)+++async def _routine(user_id: str) -> ExpertRoutine:+ return await ExpertRoutine.prisma().create(+ data={+ "userId": user_id,+ "title": "Morning and noon sweep",+ "prompt": "Check the inbox.",+ "crons": ["30 8 * * *", "0 13 * * *"],+ "asks": [],+ "scheduleIds": ["job-a", "job-b"],+ }+ )+++def _routine_run(user_id: str, session_id: str, routine: ExpertRoutine, job: str):+ return _run(+ user_id,+ session_id,+ RunOutcome(failure=_DISCORD, whole_run=True),+ schedule_key=routine.id,+ schedule_id=job,+ routine_id=routine.id,+ )++[email protected](loop_scope="session")+async def test_a_pause_that_fails_partway_puts_back_what_it_paused(+ server: SpinTestServer, scheduler: MagicMock+):+ async def pause(schedule_id: str, *, user_id: str) -> bool:+ if schedule_id == "job-b":+ raise RuntimeError("scheduler refused")+ return True++ scheduler.pause_schedule.side_effect = pause+ user_id, session_id = await _owner()+ try:+ routine = await _routine(user_id)+ for job in ("job-a", "job-b", "job-a"):+ await record_scheduled_run(_routine_run(user_id, session_id, routine, job))++ scheduler.resume_schedule.assert_awaited_once_with("job-a", user_id=user_id)+ assert await list_paused_scheduled_chats(user_id) == []+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_pause_whose_streak_cannot_be_saved_is_put_back(+ server: SpinTestServer, scheduler: MagicMock, mocker+):+ """Paused jobs with no row behind them would never be listed or resumed:+ the chat would stop without a word."""+ user_id, session_id = await _owner()+ try:+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(2):+ await record_scheduled_run(_run(user_id, session_id, failed))+ mocker.patch(f"{_SERVICE}._save", AsyncMock(side_effect=RuntimeError("db")))++ with pytest.raises(RuntimeError):+ await record_scheduled_run(_run(user_id, session_id, failed))++ scheduler.pause_schedule.assert_awaited_once_with("sched-1", user_id=user_id)+ scheduler.resume_schedule.assert_awaited_once_with("sched-1", user_id=user_id)+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_run_that_fired_before_the_pause_changes_nothing(+ server: SpinTestServer, scheduler: MagicMock+):+ """An every-minute chat can have its next run going when the third pauses+ it. That run finishes after the pause, and must not undo it."""+ user_id, session_id = await _owner()+ try:+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(3):+ await record_scheduled_run(_run(user_id, session_id, failed))++ for straggler in (failed, RunOutcome()):+ await record_scheduled_run(_run(user_id, session_id, straggler))++ assert len(await list_paused_scheduled_chats(user_id)) == 1+ assert len(await _notes(session_id)) == 1+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_two_runs_finishing_together_keep_the_pause(+ server: SpinTestServer, scheduler: MagicMock+):+ """Both read two failures; one pauses the job and records it, then the+ other finds the job already paused. Its view from before the pause must+ not erase it, or the chat stays paused with nothing to resume it by."""+ straggler_waiting = asyncio.Event()+ pause_recorded = asyncio.Event()++ async def pause(schedule_id: str, *, user_id: str) -> bool:+ if not straggler_waiting.is_set():+ straggler_waiting.set()+ await pause_recorded.wait()+ return False+ return True++ scheduler.pause_schedule.side_effect = pause+ user_id, session_id = await _owner()+ try:+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(2):+ await record_scheduled_run(_run(user_id, session_id, failed))++ straggler = asyncio.create_task(+ record_scheduled_run(_run(user_id, session_id, failed))+ )+ await straggler_waiting.wait()+ await record_scheduled_run(_run(user_id, session_id, failed))+ pause_recorded.set()+ await straggler++ assert len(await list_paused_scheduled_chats(user_id)) == 1+ assert len(await _notes(session_id)) == 1+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_good_run_that_read_the_streak_before_the_pause_keeps_it(+ server: SpinTestServer, scheduler: MagicMock+):+ """It ends the streak it read. The row another run paused since must+ stay, or its jobs stay paused with nothing to list or resume them."""+ user_id, session_id = await _owner()+ try:+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(3):+ await record_scheduled_run(_run(user_id, session_id, failed))+ streak_of = service._streak_of++ def before_the_pause(row):+ streak = streak_of(row)+ return streak.model_copy(update={"paused": False}) if streak else None++ with patch.object(service, "_streak_of", new=before_the_pause):+ await record_scheduled_run(_run(user_id, session_id, RunOutcome()))++ assert len(await list_paused_scheduled_chats(user_id)) == 1+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_new_failure_finishing_with_the_pause_keeps_it(+ server: SpinTestServer, scheduler: MagicMock, mocker+):+ """A run that failed another way read the streak before the pause, so it+ starts a new one; replacing the row would erase the pause."""+ read_it = asyncio.Event()+ pause_recorded = asyncio.Event()+ schedule_of = service._schedule_of++ async def held_schedule_of(run):+ if run.outcome == _TWITTER_OUTRIGHT:+ read_it.set()+ await pause_recorded.wait()+ return await schedule_of(run)++ mocker.patch(f"{_SERVICE}._schedule_of", new=held_schedule_of)+ user_id, session_id = await _owner()+ try:+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(2):+ await record_scheduled_run(_run(user_id, session_id, failed))++ other = asyncio.create_task(+ record_scheduled_run(_run(user_id, session_id, _TWITTER_OUTRIGHT))+ )+ await read_it.wait()+ await record_scheduled_run(_run(user_id, session_id, failed))+ pause_recorded.set()+ await other++ paused = await list_paused_scheduled_chats(user_id)+ assert [chat.provider for chat in paused] == ["discord"]+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_chat_resumed_some_other_way_starts_over(+ server: SpinTestServer, scheduler: MagicMock+):+ user_id, session_id = await _owner()+ try:+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(3):+ await record_scheduled_run(_run(user_id, session_id, failed))+ scheduler.get_execution_schedules.return_value = [MagicMock(id="sched-1")]++ await record_scheduled_run(_run(user_id, session_id, RunOutcome()))++ assert await ScheduledChatStreak.prisma().count(where={"userId": user_id}) == 0+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_note_that_could_not_be_posted_is_told_on_the_next_run(+ server: SpinTestServer, scheduler: MagicMock, mocker+):+ post = mocker.patch(+ f"{_SERVICE}.post_notice", side_effect=[RuntimeError("db"), True]+ )+ user_id, session_id = await _owner()+ try:+ failing = RunOutcome(failure=_TWITTER, whole_run=False)+ for _ in range(4):+ await record_scheduled_run(_run(user_id, session_id, failing))++ assert post.await_count == 2+ row = await ScheduledChatStreak.prisma().find_first(where={"userId": user_id})+ assert row is not None and row.notifiedAt is not None+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_resume_restarts_the_jobs_clears_the_streak_and_says_so(+ server: SpinTestServer, scheduler: MagicMock+):+ user_id, session_id = await _owner()+ try:+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(3):+ await record_scheduled_run(_run(user_id, session_id, failed))++ assert await resume_scheduled_chat(user_id, "sched-1") is True+ # A second click on the same card finds nothing paused.+ assert await resume_scheduled_chat(user_id, "sched-1") is False+ # And nobody else's chat can be resumed.+ assert await resume_scheduled_chat("someone-else", "sched-1") is False++ scheduler.resume_schedule.assert_awaited_once_with("sched-1", user_id=user_id)+ assert await ScheduledChatStreak.prisma().count(where={"userId": user_id}) == 0+ headlines = [n.metadata["headline"] for n in await _notes(session_id)]+ assert headlines[-1] == "Resumed “Discord monitor”"+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_resume_the_scheduler_refuses_keeps_the_chat_paused(+ server: SpinTestServer, scheduler: MagicMock+):+ user_id, session_id = await _owner()+ try:+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(3):+ await record_scheduled_run(_run(user_id, session_id, failed))+ scheduler.resume_schedule.side_effect = RuntimeError("scheduler down")++ with pytest.raises(RuntimeError):+ await resume_scheduled_chat(user_id, "sched-1")++ assert len(await list_paused_scheduled_chats(user_id)) == 1+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_resume_that_fails_partway_pauses_the_rest_again(+ server: SpinTestServer, scheduler: MagicMock+):+ async def resume(schedule_id: str, *, user_id: str) -> bool:+ if schedule_id == "job-b":+ raise RuntimeError("scheduler refused")+ return True++ user_id, session_id = await _owner()+ try:+ routine = await _routine(user_id)+ for job in ("job-a", "job-b", "job-a"):+ await record_scheduled_run(_routine_run(user_id, session_id, routine, job))+ scheduler.pause_schedule.reset_mock()+ scheduler.resume_schedule.side_effect = resume++ with pytest.raises(RuntimeError):+ await resume_scheduled_chat(user_id, routine.id)++ # job-b too: its resume may have landed with the answer lost.+ paused = [call.args[0] for call in scheduler.pause_schedule.await_args_list]+ assert paused == ["job-a", "job-b"]+ assert len(await list_paused_scheduled_chats(user_id)) == 1+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_resume_leaves_a_job_its_owner_paused_alone(+ server: SpinTestServer, scheduler: MagicMock+):+ """The owner had paused the noon run; the streak paused the morning one.+ Resume brings back the morning run only."""++ async def pause(schedule_id: str, *, user_id: str) -> bool:+ return schedule_id == "job-a"++ scheduler.pause_schedule.side_effect = pause+ user_id, session_id = await _owner()+ try:+ routine = await _routine(user_id)+ for _ in range(3):+ await record_scheduled_run(+ _routine_run(user_id, session_id, routine, "job-a")+ )+ assert len(await list_paused_scheduled_chats(user_id)) == 1++ assert await resume_scheduled_chat(user_id, routine.id) is True++ scheduler.resume_schedule.assert_awaited_once_with("job-a", user_id=user_id)+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_chat_whose_jobs_its_owner_paused_is_not_called_paused(+ server: SpinTestServer, scheduler: MagicMock+):+ scheduler.pause_schedule.return_value = False+ user_id, session_id = await _owner()+ try:+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(3):+ await record_scheduled_run(_run(user_id, session_id, failed))++ assert await list_paused_scheduled_chats(user_id) == []+ assert [n.metadata["headline"] for n in await _notes(session_id)] == [+ "\u201cDiscord monitor\u201d can't use Discord"+ ]+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_job_deleted_since_the_pause_does_not_block_the_resume(+ server: SpinTestServer, scheduler: MagicMock+):+ user_id, session_id = await _owner()+ try:+ routine = await _routine(user_id)+ for job in ("job-a", "job-b", "job-a"):+ await record_scheduled_run(_routine_run(user_id, session_id, routine, job))+ scheduler.resume_schedule.side_effect = [NotFoundError("gone"), True]++ assert await resume_scheduled_chat(user_id, routine.id) is True++ assert await list_paused_scheduled_chats(user_id) == []+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_deleted_chat_leaves_no_paused_ghost(+ server: SpinTestServer, scheduler: MagicMock+):+ """Deleting a paused chat used to leave its streak: still listed as+ paused, and a Resume that said it would run again."""+ user_id, session_id = await _owner()+ other_id, other_session = await _owner()+ try:+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(3):+ await record_scheduled_run(_run(user_id, session_id, failed))+ await record_scheduled_run(_run(other_id, other_session, failed))++ await service.forget_scheduled_chat(user_id, "sched-1")++ assert await ScheduledChatStreak.prisma().count(where={"userId": user_id}) == 0+ assert await list_paused_scheduled_chats(user_id) == []+ assert await resume_scheduled_chat(user_id, "sched-1") is False+ # Only the owner's own chat goes.+ [left] = await list_paused_scheduled_chats(other_id)+ # A follow-up, which the paused list can delete.+ assert left.routine is False+ finally:+ await _cleanup(user_id)+ await _cleanup(other_id)++[email protected](loop_scope="session")+async def test_forgetting_a_routines_id_leaves_its_pause_alone(+ server: SpinTestServer, scheduler: MagicMock+):+ """A delete names a schedule id; a routine's streak outlives any one of+ its jobs, so its paused row stays listed with Resume."""+ user_id, session_id = await _owner()+ try:+ routine = await _routine(user_id)+ for job in ("job-a", "job-b", "job-a"):+ await record_scheduled_run(_routine_run(user_id, session_id, routine, job))++ await service.forget_scheduled_chat(user_id, routine.id)++ paused = await list_paused_scheduled_chats(user_id)+ assert [p.schedule_key for p in paused] == [routine.id]+ finally:+ await _cleanup(user_id)++[email protected](loop_scope="session")+async def test_a_key_naming_someone_elses_routine_is_a_follow_up(+ server: SpinTestServer, scheduler: MagicMock+):+ """Only the owner's routines are kept and listed as routines: a key that+ names another account's routine is a follow-up, which a delete clears."""+ user_id, session_id = await _owner()+ other_id, _ = await _owner()+ try:+ foreign = await _routine(other_id)+ failed = RunOutcome(failure=_DISCORD, whole_run=True)+ for _ in range(3):+ await record_scheduled_run(+ _run(+ user_id,+ session_id,+ failed,+ schedule_key=foreign.id,+ schedule_id=foreign.id,+ )+ )+ [paused] = await list_paused_scheduled_chats(user_id)+ assert paused.routine is False++ await service.forget_scheduled_chat(user_id, foreign.id)++ assert await list_paused_scheduled_chats(user_id) == []+ finally:+ await _cleanup(user_id)+ await _cleanup(other_id)diff --git a/autogpt_platform/backend/backend/copilot/scheduled_chats/streaks.py b/autogpt_platform/backend/backend/copilot/scheduled_chats/streaks.pynew file mode 100644index 0000000000..ff573a3966--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/scheduled_chats/streaks.py@@ -0,0 +1,92 @@+"""How one run moves a scheduled chat's failure streak.++Pure, so the rules are testable on their own: the caller loads the streak,+asks what this run changes, and carries it out (``service``).++* Three runs in a row that failed outright the same way pause the schedule,+ and the owner is told why, with the fix.+* Three runs in a row with the same failing step tell the owner once, and the+ schedule keeps running: the rest of what it does still works.+* A run that worked clears the streak. One that called no tool continues an+ open connection failure, and otherwise changes nothing.+* A neutral run (cancelled, our outage) changes nothing.+"""++from pydantic import BaseModel++from backend.copilot.scheduled_chats.outcomes import RunFailure, RunOutcome++PAUSE_AFTER = 3+TELL_AFTER = 3++_IDLE_CONTINUES = ("connection", "chat_link")+++class Streak(BaseModel):+ failure: RunFailure+ # Runs in a row that failed this way.+ failures: int = 1+ # The latest of those runs that failed outright, in a row.+ whole_run_failures: int = 0+ notified: bool = False+ paused: bool = False+++class StreakStep(BaseModel):+ # What to keep; None clears the streak.+ streak: Streak | None+ # Pause the schedule and tell the owner why.+ pause: bool = False+ # Tell the owner, and keep running.+ tell: bool = False+ # A streak the owner was told about has ended: withdraw what it raised.+ # Its pause alert is resolved; the notes stay, as a record.+ ended_notified: bool = False+++def advance(streak: Streak | None, outcome: RunOutcome) -> StreakStep:+ if outcome.neutral:+ return StreakStep(streak=streak)+ failure, whole_run = outcome.failure, outcome.whole_run+ if failure is None:+ if not outcome.idle:+ return StreakStep(streak=None, ended_notified=_was_told(streak))+ if streak is None or not _idle_continues(streak):+ return StreakStep(streak=streak)+ failure, whole_run = streak.failure, True+ ended_notified = False+ if streak is not None and not streak.paused and streak.failure.key == failure.key:+ streak = streak.model_copy(+ update={+ "failures": streak.failures + 1,+ "whole_run_failures": (+ streak.whole_run_failures + 1 if whole_run else 0+ ),+ }+ )+ else:+ # A different failure starts its own count. A run that fired while its+ # streak said paused was resumed some other way; it starts over too.+ ended_notified = _was_told(streak)+ streak = Streak(failure=failure, whole_run_failures=int(whole_run))+ if streak.whole_run_failures >= PAUSE_AFTER:+ return StreakStep(+ streak=streak.model_copy(update={"paused": True, "notified": True}),+ pause=True,+ ended_notified=ended_notified,+ )+ if streak.failures >= TELL_AFTER and not streak.notified:+ return StreakStep(+ streak=streak.model_copy(update={"notified": True}),+ tell=True,+ ended_notified=ended_notified,+ )+ return StreakStep(streak=streak, ended_notified=ended_notified)+++def _idle_continues(streak: Streak) -> bool:+ return not streak.paused and streak.failure.kind in _IDLE_CONTINUES+++def _was_told(streak: Streak | None) -> bool:+ return streak is not None and streak.notifieddiff --git a/autogpt_platform/backend/backend/copilot/scheduled_chats/streaks_test.py b/autogpt_platform/backend/backend/copilot/scheduled_chats/streaks_test.pynew file mode 100644index 0000000000..ecf63fea86--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/scheduled_chats/streaks_test.py@@ -0,0 +1,145 @@+"""The three cases SECRT-2879 was filed for, run by run, plus the edges."""++from backend.copilot.scheduled_chats.outcomes import RunFailure, RunOutcome+from backend.copilot.scheduled_chats.streaks import Streak, advance++_DISCORD = RunFailure(+ key="connection:discord", kind="connection", provider="discord", tool="run_block"+)+_TWITTER = RunFailure(+ key="connection:twitter", kind="connection", provider="twitter", tool="run_block"+)+_BUDGET = RunFailure(key="turn:max_budget_exhausted", kind="budget")+++def _outright(failure: RunFailure) -> RunOutcome:+ return RunOutcome(failure=failure, whole_run=True)+++def _one_step(failure: RunFailure) -> RunOutcome:+ return RunOutcome(failure=failure, whole_run=False)+++_IDLE = RunOutcome(idle=True)+_FINE = RunOutcome()+_NEUTRAL = RunOutcome(neutral=True)+++def _drive(outcomes: list[RunOutcome], streak: Streak | None = None):+ steps = []+ for outcome in outcomes:+ step = advance(streak, outcome)+ steps.append(step)+ streak = step.streak+ return steps+++def test_the_every_minute_monitor_pauses_on_its_third_run():+ """Run one could not reach Discord; the next two only repeated that from+ history, calling no tool. Those count, so the third run pauses."""+ steps = _drive([_outright(_DISCORD), _IDLE, _IDLE])++ assert [s.pause for s in steps] == [False, False, True]+ final = steps[-1].streak+ assert final is not None and final.paused and final.notified+ assert final.failures == 3+++def test_a_monitor_that_cannot_post_to_discord_pauses_the_same_way():+ """An unlinked chat platform is as fixable as a missing connection, and a+ run that only repeats it from history counts the same."""+ unlinked = RunFailure(key="chat_link:discord", kind="chat_link", provider="discord")+ steps = _drive([_outright(unlinked), _IDLE, _IDLE])++ assert [s.pause for s in steps] == [False, False, True]+++def test_the_twitter_briefing_is_told_once_and_keeps_running():+ """Twitter refused every morning, the briefing still went out: tell the+ owner on the third, never pause, never tell again."""+ steps = _drive([_one_step(_TWITTER)] * 5)++ assert [s.tell for s in steps] == [False, False, True, False, False]+ assert not any(s.pause for s in steps)+ final = steps[-1].streak+ assert final is not None and final.failures == 5 and not final.paused+++def test_a_run_over_its_spending_limit_three_times_pauses():+ steps = _drive([_outright(_BUDGET)] * 3)++ assert steps[-1].pause+++def test_one_good_run_in_between_starts_the_count_over():+ steps = _drive(+ [_outright(_DISCORD), _outright(_DISCORD), _FINE, _outright(_DISCORD)]+ )++ assert not any(s.pause for s in steps)+ final = steps[-1].streak+ assert final is not None and final.failures == 1+++def test_a_different_failure_starts_its_own_count():+ steps = _drive([_outright(_DISCORD), _outright(_DISCORD), _outright(_TWITTER)])++ assert not steps[-1].pause+ final = steps[-1].streak+ assert final is not None+ assert final.failure.key == "connection:twitter"+ assert final.failures == 1+++def test_a_step_failure_breaks_a_run_of_outright_failures():+ steps = _drive(+ [+ _outright(_DISCORD),+ _outright(_DISCORD),+ _one_step(_DISCORD),+ _outright(_DISCORD),+ ]+ )++ assert not any(s.pause for s in steps)+ final = steps[-1].streak+ assert final is not None and final.whole_run_failures == 1+ # Four runs with the same failing step: the owner was told on the third.+ assert steps[2].tell+++def test_an_idle_run_without_an_open_connection_failure_changes_nothing():+ open_budget = Streak(failure=_BUDGET, failures=2, whole_run_failures=2)++ assert advance(None, _IDLE).streak is None+ assert advance(open_budget, _IDLE).streak == open_budget+++def test_the_platforms_weather_changes_nothing():+ open_streak = Streak(failure=_DISCORD, failures=2, whole_run_failures=2)++ step = advance(open_streak, _NEUTRAL)++ assert step.streak == open_streak+ assert not step.pause and not step.tell+++def test_ending_a_streak_the_owner_was_told_about_says_so():+ told = Streak(failure=_TWITTER, failures=3, notified=True)++ assert advance(told, _FINE).ended_notified+ assert not advance(Streak(failure=_TWITTER, failures=2), _FINE).ended_notified+++def test_a_run_after_a_resume_the_streak_did_not_see_starts_over():+ """Resumed some other way (the chat's own schedule tool): the next run+ fires while the row still says paused, and is counted afresh."""+ paused = Streak(+ failure=_DISCORD, failures=3, whole_run_failures=3, notified=True, paused=True+ )++ step = advance(paused, _outright(_DISCORD))++ assert step.streak is not None+ assert step.streak.failures == 1 and not step.streak.paused+ assert step.ended_notifieddiff --git a/autogpt_platform/backend/backend/copilot/tools/chat_platform.py b/autogpt_platform/backend/backend/copilot/tools/chat_platform.pyindex 0e6426ca09..b05fea824d 100644--- a/autogpt_platform/backend/backend/copilot/tools/chat_platform.py+++ b/autogpt_platform/backend/backend/copilot/tools/chat_platform.py@@ -402,6 +402,8 @@ class PostToChatPlatformTool(BaseTool):return ErrorResponse(message=_error_message(result.error, platform_name),error=result.error or "chat_platform_post_failed",+ # A scheduled chat's failure streak reads which platform.+ details={"provider": platform_name},session_id=session_id,)@@ -571,6 +573,7 @@ class EditChatPlatformMessageTool(BaseTool):return ErrorResponse(message=_error_message(result.error, platform_name),error=result.error or "chat_platform_edit_failed",+ details={"provider": platform_name},session_id=session_id,)return ChatPlatformEditedResponse(diff --git a/autogpt_platform/backend/backend/copilot/tools/helpers.py b/autogpt_platform/backend/backend/copilot/tools/helpers.pyindex 6ba878ec3c..a59a55b3b5 100644--- a/autogpt_platform/backend/backend/copilot/tools/helpers.py+++ b/autogpt_platform/backend/backend/copilot/tools/helpers.py@@ -694,6 +694,8 @@ def _credential_rejected_response("before the next run."),error="credential_rejected",+ # The scheduled chat's failure streak reads which connection to fix.+ details={"provider": provider} if provider else None,session_id=session_id,).from_outside()@@ -887,10 +889,17 @@ async def unattended_missing_credentials_error()+ grant_hint,error="missing_credentials",+ details=_single_provider(providers),session_id=session_id,).from_outside()+def _single_provider(providers: set[str]) -> dict[str, Any] | None:+ """The provider to fix, for a scheduled chat's failure streak; only when+ there is exactly one, since a reconnect button opens one."""+ return {"provider": next(iter(providers))} if len(providers) == 1 else None++async def _pinned_account_error(subject: str,providers: set[str],@@ -947,6 +956,7 @@ async def _pinned_account_error("another account for this schedule, before the next run."),error="pinned_credential_missing" if lost else "pinned_credential_unusable",+ details=_single_provider(set(lost + unfit)),session_id=session_id,).from_outside()diff --git a/autogpt_platform/backend/backend/copilot/tools/helpers_test.py b/autogpt_platform/backend/backend/copilot/tools/helpers_test.pyindex ae871a7476..b5d635d6b7 100644--- a/autogpt_platform/backend/backend/copilot/tools/helpers_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/helpers_test.py@@ -17,6 +17,7 @@ from backend.copilot.model import ChatSessionfrom backend.copilot.rate_limit import UserPaywalledErrorfrom backend.copilot.tools.helpers import (BlockPreparation,+ _single_provider,check_hitl_review,execute_block,get_block_provider,@@ -2272,3 +2273,14 @@ async def test_scheduled_expert_turn_does_not_offer_a_credential_of_the_wrong_tyassert isinstance(result, ErrorResponse)assert "exa-oauth" not in result.messageassert "connect exa" in result.message+++ ("providers", "details"),+ [({"github"}, {"provider": "github"}), (set(), None), ({"github", "slack"}, None)],+ ids=["one", "none", "several"],+)+def test_a_failure_names_its_provider_only_when_there_is_one(providers, details):+ """A scheduled chat's failure streak keys on it, and its card's Reconnect+ opens it: with several to fix, neither could pick one."""+ assert _single_provider(providers) == detailsdiff --git a/autogpt_platform/backend/backend/copilot/tools/utils_test.py b/autogpt_platform/backend/backend/copilot/tools/utils_test.pyindex b87b29664e..dd9130469d 100644--- a/autogpt_platform/backend/backend/copilot/tools/utils_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/utils_test.py@@ -409,6 +409,8 @@ async def test_the_error_names_a_pinned_account_that_cannot_do_the_step():assert result.error == "pinned_credential_unusable"assert "'Work'" in result.messageassert "did not switch" in result.message+ # What a scheduled chat's failure streak keys on and Reconnect opens.+ assert result.details == {"provider": "github"}async def test_a_pin_for_one_host_is_not_blamed_for_a_step_on_another():diff --git a/autogpt_platform/backend/backend/data/db_manager.py b/autogpt_platform/backend/backend/data/db_manager.pyindex c1a78f3260..d7dc320b08 100644--- a/autogpt_platform/backend/backend/data/db_manager.py+++ b/autogpt_platform/backend/backend/data/db_manager.py@@ -60,6 +60,7 @@ from backend.api.features.store.db import ()from backend.api.features.store.embeddings import backfill_missing_embeddingsfrom backend.copilot import db as chat_db+from backend.copilot.scheduled_chats.service import record_scheduled_runfrom backend.copilot.sharing.db import link_new_execution_to_chat_sharefrom backend.data import bot_analytics as bot_analytics_dbfrom backend.data import bot_installs as bot_installs_db@@ -782,6 +783,8 @@ class DatabaseManager(AppService):append_plain_session_message = _(chat_db.append_plain_session_message)# And to tell a scheduled chat why its run was skipped, in that chat.append_session_message = _(chat_db.append_session_message)+ # The copilot executor counts each scheduled run toward a failure streak.+ record_scheduled_run = _(record_scheduled_run)create_briefing = _(create_briefing)get_briefing_for_date = _(get_briefing_for_date)get_latest_briefings = _(get_latest_briefings)@@ -995,6 +998,7 @@ class DatabaseManagerAsyncClient(AppServiceClient):# ============ Morning Briefing ============ #append_plain_session_message = d.append_plain_session_messageappend_session_message = d.append_session_message+ record_scheduled_run = d.record_scheduled_runcreate_briefing = d.create_briefingget_briefing_for_date = d.get_briefing_for_dateget_latest_briefings = d.get_latest_briefingsdiff --git a/autogpt_platform/backend/backend/data/db_manager_test.py b/autogpt_platform/backend/backend/data/db_manager_test.pyindex adafd1566b..c1c3fee7ea 100644--- a/autogpt_platform/backend/backend/data/db_manager_test.py+++ b/autogpt_platform/backend/backend/data/db_manager_test.py@@ -54,6 +54,15 @@ def test_the_audience_consumer_can_read_the_opt_out() -> None:assert hasattr(DatabaseManagerAsyncClient, "is_marketing_opted_out")+def test_the_executor_can_record_a_scheduled_run() -> None:+ """The copilot executor has no Prisma of its own. A run's outcome, the+ deepest payload here, crosses this RPC; a schema that cannot be built+ would only show as a warning in the executor's log."""+ assert hasattr(DatabaseManagerAsyncClient, "record_scheduled_run")+ manager = DatabaseManager()+ manager._create_fastapi_endpoint(manager.record_scheduled_run)++def test_the_scheduler_can_post_a_note_into_a_chat() -> None:"""The scheduler has no Prisma of its own; a skipped run's note goesthrough this RPC."""diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex a2bbb3c5e4..989146ab64 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -1256,7 +1256,7 @@ async def _dispatch_at_reset_turn(# It carries on the user's own chat, so under the chat's own# limits, as a message sent there now would be (a builder chat's).permissions=resolve_session_permissions(session),- scheduled=ScheduledTurnOrigin(schedule_id=args.schedule_id),+ scheduled=ScheduledTurnOrigin(schedule_id=args.schedule_id, purpose=kind),)logger.info(f"Dispatched {kind}-at-reset turn for session {session.session_id[:12]}"diff --git a/autogpt_platform/backend/backend/executor/scheduler_at_reset_test.py b/autogpt_platform/backend/backend/executor/scheduler_at_reset_test.pyindex 66cb0f6850..6412ccabef 100644--- a/autogpt_platform/backend/backend/executor/scheduler_at_reset_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_at_reset_test.py@@ -276,7 +276,7 @@ async def test_it_fires_into_its_chat(fire_env, kind, message, as_user):assert sent["unattended"] is Trueassert sent["llm_auth_provider"] == "platform"assert sent["scheduled"] == ScheduledTurnOrigin(- schedule_id=f"at-reset-{kind}-{SESSION}"+ schedule_id=f"at-reset-{kind}-{SESSION}", purpose=kind)diff --git a/autogpt_platform/backend/migrations/20261011120000_scheduled_chat_streak/migration.sql b/autogpt_platform/backend/migrations/20261011120000_scheduled_chat_streak/migration.sqlnew file mode 100644index 0000000000..eb5fddb949--- /dev/null+++ b/autogpt_platform/backend/migrations/20261011120000_scheduled_chat_streak/migration.sql@@ -0,0 +1,34 @@+-- One row per scheduled chat that is failing the same way, run after run+-- (SECRT-2879). A new table: nothing existing is rewritten.++-- CreateTable+CREATE TABLE "ScheduledChatStreak" (+ "id" TEXT NOT NULL,+ "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,+ "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,+ "userId" TEXT NOT NULL,+ "scheduleKey" TEXT NOT NULL,+ "scheduleIds" TEXT[],+ "title" TEXT NOT NULL,+ "sessionId" TEXT,+ "failureKey" TEXT NOT NULL,+ "failureKind" TEXT NOT NULL,+ "provider" TEXT,+ "tool" TEXT,+ "failures" INTEGER NOT NULL DEFAULT 1,+ "wholeRunFailures" INTEGER NOT NULL DEFAULT 0,+ "notifiedAt" TIMESTAMP(3),+ "pausedAt" TIMESTAMP(3),++ CONSTRAINT "ScheduledChatStreak_pkey" PRIMARY KEY ("id")+);++-- CreateIndex+CREATE INDEX "ScheduledChatStreak_userId_pausedAt_idx" ON "ScheduledChatStreak"("userId", "pausedAt");++-- CreateIndex+CREATE UNIQUE INDEX "ScheduledChatStreak_userId_scheduleKey_key" ON "ScheduledChatStreak"("userId", "scheduleKey");++-- AddForeignKey+ALTER TABLE "ScheduledChatStreak" ADD CONSTRAINT "ScheduledChatStreak_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;+diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prismaindex 9dedb59438..1b87a61bb0 100644--- a/autogpt_platform/backend/schema.prisma+++ b/autogpt_platform/backend/schema.prisma@@ -180,6 +180,7 @@ model User {ExpertRoutines ExpertRoutine[]+ ScheduledChatStreaks ScheduledChatStreak[]MemoryScopeSchedules MemoryScopeSchedule[]Profile Profile[]@@ -1451,6 +1452,44 @@ model ExpertRoutine {@@index([userId, createdAt, id])}+// The runs a scheduled chat (an Expert routine or a follow-up) has failed the+// same way in a row. The third run that fails outright pauses it, and its+// owner is told once, with the fix; a run that works removes the row.+model ScheduledChatStreak {+ id String @id @default(uuid())+ createdAt DateTime @default(now())+ updatedAt DateTime @default(now()) @updatedAt++ userId String+ User User @relation(fields: [userId], references: [id], onDelete: Cascade)++ // The routine's id, or the follow-up's schedule id: a routine with several+ // fire times is one scheduled chat.+ scheduleKey String+ // The jobs its pause stopped, which Resume restarts. Never a job the+ // owner had paused; empty until the streak pauses.+ scheduleIds String[]+ title String+ // The chat its latest run went to, where the owner is told.+ sessionId String?++ // What the runs have in common: compared by key, explained by the rest.+ failureKey String+ failureKind String+ provider String?+ tool String?+ // Runs in a row that failed this way, and how many of the latest of those+ // failed outright, in a row.+ failures Int @default(1)+ wholeRunFailures Int @default(0)++ notifiedAt DateTime?+ pausedAt DateTime?++ @@unique([userId, scheduleKey])+ @@index([userId, pausedAt])+}+// A Skills Hub listing a roster template comes with; hiring installs its// active version into the new expert's own skills folder.model ExpertSkillListing {diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/MessageRow/MessageRow.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/MessageRow/MessageRow.tsxindex 69a5f5f407..0a1286ce0d 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/MessageRow/MessageRow.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/components/MessageRow/MessageRow.tsx@@ -125,7 +125,7 @@ export const MessageRow = memo(function MessageRow({className="duration-300 animate-in fade-in slide-in-from-bottom-2 fill-mode-both"><MessageContent className="group-[.is-assistant]:bg-transparent">- <ScheduledChatNoticeCard notice={notice} />+ <ScheduledChatNoticeCard notice={notice} sessionID={sessionID} /></MessageContent></Message>);diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CopilotModals/__tests__/CopilotModals.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CopilotModals/__tests__/CopilotModals.test.tsxindex 7bf39b1c4f..4d08884da1 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/CopilotModals/__tests__/CopilotModals.test.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/CopilotModals/__tests__/CopilotModals.test.tsx@@ -5,6 +5,7 @@ import {import {getGetV1ListExecutionSchedulesForAUserMockHandler,getListCopilotFollowupSchedulesMockHandler,+ getListPausedScheduledChatsMockHandler,} from "@/app/api/__generated__/endpoints/schedules/schedules.msw";import { getListCopilotSkillsMockHandler } from "@/app/api/__generated__/endpoints/skills/skills.msw";import { ConnectServiceDialog } from "@/components/contextual/IntegrationsPanel/components/ConnectServiceDialog/ConnectServiceDialog";@@ -148,6 +149,7 @@ describe("CopilotModals", () => {getListCopilotSkillsMockHandler([]),getListCopilotFollowupSchedulesMockHandler([]),getGetV1ListExecutionSchedulesForAUserMockHandler([]),+ getListPausedScheduledChatsMockHandler([]),getGetV1ListCredentialsMockHandler([]),);});diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/ScheduledChatNoticeCard.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/ScheduledChatNoticeCard.tsxindex 40b0bd131e..030725c92c 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/ScheduledChatNoticeCard.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/ScheduledChatNoticeCard.tsx@@ -2,28 +2,83 @@import { Button } from "@/components/atoms/Button/Button";import { Icon } from "@/components/atoms/Icon/Icon";+import { ConnectServiceDialog } from "@/components/contextual/IntegrationsPanel/components/ConnectServiceDialog/ConnectServiceDialog";import {+ Alert01Icon,+ CheckmarkCircle02Icon,Clock01Icon,CreditCardIcon,+ PauseIcon,PlugSocketIcon,} from "@hugeicons/core-free-icons";import type { IconSvgElement } from "@hugeicons/react";-import type { ScheduledChatNotice } from "./helpers";+import type { NoticeAction, ScheduledChatNotice } from "./helpers";+import { useScheduledChatNoticeCard } from "./useScheduledChatNoticeCard";interface Props {notice: ScheduledChatNotice;+ sessionID?: string | null;}const REASON_ICONS: Record<string, IconSvgElement> = {usage: Clock01Icon,no_plan: CreditCardIcon,codex_access: PlugSocketIcon,+ connection: PlugSocketIcon,+ chat_link: PlugSocketIcon,+ budget: PauseIcon,+ error: Alert01Icon,+ resumed: CheckmarkCircle02Icon,};-export function ScheduledChatNoticeCard({ notice }: Props) {+export function ScheduledChatNoticeCard({ notice, sessionID = null }: Props) {+ const card = useScheduledChatNoticeCard({+ scheduleKey: notice.scheduleKey,+ sessionID,+ });const icon = REASON_ICONS[notice.reason] ?? Clock01Icon;const showActions = notice.actions.length > 0;+ function renderAction(action: NoticeAction) {+ if (action.kind === "link") {+ return (+ <Button+ key={`link-${action.href}`}+ as="NextLink"+ href={action.href}+ variant="secondary"+ size="small"+ >+ {action.label}+ </Button>+ );+ }+ if (action.kind === "reconnect") {+ return (+ <Button+ key={`reconnect-${action.provider}`}+ variant="secondary"+ size="small"+ onClick={() => card.openConnect(action.provider)}+ >+ {action.label}+ </Button>+ );+ }+ return (+ <Button+ key="resume"+ variant="secondary"+ size="small"+ loading={card.isResuming}+ disabled={card.resumed}+ onClick={card.resume}+ >+ {card.resumed ? "Resumed" : action.label}+ </Button>+ );+ }+return (<divclassName="max-w-md rounded-2xl bg-white p-4 ring-1 ring-inset ring-zinc-200/60"@@ -36,21 +91,21 @@ export function ScheduledChatNoticeCard({ notice }: Props) {<p className="mt-1 text-sm text-zinc-500">{notice.body}</p>{showActions ? (<div className="mt-3 flex flex-wrap gap-2">- {notice.actions.map((action) => (- <Button- key={action.href}- as="NextLink"- href={action.href}- variant="secondary"- size="small"- >- {action.label}- </Button>- ))}+ {notice.actions.map(renderAction)}</div>) : null}</div></div>+ {card.connectProvider ? (+ <ConnectServiceDialog+ open+ onOpenChange={(open) => {+ if (!open) card.closeConnect();+ }}+ initialProviderId={card.connectProvider}+ onConnected={card.handleConnected}+ />+ ) : null}</div>);}diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/__tests__/ScheduledChatNoticeCard.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/__tests__/ScheduledChatNoticeCard.test.tsxindex aac22750f8..553d8374e9 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/__tests__/ScheduledChatNoticeCard.test.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/__tests__/ScheduledChatNoticeCard.test.tsx@@ -1,18 +1,46 @@-import { render, screen } from "@/tests/integrations/test-utils";-import { describe, expect, it } from "vitest";+import {+ getGetV1ListCredentialsMockHandler,+ getGetV1ListProvidersMockHandler,+} from "@/app/api/__generated__/endpoints/integrations/integrations.msw";+import { getResumeScheduledChatMockHandler } from "@/app/api/__generated__/endpoints/schedules/schedules.msw";+import { server } from "@/mocks/mock-server";+import {+ fireEvent,+ render,+ screen,+ waitFor,+} from "@/tests/integrations/test-utils";+import { http, HttpResponse } from "msw";+import { afterEach, describe, expect, it } from "vitest";import { ScheduledChatNoticeCard } from "../ScheduledChatNoticeCard";import type { ScheduledChatNotice } from "../helpers";-const NOTICE: ScheduledChatNotice = {+const SKIPPED: ScheduledChatNotice = {reason: "usage",headline: "Skipped: daily usage limit reached",body: "“Briefing” didn't run. Scheduled chats run again when the limit resets at 00:00 UTC.",+ scheduleKey: "routine-1",actions: [{ kind: "link", label: "Upgrade", href: "/settings/billing" }],};+const PAUSED: ScheduledChatNotice = {+ reason: "connection",+ headline: "Paused: “Briefing” can't use Twitter",+ body: "Its last 3 runs failed because the Twitter connection is missing or was refused.",+ scheduleKey: "routine-1",+ actions: [+ { kind: "reconnect", label: "Reconnect Twitter", provider: "twitter" },+ { kind: "resume", label: "Resume" },+ ],+};++afterEach(() => {+ server.resetHandlers();+});+describe("ScheduledChatNoticeCard", () => {it("says why the run was skipped and links to the fix", () => {- render(<ScheduledChatNoticeCard notice={NOTICE} />);+ render(<ScheduledChatNoticeCard notice={SKIPPED} />);expect(screen.getByText("Skipped: daily usage limit reached"),@@ -22,4 +50,52 @@ describe("ScheduledChatNoticeCard", () => {screen.getByRole("link", { name: "Upgrade" }).getAttribute("href"),).toBe("/settings/billing");});++ it("resumes the paused chat it is about", async () => {+ let resumedKey: string | null = null;+ server.use(+ getResumeScheduledChatMockHandler(({ params }) => {+ resumedKey = String(params.scheduleKey);+ return { resumed: true };+ }),+ );+ render(<ScheduledChatNoticeCard notice={PAUSED} sessionID="session-1" />);++ fireEvent.click(screen.getByRole("button", { name: "Resume" }));++ await waitFor(() => expect(resumedKey).toBe("routine-1"));+ expect(+ await screen.findByRole("button", { name: "Resumed" }),+ ).toHaveProperty("disabled", true);+ });++ it("treats a chat that is no longer paused as resumed", async () => {+ server.use(+ http.post("*/api/schedules/followups/:scheduleKey/resume", () =>+ HttpResponse.json(+ { detail: "This scheduled chat is not paused." },+ { status: 404 },+ ),+ ),+ );+ render(<ScheduledChatNoticeCard notice={PAUSED} />);++ fireEvent.click(screen.getByRole("button", { name: "Resume" }));++ expect(+ await screen.findByRole("button", { name: "Resumed" }),+ ).toBeDefined();+ });++ it("opens the connect dialog for the provider that failed", async () => {+ server.use(+ getGetV1ListProvidersMockHandler([]),+ getGetV1ListCredentialsMockHandler([]),+ );+ render(<ScheduledChatNoticeCard notice={PAUSED} />);++ fireEvent.click(screen.getByRole("button", { name: "Reconnect Twitter" }));++ expect(await screen.findByRole("dialog")).toBeDefined();+ });});diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/__tests__/helpers.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/__tests__/helpers.test.tsindex 2a6aa9519e..17d5601785 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/__tests__/helpers.test.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/__tests__/helpers.test.ts@@ -16,6 +16,7 @@ describe("getScheduledChatNotice", () => {reason: "usage",headline: "Skipped: daily usage limit reached",body: "“Briefing” didn't run.",+ scheduleKey: null,actions: [{ kind: "link", label: "Upgrade", href: "/settings/billing" }],});});@@ -28,7 +29,7 @@ describe("getScheduledChatNotice", () => {).toBeNull();});- it("keeps only buttons that stay on the platform", () => {+ it("keeps only links that stay on the platform", () => {const notice = getScheduledChatNotice({...NOTICE,actions: [@@ -37,11 +38,40 @@ describe("getScheduledChatNotice", () => {{ kind: "link", label: "Protocol-relative", href: "//example.com" },{ kind: "link", label: "Backslash", href: "/\\example.com" },{ kind: "link", label: "Script", href: "javascript:alert(1)" },- { kind: "resume", label: "Resume" },"not an action",],});expect(notice?.actions.map((action) => action.label)).toEqual(["Upgrade"]);});++ it("reads a paused chat's reconnect and resume", () => {+ const notice = getScheduledChatNotice({+ ...NOTICE,+ reason: "connection",+ schedule_key: "routine-1",+ actions: [+ { kind: "reconnect", label: "Reconnect Twitter", provider: "twitter" },+ { kind: "resume", label: "Resume" },+ ],+ });++ expect(notice?.scheduleKey).toBe("routine-1");+ expect(notice?.actions).toEqual([+ { kind: "reconnect", label: "Reconnect Twitter", provider: "twitter" },+ { kind: "resume", label: "Resume" },+ ]);+ });++ it("drops a resume it could not send anywhere", () => {+ const notice = getScheduledChatNotice({+ ...NOTICE,+ actions: [+ { kind: "resume", label: "Resume" },+ { kind: "reconnect", label: "Reconnect" },+ ],+ });++ expect(notice?.actions).toEqual([]);+ });});diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/helpers.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/helpers.tsindex b40273b061..1507b3103e 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/helpers.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/helpers.ts@@ -4,11 +4,29 @@ export interface NoticeLinkAction {href: string;}+export interface NoticeResumeAction {+ kind: "resume";+ label: string;+}++export interface NoticeReconnectAction {+ kind: "reconnect";+ label: string;+ provider: string;+}++export type NoticeAction =+ | NoticeLinkAction+ | NoticeResumeAction+ | NoticeReconnectAction;+export interface ScheduledChatNotice {reason: string;headline: string;body: string;- actions: NoticeLinkAction[];+ // What Resume takes: the routine's id, or the follow-up's schedule id.+ scheduleKey: string | null;+ actions: NoticeAction[];}const NOTICE_METADATA_KIND = "scheduled_chat_notice";@@ -17,13 +35,19 @@ function asString(value: unknown): string | null {return typeof value === "string" && value.length > 0 ? value : null;}-function toLinkAction(value: unknown): NoticeLinkAction | null {+function toAction(value: unknown): NoticeAction | null {if (!value || typeof value !== "object") return null;const action = value as Record<string, unknown>;const label = asString(action.label);+ if (!label) return null;+ if (action.kind === "resume") return { kind: "resume", label };+ if (action.kind === "reconnect") {+ const provider = asString(action.provider);+ return provider ? { kind: "reconnect", label, provider } : null;+ }const href = asString(action.href);// In-app paths only: the card never sends anyone off the platform.- if (action.kind !== "link" || !label || !href) return null;+ if (action.kind !== "link" || !href) return null;// A backslash reads as a slash to the browser, so /\host leaves too.if (!href.startsWith("/") || href.startsWith("//")) return null;if (href.includes("\\")) return null;@@ -31,8 +55,8 @@ function toLinkAction(value: unknown): NoticeLinkAction | null {}/**- * Read the note a skipped scheduled chat leaves in its chat, from the- * message's ``metadata`` bag. Returns null for any other message.+ * Read the note a skipped or failing scheduled chat leaves in its chat, from+ * the message's ``metadata`` bag. Returns null for any other message.*/export function getScheduledChatNotice(value: unknown,@@ -45,16 +69,20 @@ export function getScheduledChatNotice(const body = asString(meta.body);if (!headline || !body) return null;+ const scheduleKey = asString(meta.schedule_key);const actions = Array.isArray(meta.actions)? meta.actions- .map(toLinkAction)- .filter((action): action is NoticeLinkAction => action !== null)+ .map(toAction)+ .filter((action): action is NoticeAction => action !== null)+ // Resume needs to know which scheduled chat it resumes.+ .filter((action) => action.kind !== "resume" || scheduleKey !== null): [];return {reason: asString(meta.reason) ?? "usage",headline,body,+ scheduleKey,actions,};}diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/useScheduledChatNoticeCard.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/useScheduledChatNoticeCard.tsnew file mode 100644index 0000000000..03aafa0d92--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ScheduledChatNoticeCard/useScheduledChatNoticeCard.ts@@ -0,0 +1,86 @@+import { getGetV2GetSessionQueryKey } from "@/app/api/__generated__/endpoints/chat/chat";+import {+ getListPausedScheduledChatsQueryKey,+ useResumeScheduledChat,+} from "@/app/api/__generated__/endpoints/schedules/schedules";+import { toast } from "@/components/molecules/Toast/use-toast";+import { ApiError } from "@/lib/autogpt-server-api/helpers";+import { invalidateAllScheduleQueries } from "@/services/schedules/invalidate-schedules";+import * as Sentry from "@sentry/nextjs";+import { useQueryClient } from "@tanstack/react-query";+import { useRef, useState } from "react";++interface Args {+ scheduleKey: string | null;+ sessionID?: string | null;+}++export function useScheduledChatNoticeCard({ scheduleKey, sessionID }: Args) {+ const queryClient = useQueryClient();+ const [connectProvider, setConnectProvider] = useState<string | null>(null);+ const [resumed, setResumed] = useState(false);+ // A resume that follows a reconnect is quiet when there was nothing to+ // resume: a chat that was only warned, never paused, answers 404.+ const quietRef = useRef(false);++ function refresh() {+ queryClient.invalidateQueries({+ queryKey: getListPausedScheduledChatsQueryKey(),+ });+ void invalidateAllScheduleQueries(queryClient);+ if (sessionID) {+ queryClient.invalidateQueries({+ queryKey: getGetV2GetSessionQueryKey(sessionID),+ });+ }+ }++ const { mutate, isPending } = useResumeScheduledChat({+ mutation: {+ onSuccess: () => {+ setResumed(true);+ refresh();+ toast({+ title: "Resumed",+ description: "It runs again at its next scheduled time.",+ });+ },+ onError: (error) => {+ if (error instanceof ApiError && error.status === 404) {+ setResumed(true);+ refresh();+ if (!quietRef.current) toast({ title: "Already running" });+ return;+ }+ Sentry.captureException(error);+ toast({+ variant: "destructive",+ title: "Couldn't resume this scheduled chat",+ description: "Please try again.",+ });+ },+ },+ });++ function resume({ quiet = false } = {}) {+ if (!scheduleKey || isPending || resumed) return;+ quietRef.current = quiet;+ mutate({ scheduleKey });+ }++ function handleConnected() {+ setConnectProvider(null);+ // Reconnecting is the fix, so it resumes the chat too.+ resume({ quiet: true });+ }++ return {+ connectProvider,+ openConnect: setConnectProvider,+ closeConnect: () => setConnectProvider(null),+ handleConnected,+ resume: () => resume(),+ isResuming: isPending,+ resumed,+ };+}diff --git a/autogpt_platform/frontend/src/app/(platform)/library/followups/__tests__/main.test.tsx b/autogpt_platform/frontend/src/app/(platform)/library/followups/__tests__/main.test.tsxindex d6eeac4666..5a0bb78f5e 100644--- a/autogpt_platform/frontend/src/app/(platform)/library/followups/__tests__/main.test.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/library/followups/__tests__/main.test.tsx@@ -3,6 +3,8 @@ import {getDeleteV1DeleteExecutionScheduleMockHandler422,getGetV1ListExecutionSchedulesForAUserMockHandler,getListCopilotFollowupSchedulesMockHandler,+ getListPausedScheduledChatsMockHandler,+ getResumeScheduledChatMockHandler,} from "@/app/api/__generated__/endpoints/schedules/schedules.msw";import type { CopilotTurnJobInfo } from "@/app/api/__generated__/models/copilotTurnJobInfo";import type { GraphExecutionJobInfo } from "@/app/api/__generated__/models/graphExecutionJobInfo";@@ -12,6 +14,7 @@ import {fireEvent,render,screen,+ waitFor,within,} from "@/tests/integrations/test-utils";import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";@@ -94,6 +97,8 @@ describe("FollowupsPage", () => {beforeEach(() => {toastMock.mockClear();pushMock.mockClear();+ // Nothing paused unless a test says so.+ server.use(getListPausedScheduledChatsMockHandler([]));});afterEach(() => {@@ -494,4 +499,190 @@ describe("FollowupsPage", () => {expect(screen.queryByTestId("followups-list")).toBeNull();expect(screen.queryByTestId("schedules-partial-error")).toBeNull();});++ test("a paused list that fails to load is an error, not the empty state", async () => {+ server.use(+ getListCopilotFollowupSchedulesMockHandler([]),+ defaultGraphSchedulesHandler(),+ http.get(+ "http://localhost:3000/api/proxy/api/schedules/followups/paused",+ () => HttpResponse.error(),+ ),+ );++ render(<FollowupsPage />);++ expect(await screen.findByText(/scheduled items/i)).toBeDefined();+ expect(screen.queryByTestId("followups-empty")).toBeNull();+ });++ test("a paused chat stays listed when the schedules fail to load", async () => {+ server.use(+ http.get("http://localhost:3000/api/proxy/api/schedules/followups", () =>+ HttpResponse.error(),+ ),+ http.get("http://localhost:3000/api/proxy/api/schedules", () =>+ HttpResponse.error(),+ ),+ getListPausedScheduledChatsMockHandler([+ {+ schedule_key: "routine-1",+ title: "Discord monitor",+ session_id: "session-1",+ reason: "chat_link",+ provider: "discord",+ summary: "Paused: can't post to Discord",+ paused_at: new Date("2026-10-11T12:00:00Z"),+ },+ ]),+ );++ render(<FollowupsPage />);++ expect(await screen.findByTestId("paused-chat-row")).toBeDefined();+ expect(await screen.findByTestId("schedules-partial-error")).toBeDefined();+ });++ test("lists a paused scheduled chat with why, and Resume resumes it", async () => {+ let resumedKey: string | null = null;+ server.use(+ getListCopilotFollowupSchedulesMockHandler([]),+ defaultGraphSchedulesHandler(),+ getListPausedScheduledChatsMockHandler([+ {+ schedule_key: "routine-1",+ title: "Discord monitor",+ session_id: "session-1",+ reason: "chat_link",+ provider: "discord",+ summary: "Paused: can't post to Discord",+ paused_at: new Date("2026-10-11T12:00:00Z"),+ },+ ]),+ getResumeScheduledChatMockHandler(({ params }) => {+ resumedKey = String(params.scheduleKey);+ return { resumed: true };+ }),+ );++ render(<FollowupsPage />);++ const row = await screen.findByTestId("paused-chat-row");+ expect(within(row).getByText("Discord monitor")).toBeDefined();+ expect(+ within(row).getByText("Paused: can't post to Discord"),+ ).toBeDefined();+ // A paused chat is still something scheduled: no empty state.+ expect(screen.queryByTestId("followups-empty")).toBeNull();++ fireEvent.click(+ within(row).getByRole("button", { name: "Resume Discord monitor" }),+ );+ await waitFor(() => expect(resumedKey).toBe("routine-1"));+ });++ test("a paused follow-up can be deleted from its row", async () => {+ let deletedID: string | null = null;+ server.use(+ getListCopilotFollowupSchedulesMockHandler([]),+ defaultGraphSchedulesHandler(),+ getListPausedScheduledChatsMockHandler([+ {+ schedule_key: "sched-1",+ title: "Discord monitor",+ session_id: "session-1",+ reason: "connection",+ provider: "discord",+ summary: "Paused: can't use Discord",+ paused_at: new Date("2026-10-11T12:00:00Z"),+ routine: false,+ },+ ]),+ getDeleteV1DeleteExecutionScheduleMockHandler(({ params }) => {+ deletedID = String(params.scheduleId);+ return { id: deletedID };+ }),+ );++ render(<FollowupsPage />);++ const row = await screen.findByTestId("paused-chat-row");+ fireEvent.click(+ within(row).getByRole("button", { name: "Delete Discord monitor" }),+ );+ fireEvent.click(await screen.findByRole("button", { name: "Yes, delete" }));++ await waitFor(() => expect(deletedID).toBe("sched-1"));+ });++ test("a paused follow-up whose schedule was gone already reads as deleted", async () => {+ server.use(+ getListCopilotFollowupSchedulesMockHandler([]),+ defaultGraphSchedulesHandler(),+ getListPausedScheduledChatsMockHandler([+ {+ schedule_key: "sched-1",+ title: "Discord monitor",+ session_id: "session-1",+ reason: "connection",+ provider: "discord",+ summary: "Paused: can't use Discord",+ paused_at: new Date("2026-10-11T12:00:00Z"),+ routine: false,+ },+ ]),+ http.delete("*/api/schedules/:scheduleId", () =>+ HttpResponse.json(+ { detail: "Schedule #sched-1 not found" },+ { status: 404 },+ ),+ ),+ );++ render(<FollowupsPage />);++ const row = await screen.findByTestId("paused-chat-row");+ fireEvent.click(+ within(row).getByRole("button", { name: "Delete Discord monitor" }),+ );+ fireEvent.click(await screen.findByRole("button", { name: "Yes, delete" }));++ await waitFor(() =>+ expect(toastMock).toHaveBeenCalledWith(+ expect.objectContaining({ title: "Scheduled chat deleted" }),+ ),+ );+ expect(toastMock).not.toHaveBeenCalledWith(+ expect.objectContaining({ variant: "destructive" }),+ );+ });++ test("a paused routine offers no Delete: it goes with its expert", async () => {+ server.use(+ getListCopilotFollowupSchedulesMockHandler([]),+ defaultGraphSchedulesHandler(),+ getListPausedScheduledChatsMockHandler([+ {+ schedule_key: "routine-1",+ title: "Morning sweep",+ session_id: "session-1",+ reason: "connection",+ provider: "discord",+ summary: "Paused: can't use Discord",+ paused_at: new Date("2026-10-11T12:00:00Z"),+ routine: true,+ },+ ]),+ );++ render(<FollowupsPage />);++ const row = await screen.findByTestId("paused-chat-row");+ expect(+ within(row).getByRole("button", { name: "Resume Morning sweep" }),+ ).toBeDefined();+ expect(+ within(row).queryByRole("button", { name: "Delete Morning sweep" }),+ ).toBeNull();+ });});diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.jsonindex a057937152..5d60f30824 100644--- a/autogpt_platform/frontend/src/app/api/openapi.json+++ b/autogpt_platform/frontend/src/app/api/openapi.json@@ -14695,6 +14695,78 @@"security": [{ "HTTPBearerJWT": [] }]}},+ "/api/schedules/followups/paused": {+ "get": {+ "tags": ["v1", "schedules"],+ "summary": "List scheduled chats paused after failing the same way",+ "description": "A paused job drops out of the follow-ups list above, so the chats a\nfailure streak paused are listed here, each once however many fire times\nit has, with why and what Resume takes.",+ "operationId": "listPausedScheduledChats",+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "items": {+ "$ref": "#/components/schemas/PausedScheduledChat"+ },+ "type": "array",+ "title": "Response Listpausedscheduledchats"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/schedules/followups/{schedule_key}/resume": {+ "post": {+ "tags": ["v1", "schedules"],+ "summary": "Resume a scheduled chat paused after failing the same way",+ "operationId": "resumeScheduledChat",+ "security": [{ "HTTPBearerJWT": [] }],+ "parameters": [+ {+ "name": "schedule_key",+ "in": "path",+ "required": true,+ "schema": {+ "type": "string",+ "description": "The routine's id, or the follow-up's schedule id",+ "title": "Schedule Key"+ },+ "description": "The routine's id, or the follow-up's schedule id"+ }+ ],+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ResumeScheduledChatResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ }+ }+ },"/api/schedules/{schedule_id}": {"delete": {"tags": ["v1", "schedules"],@@ -29326,6 +29398,40 @@"required": ["total_items", "total_pages", "current_page", "page_size"],"title": "Pagination"},+ "PausedScheduledChat": {+ "properties": {+ "schedule_key": { "type": "string", "title": "Schedule Key" },+ "title": { "type": "string", "title": "Title" },+ "session_id": {+ "anyOf": [{ "type": "string" }, { "type": "null" }],+ "title": "Session Id"+ },+ "reason": { "type": "string", "title": "Reason" },+ "provider": {+ "anyOf": [{ "type": "string" }, { "type": "null" }],+ "title": "Provider"+ },+ "summary": { "type": "string", "title": "Summary" },+ "paused_at": {+ "type": "string",+ "format": "date-time",+ "title": "Paused At"+ },+ "routine": { "type": "boolean", "title": "Routine", "default": false }+ },+ "type": "object",+ "required": [+ "schedule_key",+ "title",+ "session_id",+ "reason",+ "provider",+ "summary",+ "paused_at"+ ],+ "title": "PausedScheduledChat",+ "description": "A scheduled chat its failure streak paused, as the lists show it."+ },"PeekPendingMessagesResponse": {"properties": {"messages": {@@ -31131,6 +31237,12 @@"required": ["version_id"],"title": "RestoreVersionRequest"},+ "ResumeScheduledChatResponse": {+ "properties": { "resumed": { "type": "boolean", "title": "Resumed" } },+ "type": "object",+ "required": ["resumed"],+ "title": "ResumeScheduledChatResponse"+ },"ReviewItem": {"properties": {"node_exec_id": {diff --git a/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/SchedulesPanel.tsx b/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/SchedulesPanel.tsxindex e4f632fac8..73a6958f33 100644--- a/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/SchedulesPanel.tsx+++ b/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/SchedulesPanel.tsx@@ -8,6 +8,7 @@ import { NEW_SCHEDULED_TASK_PROMPT } from "../guidedPrompts";import { EmptyFollowups } from "./components/EmptyFollowups/EmptyFollowups";import { FollowupListItem } from "./components/FollowupListItem/FollowupListItem";import { GraphScheduleListItem } from "./components/GraphScheduleListItem/GraphScheduleListItem";+import { PausedChatListItem } from "./components/PausedChatListItem/PausedChatListItem";import { useSchedulesPanel } from "./useSchedulesPanel";import { PlusSignIcon } from "@hugeicons/core-free-icons";import { Icon } from "@/components/atoms/Icon/Icon";@@ -18,7 +19,8 @@ interface Props {}export function SchedulesPanel({ onGuidedPrompt, withHeading = true }: Props) {- const { schedules, isLoading, error, partialError } = useSchedulesPanel();+ const { schedules, paused, isLoading, error, partialError } =+ useSchedulesPanel();return (<section className="space-y-6">@@ -55,6 +57,20 @@ export function SchedulesPanel({ onGuidedPrompt, withHeading = true }: Props) {</Text>)}+ {paused.length > 0 && (+ <ul+ className="flex flex-col gap-3"+ data-testid="paused-chats-list"+ aria-label="Paused scheduled chats"+ >+ {paused.map((chat) => (+ <li key={`paused:${chat.schedule_key}`}>+ <PausedChatListItem paused={chat} />+ </li>+ ))}+ </ul>+ )}+{error ? (<ErrorCardresponseError={{@@ -73,7 +89,7 @@ export function SchedulesPanel({ onGuidedPrompt, withHeading = true }: Props) {<LoadingSpinner /></div>) : schedules.length === 0 ? (- <EmptyFollowups />+ paused.length === 0 && <EmptyFollowups />) : (<ulclassName="flex flex-col gap-3"diff --git a/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/components/PausedChatListItem/PausedChatListItem.tsx b/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/components/PausedChatListItem/PausedChatListItem.tsxnew file mode 100644index 0000000000..ce4c662ec8--- /dev/null+++ b/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/components/PausedChatListItem/PausedChatListItem.tsx@@ -0,0 +1,119 @@+"use client";++import type { PausedScheduledChat } from "@/app/api/__generated__/models/pausedScheduledChat";+import { Button } from "@/components/atoms/Button/Button";+import { Icon } from "@/components/atoms/Icon/Icon";+import { Text } from "@/components/atoms/Text/Text";+import { Dialog } from "@/components/molecules/Dialog/Dialog";+import { Delete02Icon, PauseIcon, PlayIcon } from "@hugeicons/core-free-icons";+import Link from "next/link";+import { usePausedChatListItem } from "./usePausedChatListItem";++interface Props {+ paused: PausedScheduledChat;+}++export function PausedChatListItem({ paused }: Props) {+ const {+ sessionHref,+ handleResume,+ isResuming,+ canDelete,+ isDeleteOpen,+ setIsDeleteOpen,+ handleDelete,+ isDeleting,+ } = usePausedChatListItem({ paused });++ const detail = (+ <>+ <div className="flex h-9 w-9 flex-shrink-0 items-center justify-center rounded-large border border-slate-50 bg-amber-50">+ <Icon icon={PauseIcon} size={18} className="text-amber-700" />+ </div>+ <div className="flex min-w-0 flex-col gap-1">+ <Text+ variant="body-medium"+ className="block w-full truncate text-ellipsis"+ >+ {paused.title}+ </Text>+ <Text variant="small" className="!text-amber-700">+ {paused.summary}+ </Text>+ </div>+ </>+ );++ return (+ <div+ className="flex w-full flex-col gap-3 rounded-large border border-amber-200 bg-white p-4 sm:flex-row sm:items-center sm:justify-between"+ data-testid="paused-chat-row"+ data-schedule-key={paused.schedule_key}+ >+ {sessionHref ? (+ <Link+ href={sessionHref}+ className="flex min-w-0 flex-1 items-start gap-3 hover:opacity-80"+ >+ {detail}+ </Link>+ ) : (+ <div className="flex min-w-0 flex-1 items-start gap-3">{detail}</div>+ )}+ <div className="flex flex-shrink-0 items-center gap-2">+ <Button+ variant="secondary"+ size="small"+ loading={isResuming}+ onClick={handleResume}+ aria-label={`Resume ${paused.title}`}+ >+ <Icon icon={PlayIcon} className="mr-1 h-4 w-4" />+ Resume+ </Button>+ {canDelete ? (+ <Button+ variant="secondary"+ size="small"+ onClick={() => setIsDeleteOpen(true)}+ aria-label={`Delete ${paused.title}`}+ >+ <Icon icon={Delete02Icon} className="mr-1 h-4 w-4" />+ Delete+ </Button>+ ) : null}+ </div>++ <Dialog+ controlled={{ isOpen: isDeleteOpen, set: setIsDeleteOpen }}+ styling={{ maxWidth: "32rem" }}+ title="Delete scheduled chat"+ >+ <Dialog.Content>+ <div className="flex flex-col gap-4">+ <Text variant="large">+ Delete “{paused.title}”? It won't run again. You can recreate+ it from chat if needed.+ </Text>+ <Dialog.Footer>+ <Button+ variant="secondary"+ disabled={isDeleting}+ onClick={() => setIsDeleteOpen(false)}+ >+ Keep it+ </Button>+ <Button+ variant="destructive"+ onClick={handleDelete}+ loading={isDeleting}+ >+ Yes, delete+ </Button>+ </Dialog.Footer>+ </div>+ </Dialog.Content>+ </Dialog>+ </div>+ );+}diff --git a/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/components/PausedChatListItem/usePausedChatListItem.ts b/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/components/PausedChatListItem/usePausedChatListItem.tsnew file mode 100644index 0000000000..a5a4af4ca3--- /dev/null+++ b/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/components/PausedChatListItem/usePausedChatListItem.ts@@ -0,0 +1,104 @@+import {+ getListPausedScheduledChatsQueryKey,+ useDeleteV1DeleteExecutionSchedule,+ useResumeScheduledChat,+} from "@/app/api/__generated__/endpoints/schedules/schedules";+import type { PausedScheduledChat } from "@/app/api/__generated__/models/pausedScheduledChat";+import { useToast } from "@/components/molecules/Toast/use-toast";+import { ApiError } from "@/lib/autogpt-server-api/helpers";+import { invalidateAllScheduleQueries } from "@/services/schedules/invalidate-schedules";+import * as Sentry from "@sentry/nextjs";+import { useQueryClient } from "@tanstack/react-query";+import { useState } from "react";++interface Args {+ paused: PausedScheduledChat;+}++export function usePausedChatListItem({ paused }: Args) {+ const queryClient = useQueryClient();+ const { toast } = useToast();+ const [isDeleteOpen, setIsDeleteOpen] = useState(false);++ function refresh() {+ queryClient.invalidateQueries({+ queryKey: getListPausedScheduledChatsQueryKey(),+ });+ return invalidateAllScheduleQueries(queryClient);+ }++ const { mutate, isPending } = useResumeScheduledChat({+ mutation: {+ onSuccess: () => {+ void refresh();+ toast({+ title: "Resumed",+ description: `“${paused.title}” runs again at its next scheduled time.`,+ });+ },+ onError: (error) => {+ void refresh();+ // Resumed elsewhere already: the refresh drops the row.+ if (error instanceof ApiError && error.status === 404) return;+ Sentry.captureException(error);+ toast({+ variant: "destructive",+ title: "Couldn't resume this scheduled chat",+ description: "Please try again.",+ });+ },+ },+ });++ function handleDeleted() {+ setIsDeleteOpen(false);+ void refresh();+ toast({ title: "Scheduled chat deleted" });+ }++ const { mutate: deleteSchedule, isPending: isDeleting } =+ useDeleteV1DeleteExecutionSchedule({+ mutation: {+ onSuccess: handleDeleted,+ onError: (error) => {+ // Its schedule was gone already; the delete still cleared the row.+ if (error instanceof ApiError && error.status === 404) {+ handleDeleted();+ return;+ }+ void refresh();+ Sentry.captureException(error);+ toast({+ variant: "destructive",+ title: "Couldn't delete this scheduled chat",+ description: "Please try again.",+ });+ },+ },+ });++ const sessionHref = paused.session_id+ ? `/copilot?sessionId=${encodeURIComponent(paused.session_id)}`+ : null;++ function handleResume() {+ if (isPending) return;+ mutate({ scheduleKey: paused.schedule_key });+ }++ function handleDelete() {+ if (isDeleting) return;+ deleteSchedule({ scheduleId: paused.schedule_key });+ }++ return {+ sessionHref,+ handleResume,+ isResuming: isPending,+ canDelete: !paused.routine,+ isDeleteOpen,+ setIsDeleteOpen,+ handleDelete,+ isDeleting,+ };+}diff --git a/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/useSchedulesPanel.ts b/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/useSchedulesPanel.tsindex fa3e0a2ffe..e1a2db6f50 100644--- a/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/useSchedulesPanel.ts+++ b/autogpt_platform/frontend/src/components/contextual/SchedulesPanel/useSchedulesPanel.ts@@ -1,6 +1,7 @@import {useGetV1ListExecutionSchedulesForAUser,useListCopilotFollowupSchedules,+ useListPausedScheduledChats,} from "@/app/api/__generated__/endpoints/schedules/schedules";import type { CopilotTurnJobInfo } from "@/app/api/__generated__/models/copilotTurnJobInfo";import type { GraphExecutionJobInfo } from "@/app/api/__generated__/models/graphExecutionJobInfo";@@ -29,6 +30,11 @@ export function useSchedulesPanel() {const graphQuery = useGetV1ListExecutionSchedulesForAUser({query: { select: (res) => okData(res) ?? [] },});+ // Scheduled chats paused after failing the same way: their jobs drop out+ // of the followups list, so they are listed apart, with Resume.+ const pausedQuery = useListPausedScheduledChats({+ query: { select: (res) => okData(res) ?? [] },+ });const copilotItems: ScheduleItem[] = (copilotQuery.data ?? []).map((item) => ({ kind: "copilot_turn" as const, item }),@@ -46,8 +52,12 @@ export function useSchedulesPanel() {(a, b) => nextRunMs(a.item) - nextRunMs(b.item),);- const isLoading = copilotQuery.isLoading || graphQuery.isLoading;- const fetchError = copilotQuery.error ?? graphQuery.error;+ const paused = pausedQuery.data ?? [];+ const isLoading =+ copilotQuery.isLoading || graphQuery.isLoading || pausedQuery.isLoading;+ const fetchError =+ copilotQuery.error ?? graphQuery.error ?? pausedQuery.error;+ const hasContent = schedules.length > 0 || paused.length > 0;return {// Backwards-compat alias — ``followups`` was the only-copilot@@ -55,11 +65,12 @@ export function useSchedulesPanel() {// reference it.followups: copilotQuery.data ?? [],schedules,+ paused,isLoading,- // The two sources are independent — only hard-fail once both have- // settled and there is nothing to show; a partial failure keeps the- // loaded list visible and surfaces a non-blocking warning instead.- error: !isLoading && schedules.length === 0 ? fetchError : null,- partialError: !isLoading && schedules.length > 0 ? fetchError : null,+ // The sources are independent — only hard-fail once all have settled+ // and there is nothing to show; a partial failure keeps the loaded+ // lists visible and surfaces a non-blocking warning instead.+ error: !isLoading && !hasContent ? fetchError : null,+ partialError: !isLoading && hasContent ? fetchError : null,};}