AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(admin): Add system diagnostics and execution management dashboard (#11235)
Nicholas Tindle38c2844b83ce
diff --git a/autogpt_platform/backend/backend/api/features/admin/diagnostics_admin_routes.py b/autogpt_platform/backend/backend/api/features/admin/diagnostics_admin_routes.pynew file mode 100644index 0000000000..4cb8ff0729--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/admin/diagnostics_admin_routes.py@@ -0,0 +1,932 @@+import asyncio+import logging+from typing import List++from autogpt_libs.auth import requires_admin_user+from autogpt_libs.auth.models import User as AuthUser+from fastapi import APIRouter, HTTPException, Security+from prisma.enums import AgentExecutionStatus+from pydantic import BaseModel++from backend.api.features.admin.model import (+ AgentDiagnosticsResponse,+ ExecutionDiagnosticsResponse,+)+from backend.data.diagnostics import (+ FailedExecutionDetail,+ OrphanedScheduleDetail,+ RunningExecutionDetail,+ ScheduleDetail,+ ScheduleHealthMetrics,+ cleanup_all_stuck_queued_executions,+ cleanup_orphaned_executions_bulk,+ cleanup_orphaned_schedules_bulk,+ get_agent_diagnostics,+ get_all_orphaned_execution_ids,+ get_all_schedules_details,+ get_all_stuck_queued_execution_ids,+ get_execution_diagnostics,+ get_failed_executions_count,+ get_failed_executions_details,+ get_invalid_executions_details,+ get_long_running_executions_details,+ get_orphaned_executions_details,+ get_orphaned_schedules_details,+ get_running_executions_details,+ get_schedule_health_metrics,+ get_stuck_queued_executions_details,+ stop_all_long_running_executions,+)+from backend.data.execution import get_graph_executions+from backend.executor.utils import add_graph_execution, stop_graph_execution++logger = logging.getLogger(__name__)++router = APIRouter(+ prefix="/admin",+ tags=["diagnostics", "admin"],+ dependencies=[Security(requires_admin_user)],+)+++class RunningExecutionsListResponse(BaseModel):+ """Response model for list of running executions"""++ executions: List[RunningExecutionDetail]+ total: int+++class FailedExecutionsListResponse(BaseModel):+ """Response model for list of failed executions"""++ executions: List[FailedExecutionDetail]+ total: int+++class StopExecutionRequest(BaseModel):+ """Request model for stopping a single execution"""++ execution_id: str+++class StopExecutionsRequest(BaseModel):+ """Request model for stopping multiple executions"""++ execution_ids: List[str]+++class StopExecutionResponse(BaseModel):+ """Response model for stop execution operations"""++ success: bool+ stopped_count: int = 0+ message: str+++class RequeueExecutionResponse(BaseModel):+ """Response model for requeue execution operations"""++ success: bool+ requeued_count: int = 0+ message: str+++ "/diagnostics/executions",+ response_model=ExecutionDiagnosticsResponse,+ summary="Get Execution Diagnostics",+)+async def get_execution_diagnostics_endpoint():+ """+ Get comprehensive diagnostic information about execution status.++ Returns all execution metrics including:+ - Current state (running, queued)+ - Orphaned executions (>24h old, likely not in executor)+ - Failure metrics (1h, 24h, rate)+ - Long-running detection (stuck >1h, >24h)+ - Stuck queued detection+ - Throughput metrics (completions/hour)+ - RabbitMQ queue depths+ """+ logger.info("Getting execution diagnostics")++ diagnostics = await get_execution_diagnostics()++ response = ExecutionDiagnosticsResponse(+ running_executions=diagnostics.running_count,+ queued_executions_db=diagnostics.queued_db_count,+ queued_executions_rabbitmq=diagnostics.rabbitmq_queue_depth,+ cancel_queue_depth=diagnostics.cancel_queue_depth,+ orphaned_running=diagnostics.orphaned_running,+ orphaned_queued=diagnostics.orphaned_queued,+ failed_count_1h=diagnostics.failed_count_1h,+ failed_count_24h=diagnostics.failed_count_24h,+ failure_rate_24h=diagnostics.failure_rate_24h,+ stuck_running_24h=diagnostics.stuck_running_24h,+ stuck_running_1h=diagnostics.stuck_running_1h,+ oldest_running_hours=diagnostics.oldest_running_hours,+ stuck_queued_1h=diagnostics.stuck_queued_1h,+ queued_never_started=diagnostics.queued_never_started,+ invalid_queued_with_start=diagnostics.invalid_queued_with_start,+ invalid_running_without_start=diagnostics.invalid_running_without_start,+ completed_1h=diagnostics.completed_1h,+ completed_24h=diagnostics.completed_24h,+ throughput_per_hour=diagnostics.throughput_per_hour,+ timestamp=diagnostics.timestamp,+ )++ logger.info(+ f"Execution diagnostics: running={diagnostics.running_count}, "+ f"queued_db={diagnostics.queued_db_count}, "+ f"orphaned={diagnostics.orphaned_running + diagnostics.orphaned_queued}, "+ f"failed_24h={diagnostics.failed_count_24h}"+ )++ return response+++ "/diagnostics/agents",+ response_model=AgentDiagnosticsResponse,+ summary="Get Agent Diagnostics",+)+async def get_agent_diagnostics_endpoint():+ """+ Get diagnostic information about agents.++ Returns:+ - agents_with_active_executions: Number of unique agents with running/queued executions+ - timestamp: Current timestamp+ """+ logger.info("Getting agent diagnostics")++ diagnostics = await get_agent_diagnostics()++ response = AgentDiagnosticsResponse(+ agents_with_active_executions=diagnostics.agents_with_active_executions,+ timestamp=diagnostics.timestamp,+ )++ logger.info(+ f"Agent diagnostics: with_active_executions={diagnostics.agents_with_active_executions}"+ )++ return response+++ "/diagnostics/executions/running",+ response_model=RunningExecutionsListResponse,+ summary="List Running Executions",+)+async def list_running_executions(+ limit: int = 100,+ offset: int = 0,+):+ """+ Get detailed list of running and queued executions (recent, likely active).++ Args:+ limit: Maximum number of executions to return (default 100)+ offset: Number of executions to skip (default 0)++ Returns:+ List of running executions with details+ """+ logger.info(f"Listing running executions (limit={limit}, offset={offset})")++ executions = await get_running_executions_details(limit=limit, offset=offset)++ # Get total count for pagination+ diagnostics = await get_execution_diagnostics()+ total = diagnostics.running_count + diagnostics.queued_db_count++ return RunningExecutionsListResponse(executions=executions, total=total)+++ "/diagnostics/executions/orphaned",+ response_model=RunningExecutionsListResponse,+ summary="List Orphaned Executions",+)+async def list_orphaned_executions(+ limit: int = 100,+ offset: int = 0,+):+ """+ Get detailed list of orphaned executions (>24h old, likely not in executor).++ Args:+ limit: Maximum number of executions to return (default 100)+ offset: Number of executions to skip (default 0)++ Returns:+ List of orphaned executions with details+ """+ logger.info(f"Listing orphaned executions (limit={limit}, offset={offset})")++ executions = await get_orphaned_executions_details(limit=limit, offset=offset)++ # Get total count for pagination+ diagnostics = await get_execution_diagnostics()+ total = diagnostics.orphaned_running + diagnostics.orphaned_queued++ return RunningExecutionsListResponse(executions=executions, total=total)+++ "/diagnostics/executions/failed",+ response_model=FailedExecutionsListResponse,+ summary="List Failed Executions",+)+async def list_failed_executions(+ limit: int = 100,+ offset: int = 0,+ hours: int = 24,+):+ """+ Get detailed list of failed executions.++ Args:+ limit: Maximum number of executions to return (default 100)+ offset: Number of executions to skip (default 0)+ hours: Number of hours to look back (default 24)++ Returns:+ List of failed executions with error details+ """+ logger.info(+ f"Listing failed executions (limit={limit}, offset={offset}, hours={hours})"+ )++ executions = await get_failed_executions_details(+ limit=limit, offset=offset, hours=hours+ )++ # Get total count for pagination+ # Always count actual total for given hours parameter+ total = await get_failed_executions_count(hours=hours)++ return FailedExecutionsListResponse(executions=executions, total=total)+++ "/diagnostics/executions/long-running",+ response_model=RunningExecutionsListResponse,+ summary="List Long-Running Executions",+)+async def list_long_running_executions(+ limit: int = 100,+ offset: int = 0,+):+ """+ Get detailed list of long-running executions (RUNNING status >24h).++ Args:+ limit: Maximum number of executions to return (default 100)+ offset: Number of executions to skip (default 0)++ Returns:+ List of long-running executions with details+ """+ logger.info(f"Listing long-running executions (limit={limit}, offset={offset})")++ executions = await get_long_running_executions_details(limit=limit, offset=offset)++ # Get total count for pagination+ diagnostics = await get_execution_diagnostics()+ total = diagnostics.stuck_running_24h++ return RunningExecutionsListResponse(executions=executions, total=total)+++ "/diagnostics/executions/stuck-queued",+ response_model=RunningExecutionsListResponse,+ summary="List Stuck Queued Executions",+)+async def list_stuck_queued_executions(+ limit: int = 100,+ offset: int = 0,+):+ """+ Get detailed list of stuck queued executions (QUEUED >1h, never started).++ Args:+ limit: Maximum number of executions to return (default 100)+ offset: Number of executions to skip (default 0)++ Returns:+ List of stuck queued executions with details+ """+ logger.info(f"Listing stuck queued executions (limit={limit}, offset={offset})")++ executions = await get_stuck_queued_executions_details(limit=limit, offset=offset)++ # Get total count for pagination+ diagnostics = await get_execution_diagnostics()+ total = diagnostics.stuck_queued_1h++ return RunningExecutionsListResponse(executions=executions, total=total)+++ "/diagnostics/executions/invalid",+ response_model=RunningExecutionsListResponse,+ summary="List Invalid Executions",+)+async def list_invalid_executions(+ limit: int = 100,+ offset: int = 0,+):+ """+ Get detailed list of executions in invalid states (READ-ONLY).++ Invalid states indicate data corruption and require manual investigation:+ - QUEUED but has startedAt (impossible - can't start while queued)+ - RUNNING but no startedAt (impossible - can't run without starting)++ ⚠️ NO BULK ACTIONS PROVIDED - These need case-by-case investigation.++ Each invalid execution likely has a different root cause (crashes, race conditions,+ DB corruption). Investigate the execution history and logs to determine appropriate+ action (manual cleanup, status fix, or leave as-is if system recovered).++ Args:+ limit: Maximum number of executions to return (default 100)+ offset: Number of executions to skip (default 0)++ Returns:+ List of invalid state executions with details+ """+ logger.info(f"Listing invalid state executions (limit={limit}, offset={offset})")++ executions = await get_invalid_executions_details(limit=limit, offset=offset)++ # Get total count for pagination+ diagnostics = await get_execution_diagnostics()+ total = (+ diagnostics.invalid_queued_with_start+ + diagnostics.invalid_running_without_start+ )++ return RunningExecutionsListResponse(executions=executions, total=total)+++ "/diagnostics/executions/requeue",+ response_model=RequeueExecutionResponse,+ summary="Requeue Stuck Execution",+)+async def requeue_single_execution(+ request: StopExecutionRequest, # Reuse same request model (has execution_id)+ user: AuthUser = Security(requires_admin_user),+):+ """+ Requeue a stuck QUEUED execution (admin only).++ Uses add_graph_execution with existing graph_exec_id to requeue.++ ⚠️ WARNING: Only use for stuck executions. This will re-execute and may cost credits.++ Args:+ request: Contains execution_id to requeue++ Returns:+ Success status and message+ """+ logger.info(f"Admin {user.user_id} requeueing execution {request.execution_id}")++ # Get the execution (validation - must be QUEUED)+ executions = await get_graph_executions(+ graph_exec_id=request.execution_id,+ statuses=[AgentExecutionStatus.QUEUED],+ )++ if not executions:+ raise HTTPException(+ status_code=404,+ detail="Execution not found or not in QUEUED status",+ )++ execution = executions[0]++ # Use add_graph_execution in requeue mode+ await add_graph_execution(+ graph_id=execution.graph_id,+ user_id=execution.user_id,+ graph_version=execution.graph_version,+ graph_exec_id=request.execution_id, # Requeue existing execution+ )++ return RequeueExecutionResponse(+ success=True,+ requeued_count=1,+ message="Execution requeued successfully",+ )+++ "/diagnostics/executions/requeue-bulk",+ response_model=RequeueExecutionResponse,+ summary="Requeue Multiple Stuck Executions",+)+async def requeue_multiple_executions(+ request: StopExecutionsRequest, # Reuse same request model (has execution_ids)+ user: AuthUser = Security(requires_admin_user),+):+ """+ Requeue multiple stuck QUEUED executions (admin only).++ Uses add_graph_execution with existing graph_exec_id to requeue.++ ⚠️ WARNING: Only use for stuck executions. This will re-execute and may cost credits.++ Args:+ request: Contains list of execution_ids to requeue++ Returns:+ Number of executions requeued and success message+ """+ logger.info(+ f"Admin {user.user_id} requeueing {len(request.execution_ids)} executions"+ )++ # Get executions by ID list (must be QUEUED)+ executions = await get_graph_executions(+ execution_ids=request.execution_ids,+ statuses=[AgentExecutionStatus.QUEUED],+ )++ if not executions:+ return RequeueExecutionResponse(+ success=False,+ requeued_count=0,+ message="No QUEUED executions found to requeue",+ )++ # Requeue all executions in parallel using add_graph_execution+ async def requeue_one(exec) -> bool:+ try:+ await add_graph_execution(+ graph_id=exec.graph_id,+ user_id=exec.user_id,+ graph_version=exec.graph_version,+ graph_exec_id=exec.id, # Requeue existing+ )+ return True+ except Exception as e:+ logger.error(f"Failed to requeue {exec.id}: {e}")+ return False++ results = await asyncio.gather(+ *[requeue_one(exec) for exec in executions], return_exceptions=False+ )++ requeued_count = sum(1 for success in results if success)++ return RequeueExecutionResponse(+ success=requeued_count > 0,+ requeued_count=requeued_count,+ message=f"Requeued {requeued_count} of {len(request.execution_ids)} executions",+ )+++ "/diagnostics/executions/stop",+ response_model=StopExecutionResponse,+ summary="Stop Single Execution",+)+async def stop_single_execution(+ request: StopExecutionRequest,+ user: AuthUser = Security(requires_admin_user),+):+ """+ Stop a single execution (admin only).++ Uses robust stop_graph_execution which cascades to children and waits for termination.++ Args:+ request: Contains execution_id to stop++ Returns:+ Success status and message+ """+ logger.info(f"Admin {user.user_id} stopping execution {request.execution_id}")++ # Get the execution to find its owner user_id (required by stop_graph_execution)+ executions = await get_graph_executions(+ graph_exec_id=request.execution_id,+ )++ if not executions:+ raise HTTPException(status_code=404, detail="Execution not found")++ execution = executions[0]++ # Use robust stop_graph_execution (cascades to children, waits for termination)+ await stop_graph_execution(+ user_id=execution.user_id,+ graph_exec_id=request.execution_id,+ wait_timeout=15.0,+ cascade=True,+ )++ return StopExecutionResponse(+ success=True,+ stopped_count=1,+ message="Execution stopped successfully",+ )+++ "/diagnostics/executions/stop-bulk",+ response_model=StopExecutionResponse,+ summary="Stop Multiple Executions",+)+async def stop_multiple_executions(+ request: StopExecutionsRequest,+ user: AuthUser = Security(requires_admin_user),+):+ """+ Stop multiple active executions (admin only).++ Uses robust stop_graph_execution which cascades to children and waits for termination.++ Args:+ request: Contains list of execution_ids to stop++ Returns:+ Number of executions stopped and success message+ """++ logger.info(+ f"Admin {user.user_id} stopping {len(request.execution_ids)} executions"+ )++ # Get executions by ID list+ executions = await get_graph_executions(+ execution_ids=request.execution_ids,+ )++ if not executions:+ return StopExecutionResponse(+ success=False,+ stopped_count=0,+ message="No executions found",+ )++ # Stop all executions in parallel using robust stop_graph_execution+ async def stop_one(exec) -> bool:+ try:+ await stop_graph_execution(+ user_id=exec.user_id,+ graph_exec_id=exec.id,+ wait_timeout=15.0,+ cascade=True,+ )+ return True+ except Exception as e:+ logger.error(f"Failed to stop execution {exec.id}: {e}")+ return False++ results = await asyncio.gather(+ *[stop_one(exec) for exec in executions], return_exceptions=False+ )++ stopped_count = sum(1 for success in results if success)++ return StopExecutionResponse(+ success=stopped_count > 0,+ stopped_count=stopped_count,+ message=f"Stopped {stopped_count} of {len(request.execution_ids)} executions",+ )+++ "/diagnostics/executions/cleanup-orphaned",+ response_model=StopExecutionResponse,+ summary="Cleanup Orphaned Executions",+)+async def cleanup_orphaned_executions(+ request: StopExecutionsRequest,+ user: AuthUser = Security(requires_admin_user),+):+ """+ Cleanup orphaned executions by directly updating DB status (admin only).+ For executions in DB but not actually running in executor (old/stale records).++ Args:+ request: Contains list of execution_ids to cleanup++ Returns:+ Number of executions cleaned up and success message+ """+ logger.info(+ f"Admin {user.user_id} cleaning up {len(request.execution_ids)} orphaned executions"+ )++ cleaned_count = await cleanup_orphaned_executions_bulk(+ request.execution_ids, user.user_id+ )++ return StopExecutionResponse(+ success=cleaned_count > 0,+ stopped_count=cleaned_count,+ message=f"Cleaned up {cleaned_count} of {len(request.execution_ids)} orphaned executions",+ )+++# ============================================================================+# SCHEDULE DIAGNOSTICS ENDPOINTS+# ============================================================================+++class SchedulesListResponse(BaseModel):+ """Response model for list of schedules"""++ schedules: List[ScheduleDetail]+ total: int+++class OrphanedSchedulesListResponse(BaseModel):+ """Response model for list of orphaned schedules"""++ schedules: List[OrphanedScheduleDetail]+ total: int+++class ScheduleCleanupRequest(BaseModel):+ """Request model for cleaning up schedules"""++ schedule_ids: List[str]+++class ScheduleCleanupResponse(BaseModel):+ """Response model for schedule cleanup operations"""++ success: bool+ deleted_count: int = 0+ message: str+++ "/diagnostics/schedules",+ response_model=ScheduleHealthMetrics,+ summary="Get Schedule Diagnostics",+)+async def get_schedule_diagnostics_endpoint():+ """+ Get comprehensive diagnostic information about schedule health.++ Returns schedule metrics including:+ - Total schedules (user vs system)+ - Orphaned schedules by category+ - Upcoming executions+ """+ logger.info("Getting schedule diagnostics")++ diagnostics = await get_schedule_health_metrics()++ logger.info(+ f"Schedule diagnostics: total={diagnostics.total_schedules}, "+ f"user={diagnostics.user_schedules}, "+ f"orphaned={diagnostics.total_orphaned}"+ )++ return diagnostics+++ "/diagnostics/schedules/all",+ response_model=SchedulesListResponse,+ summary="List All User Schedules",+)+async def list_all_schedules(+ limit: int = 100,+ offset: int = 0,+):+ """+ Get detailed list of all user schedules (excludes system monitoring jobs).++ Args:+ limit: Maximum number of schedules to return (default 100)+ offset: Number of schedules to skip (default 0)++ Returns:+ List of schedules with details+ """+ logger.info(f"Listing all schedules (limit={limit}, offset={offset})")++ schedules = await get_all_schedules_details(limit=limit, offset=offset)++ # Get total count+ diagnostics = await get_schedule_health_metrics()+ total = diagnostics.user_schedules++ return SchedulesListResponse(schedules=schedules, total=total)+++ "/diagnostics/schedules/orphaned",+ response_model=OrphanedSchedulesListResponse,+ summary="List Orphaned Schedules",+)+async def list_orphaned_schedules():+ """+ Get detailed list of orphaned schedules with orphan reasons.++ Returns:+ List of orphaned schedules categorized by orphan type+ """+ logger.info("Listing orphaned schedules")++ schedules = await get_orphaned_schedules_details()++ return OrphanedSchedulesListResponse(schedules=schedules, total=len(schedules))+++ "/diagnostics/schedules/cleanup-orphaned",+ response_model=ScheduleCleanupResponse,+ summary="Cleanup Orphaned Schedules",+)+async def cleanup_orphaned_schedules(+ request: ScheduleCleanupRequest,+ user: AuthUser = Security(requires_admin_user),+):+ """+ Cleanup orphaned schedules by deleting from scheduler (admin only).++ Args:+ request: Contains list of schedule_ids to delete++ Returns:+ Number of schedules deleted and success message+ """+ logger.info(+ f"Admin {user.user_id} cleaning up {len(request.schedule_ids)} orphaned schedules"+ )++ deleted_count = await cleanup_orphaned_schedules_bulk(+ request.schedule_ids, user.user_id+ )++ return ScheduleCleanupResponse(+ success=deleted_count > 0,+ deleted_count=deleted_count,+ message=f"Deleted {deleted_count} of {len(request.schedule_ids)} orphaned schedules",+ )+++ "/diagnostics/executions/stop-all-long-running",+ response_model=StopExecutionResponse,+ summary="Stop ALL Long-Running Executions",+)+async def stop_all_long_running_executions_endpoint(+ user: AuthUser = Security(requires_admin_user),+):+ """+ Stop ALL long-running executions (RUNNING >24h) by sending cancel signals (admin only).+ Operates on entire dataset, not limited to pagination.++ Returns:+ Number of executions stopped and success message+ """+ logger.info(f"Admin {user.user_id} stopping ALL long-running executions")++ stopped_count = await stop_all_long_running_executions(user.user_id)++ return StopExecutionResponse(+ success=stopped_count > 0,+ stopped_count=stopped_count,+ message=f"Stopped {stopped_count} long-running executions",+ )+++ "/diagnostics/executions/cleanup-all-orphaned",+ response_model=StopExecutionResponse,+ summary="Cleanup ALL Orphaned Executions",+)+async def cleanup_all_orphaned_executions(+ user: AuthUser = Security(requires_admin_user),+):+ """+ Cleanup ALL orphaned executions (>24h old) by directly updating DB status.+ Operates on all executions, not just paginated results.++ Returns:+ Number of executions cleaned up and success message+ """+ logger.info(f"Admin {user.user_id} cleaning up ALL orphaned executions")++ # Fetch all orphaned execution IDs+ execution_ids = await get_all_orphaned_execution_ids()++ if not execution_ids:+ return StopExecutionResponse(+ success=True,+ stopped_count=0,+ message="No orphaned executions to cleanup",+ )++ cleaned_count = await cleanup_orphaned_executions_bulk(execution_ids, user.user_id)++ return StopExecutionResponse(+ success=cleaned_count > 0,+ stopped_count=cleaned_count,+ message=f"Cleaned up {cleaned_count} orphaned executions",+ )+++ "/diagnostics/executions/cleanup-all-stuck-queued",+ response_model=StopExecutionResponse,+ summary="Cleanup ALL Stuck Queued Executions",+)+async def cleanup_all_stuck_queued_executions_endpoint(+ user: AuthUser = Security(requires_admin_user),+):+ """+ Cleanup ALL stuck queued executions (QUEUED >1h) by updating DB status (admin only).+ Operates on entire dataset, not limited to pagination.++ Returns:+ Number of executions cleaned up and success message+ """+ logger.info(f"Admin {user.user_id} cleaning up ALL stuck queued executions")++ cleaned_count = await cleanup_all_stuck_queued_executions(user.user_id)++ return StopExecutionResponse(+ success=cleaned_count > 0,+ stopped_count=cleaned_count,+ message=f"Cleaned up {cleaned_count} stuck queued executions",+ )+++ "/diagnostics/executions/requeue-all-stuck",+ response_model=RequeueExecutionResponse,+ summary="Requeue ALL Stuck Queued Executions",+)+async def requeue_all_stuck_executions(+ user: AuthUser = Security(requires_admin_user),+):+ """+ Requeue ALL stuck queued executions (QUEUED >1h) by publishing to RabbitMQ.+ Operates on all executions, not just paginated results.++ Uses add_graph_execution with existing graph_exec_id to requeue.++ ⚠️ WARNING: This will re-execute ALL stuck executions and may cost significant credits.++ Returns:+ Number of executions requeued and success message+ """+ logger.info(f"Admin {user.user_id} requeueing ALL stuck queued executions")++ # Fetch all stuck queued execution IDs+ execution_ids = await get_all_stuck_queued_execution_ids()++ if not execution_ids:+ return RequeueExecutionResponse(+ success=True,+ requeued_count=0,+ message="No stuck queued executions to requeue",+ )++ # Get stuck executions by ID list (must be QUEUED)+ executions = await get_graph_executions(+ execution_ids=execution_ids,+ statuses=[AgentExecutionStatus.QUEUED],+ )++ # Requeue all in parallel using add_graph_execution+ async def requeue_one(exec) -> bool:+ try:+ await add_graph_execution(+ graph_id=exec.graph_id,+ user_id=exec.user_id,+ graph_version=exec.graph_version,+ graph_exec_id=exec.id, # Requeue existing+ )+ return True+ except Exception as e:+ logger.error(f"Failed to requeue {exec.id}: {e}")+ return False++ results = await asyncio.gather(+ *[requeue_one(exec) for exec in executions], return_exceptions=False+ )++ requeued_count = sum(1 for success in results if success)++ return RequeueExecutionResponse(+ success=requeued_count > 0,+ requeued_count=requeued_count,+ message=f"Requeued {requeued_count} stuck executions",+ )diff --git a/autogpt_platform/backend/backend/api/features/admin/diagnostics_admin_routes_test.py b/autogpt_platform/backend/backend/api/features/admin/diagnostics_admin_routes_test.pynew file mode 100644index 0000000000..a3783312b0--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/admin/diagnostics_admin_routes_test.py@@ -0,0 +1,889 @@+from datetime import datetime, timezone+from unittest.mock import AsyncMock++import fastapi+import fastapi.testclient+import pytest+import pytest_mock+from autogpt_libs.auth.jwt_utils import get_jwt_payload+from prisma.enums import AgentExecutionStatus++import backend.api.features.admin.diagnostics_admin_routes as diagnostics_admin_routes+from backend.data.diagnostics import (+ AgentDiagnosticsSummary,+ ExecutionDiagnosticsSummary,+ FailedExecutionDetail,+ OrphanedScheduleDetail,+ RunningExecutionDetail,+ ScheduleDetail,+ ScheduleHealthMetrics,+)+from backend.data.execution import GraphExecutionMeta++app = fastapi.FastAPI()+app.include_router(diagnostics_admin_routes.router)++client = fastapi.testclient.TestClient(app)++[email protected](autouse=True)+def setup_app_admin_auth(mock_jwt_admin):+ """Setup admin auth overrides for all tests in this module"""+ app.dependency_overrides[get_jwt_payload] = mock_jwt_admin["get_jwt_payload"]+ yield+ app.dependency_overrides.clear()+++def test_get_execution_diagnostics_success(+ mocker: pytest_mock.MockFixture,+):+ """Test fetching execution diagnostics with invalid state detection"""+ mock_diagnostics = ExecutionDiagnosticsSummary(+ running_count=10,+ queued_db_count=5,+ rabbitmq_queue_depth=3,+ cancel_queue_depth=0,+ orphaned_running=2,+ orphaned_queued=1,+ failed_count_1h=5,+ failed_count_24h=20,+ failure_rate_24h=0.83,+ stuck_running_24h=1,+ stuck_running_1h=3,+ oldest_running_hours=26.5,+ stuck_queued_1h=2,+ queued_never_started=1,+ invalid_queued_with_start=1, # New invalid state+ invalid_running_without_start=1, # New invalid state+ completed_1h=50,+ completed_24h=1200,+ throughput_per_hour=50.0,+ timestamp=datetime.now(timezone.utc).isoformat(),+ )++ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_execution_diagnostics",+ return_value=mock_diagnostics,+ )++ response = client.get("/admin/diagnostics/executions")++ assert response.status_code == 200+ data = response.json()++ # Verify new invalid state fields are included+ assert data["invalid_queued_with_start"] == 1+ assert data["invalid_running_without_start"] == 1+ # Verify all expected fields present+ assert "running_executions" in data+ assert "orphaned_running" in data+ assert "failed_count_24h" in data+++def test_list_invalid_executions(+ mocker: pytest_mock.MockFixture,+):+ """Test listing executions in invalid states (read-only endpoint)"""+ mock_invalid_executions = [+ RunningExecutionDetail(+ execution_id="exec-invalid-1",+ graph_id="graph-123",+ graph_name="Test Graph",+ graph_version=1,+ user_id="user-123",+ user_email="[email protected]",+ status="QUEUED",+ created_at=datetime.now(timezone.utc),+ started_at=datetime.now(+ timezone.utc+ ), # QUEUED but has startedAt - INVALID!+ queue_status=None,+ ),+ RunningExecutionDetail(+ execution_id="exec-invalid-2",+ graph_id="graph-456",+ graph_name="Another Graph",+ graph_version=2,+ user_id="user-456",+ user_email="[email protected]",+ status="RUNNING",+ created_at=datetime.now(timezone.utc),+ started_at=None, # RUNNING but no startedAt - INVALID!+ queue_status=None,+ ),+ ]++ mock_diagnostics = ExecutionDiagnosticsSummary(+ running_count=10,+ queued_db_count=5,+ rabbitmq_queue_depth=3,+ cancel_queue_depth=0,+ orphaned_running=0,+ orphaned_queued=0,+ failed_count_1h=0,+ failed_count_24h=0,+ failure_rate_24h=0.0,+ stuck_running_24h=0,+ stuck_running_1h=0,+ oldest_running_hours=None,+ stuck_queued_1h=0,+ queued_never_started=0,+ invalid_queued_with_start=1,+ invalid_running_without_start=1,+ completed_1h=0,+ completed_24h=0,+ throughput_per_hour=0.0,+ timestamp=datetime.now(timezone.utc).isoformat(),+ )++ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_invalid_executions_details",+ return_value=mock_invalid_executions,+ )+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_execution_diagnostics",+ return_value=mock_diagnostics,+ )++ response = client.get("/admin/diagnostics/executions/invalid?limit=100&offset=0")++ assert response.status_code == 200+ data = response.json()+ assert data["total"] == 2 # Sum of both invalid state types+ assert len(data["executions"]) == 2+ # Verify both types of invalid states are returned+ assert data["executions"][0]["execution_id"] in [+ "exec-invalid-1",+ "exec-invalid-2",+ ]+ assert data["executions"][1]["execution_id"] in [+ "exec-invalid-1",+ "exec-invalid-2",+ ]+++def test_requeue_single_execution_with_add_graph_execution(+ mocker: pytest_mock.MockFixture,+ admin_user_id: str,+):+ """Test requeueing uses add_graph_execution in requeue mode"""+ mock_exec_meta = GraphExecutionMeta(+ id="exec-stuck-123",+ user_id="user-123",+ graph_id="graph-456",+ graph_version=1,+ inputs=None,+ credential_inputs=None,+ nodes_input_masks=None,+ preset_id=None,+ status=AgentExecutionStatus.QUEUED,+ started_at=datetime.now(timezone.utc),+ ended_at=datetime.now(timezone.utc),+ stats=None,+ )++ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_graph_executions",+ return_value=[mock_exec_meta],+ )++ mock_add_graph_execution = mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.add_graph_execution",+ return_value=AsyncMock(),+ )++ response = client.post(+ "/admin/diagnostics/executions/requeue",+ json={"execution_id": "exec-stuck-123"},+ )++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is True+ assert data["requeued_count"] == 1++ # Verify it used add_graph_execution in requeue mode+ mock_add_graph_execution.assert_called_once()+ call_kwargs = mock_add_graph_execution.call_args.kwargs+ assert call_kwargs["graph_exec_id"] == "exec-stuck-123" # Requeue mode!+ assert call_kwargs["graph_id"] == "graph-456"+ assert call_kwargs["user_id"] == "user-123"+++def test_stop_single_execution_with_stop_graph_execution(+ mocker: pytest_mock.MockFixture,+ admin_user_id: str,+):+ """Test stopping uses robust stop_graph_execution"""+ mock_exec_meta = GraphExecutionMeta(+ id="exec-running-123",+ user_id="user-789",+ graph_id="graph-999",+ graph_version=2,+ inputs=None,+ credential_inputs=None,+ nodes_input_masks=None,+ preset_id=None,+ status=AgentExecutionStatus.RUNNING,+ started_at=datetime.now(timezone.utc),+ ended_at=datetime.now(timezone.utc),+ stats=None,+ )++ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_graph_executions",+ return_value=[mock_exec_meta],+ )++ mock_stop_graph_execution = mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.stop_graph_execution",+ return_value=AsyncMock(),+ )++ response = client.post(+ "/admin/diagnostics/executions/stop",+ json={"execution_id": "exec-running-123"},+ )++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is True+ assert data["stopped_count"] == 1++ # Verify it used stop_graph_execution with cascade+ mock_stop_graph_execution.assert_called_once()+ call_kwargs = mock_stop_graph_execution.call_args.kwargs+ assert call_kwargs["graph_exec_id"] == "exec-running-123"+ assert call_kwargs["user_id"] == "user-789"+ assert call_kwargs["cascade"] is True # Stops children too!+ assert call_kwargs["wait_timeout"] == 15.0+++def test_requeue_not_queued_execution_fails(+ mocker: pytest_mock.MockFixture,+):+ """Test that requeue fails if execution is not in QUEUED status"""+ # Mock an execution that's RUNNING (not QUEUED)+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_graph_executions",+ return_value=[], # No QUEUED executions found+ )++ response = client.post(+ "/admin/diagnostics/executions/requeue",+ json={"execution_id": "exec-running-123"},+ )++ assert response.status_code == 404+ assert "not found or not in QUEUED status" in response.json()["detail"]+++def test_list_invalid_executions_no_bulk_actions(+ mocker: pytest_mock.MockFixture,+):+ """Verify invalid executions endpoint is read-only (no bulk actions)"""+ # This is a documentation test - the endpoint exists but should not+ # have corresponding cleanup/stop/requeue endpoints++ # These endpoints should NOT exist for invalid states:+ invalid_bulk_endpoints = [+ "/admin/diagnostics/executions/cleanup-invalid",+ "/admin/diagnostics/executions/stop-invalid",+ "/admin/diagnostics/executions/requeue-invalid",+ ]++ for endpoint in invalid_bulk_endpoints:+ response = client.post(endpoint, json={"execution_ids": ["test"]})+ assert response.status_code == 404, f"{endpoint} should not exist (read-only)"+++def test_execution_ids_filter_efficiency(+ mocker: pytest_mock.MockFixture,+):+ """Test that bulk operations use efficient execution_ids filter"""+ mock_exec_metas = [+ GraphExecutionMeta(+ id=f"exec-{i}",+ user_id=f"user-{i}",+ graph_id="graph-123",+ graph_version=1,+ inputs=None,+ credential_inputs=None,+ nodes_input_masks=None,+ preset_id=None,+ status=AgentExecutionStatus.QUEUED,+ started_at=datetime.now(timezone.utc),+ ended_at=datetime.now(timezone.utc),+ stats=None,+ )+ for i in range(3)+ ]++ mock_get_graph_executions = mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_graph_executions",+ return_value=mock_exec_metas,+ )++ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.add_graph_execution",+ return_value=AsyncMock(),+ )++ response = client.post(+ "/admin/diagnostics/executions/requeue-bulk",+ json={"execution_ids": ["exec-0", "exec-1", "exec-2"]},+ )++ assert response.status_code == 200++ # Verify it used execution_ids filter (not fetching all queued)+ mock_get_graph_executions.assert_called_once()+ call_kwargs = mock_get_graph_executions.call_args.kwargs+ assert "execution_ids" in call_kwargs+ assert call_kwargs["execution_ids"] == ["exec-0", "exec-1", "exec-2"]+ assert call_kwargs["statuses"] == [AgentExecutionStatus.QUEUED]+++# ---------------------------------------------------------------------------+# Helper: reusable mock diagnostics summary+# ---------------------------------------------------------------------------+++def _make_mock_diagnostics(**overrides) -> ExecutionDiagnosticsSummary:+ defaults = dict(+ running_count=10,+ queued_db_count=5,+ rabbitmq_queue_depth=3,+ cancel_queue_depth=0,+ orphaned_running=2,+ orphaned_queued=1,+ failed_count_1h=5,+ failed_count_24h=20,+ failure_rate_24h=0.83,+ stuck_running_24h=3,+ stuck_running_1h=5,+ oldest_running_hours=26.5,+ stuck_queued_1h=2,+ queued_never_started=1,+ invalid_queued_with_start=1,+ invalid_running_without_start=1,+ completed_1h=50,+ completed_24h=1200,+ throughput_per_hour=50.0,+ timestamp=datetime.now(timezone.utc).isoformat(),+ )+ defaults.update(overrides)+ return ExecutionDiagnosticsSummary(**defaults)+++_SENTINEL = object()+++def _make_mock_execution(+ exec_id: str = "exec-1",+ status: str = "RUNNING",+ started_at: datetime | None | object = _SENTINEL,+) -> RunningExecutionDetail:+ return RunningExecutionDetail(+ execution_id=exec_id,+ graph_id="graph-123",+ graph_name="Test Graph",+ graph_version=1,+ user_id="user-123",+ user_email="[email protected]",+ status=status,+ created_at=datetime.now(timezone.utc),+ started_at=(+ datetime.now(timezone.utc) if started_at is _SENTINEL else started_at+ ),+ queue_status=None,+ )+++def _make_mock_failed_execution(+ exec_id: str = "exec-fail-1",+) -> FailedExecutionDetail:+ return FailedExecutionDetail(+ execution_id=exec_id,+ graph_id="graph-123",+ graph_name="Test Graph",+ graph_version=1,+ user_id="user-123",+ user_email="[email protected]",+ status="FAILED",+ created_at=datetime.now(timezone.utc),+ started_at=datetime.now(timezone.utc),+ failed_at=datetime.now(timezone.utc),+ error_message="Something went wrong",+ )+++def _make_mock_schedule_health(**overrides) -> ScheduleHealthMetrics:+ defaults = dict(+ total_schedules=15,+ user_schedules=10,+ system_schedules=5,+ orphaned_deleted_graph=2,+ orphaned_no_library_access=1,+ orphaned_invalid_credentials=0,+ orphaned_validation_failed=0,+ total_orphaned=3,+ schedules_next_hour=4,+ schedules_next_24h=8,+ total_runs_next_hour=12,+ total_runs_next_24h=48,+ timestamp=datetime.now(timezone.utc).isoformat(),+ )+ defaults.update(overrides)+ return ScheduleHealthMetrics(**defaults)+++# ---------------------------------------------------------------------------+# GET endpoints: execution list variants+# ---------------------------------------------------------------------------+++def test_list_running_executions(mocker: pytest_mock.MockFixture):+ mock_execs = [+ _make_mock_execution("exec-run-1"),+ _make_mock_execution("exec-run-2"),+ ]+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_running_executions_details",+ return_value=mock_execs,+ )+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_execution_diagnostics",+ return_value=_make_mock_diagnostics(),+ )++ response = client.get("/admin/diagnostics/executions/running?limit=50&offset=0")++ assert response.status_code == 200+ data = response.json()+ assert data["total"] == 15 # running_count(10) + queued_db_count(5)+ assert len(data["executions"]) == 2+ assert data["executions"][0]["execution_id"] == "exec-run-1"+++def test_list_orphaned_executions(mocker: pytest_mock.MockFixture):+ mock_execs = [_make_mock_execution("exec-orphan-1", status="RUNNING")]+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_orphaned_executions_details",+ return_value=mock_execs,+ )+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_execution_diagnostics",+ return_value=_make_mock_diagnostics(),+ )++ response = client.get("/admin/diagnostics/executions/orphaned?limit=50&offset=0")++ assert response.status_code == 200+ data = response.json()+ assert data["total"] == 3 # orphaned_running(2) + orphaned_queued(1)+ assert len(data["executions"]) == 1+++def test_list_failed_executions(mocker: pytest_mock.MockFixture):+ mock_execs = [_make_mock_failed_execution("exec-fail-1")]+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_failed_executions_details",+ return_value=mock_execs,+ )+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_failed_executions_count",+ return_value=42,+ )++ response = client.get(+ "/admin/diagnostics/executions/failed?limit=50&offset=0&hours=24"+ )++ assert response.status_code == 200+ data = response.json()+ assert data["total"] == 42+ assert len(data["executions"]) == 1+ assert data["executions"][0]["error_message"] == "Something went wrong"+++def test_list_long_running_executions(mocker: pytest_mock.MockFixture):+ mock_execs = [_make_mock_execution("exec-long-1")]+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_long_running_executions_details",+ return_value=mock_execs,+ )+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_execution_diagnostics",+ return_value=_make_mock_diagnostics(),+ )++ response = client.get(+ "/admin/diagnostics/executions/long-running?limit=50&offset=0"+ )++ assert response.status_code == 200+ data = response.json()+ assert data["total"] == 3 # stuck_running_24h+ assert len(data["executions"]) == 1+++def test_list_stuck_queued_executions(mocker: pytest_mock.MockFixture):+ mock_execs = [+ _make_mock_execution("exec-stuck-1", status="QUEUED", started_at=None)+ ]+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_stuck_queued_executions_details",+ return_value=mock_execs,+ )+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_execution_diagnostics",+ return_value=_make_mock_diagnostics(),+ )++ response = client.get(+ "/admin/diagnostics/executions/stuck-queued?limit=50&offset=0"+ )++ assert response.status_code == 200+ data = response.json()+ assert data["total"] == 2 # stuck_queued_1h+ assert len(data["executions"]) == 1+++# ---------------------------------------------------------------------------+# GET endpoints: agent + schedule diagnostics+# ---------------------------------------------------------------------------+++def test_get_agent_diagnostics(mocker: pytest_mock.MockFixture):+ mock_diag = AgentDiagnosticsSummary(+ agents_with_active_executions=7,+ timestamp=datetime.now(timezone.utc).isoformat(),+ )+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_agent_diagnostics",+ return_value=mock_diag,+ )++ response = client.get("/admin/diagnostics/agents")++ assert response.status_code == 200+ data = response.json()+ assert data["agents_with_active_executions"] == 7+++def test_get_schedule_diagnostics(mocker: pytest_mock.MockFixture):+ mock_metrics = _make_mock_schedule_health()+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_schedule_health_metrics",+ return_value=mock_metrics,+ )++ response = client.get("/admin/diagnostics/schedules")++ assert response.status_code == 200+ data = response.json()+ assert data["user_schedules"] == 10+ assert data["total_orphaned"] == 3+ assert data["total_runs_next_hour"] == 12+++def test_list_all_schedules(mocker: pytest_mock.MockFixture):+ mock_schedules = [+ ScheduleDetail(+ schedule_id="sched-1",+ schedule_name="Daily Run",+ graph_id="graph-1",+ graph_name="My Agent",+ graph_version=1,+ user_id="user-1",+ user_email="[email protected]",+ cron="0 9 * * *",+ timezone="UTC",+ next_run_time=datetime.now(timezone.utc).isoformat(),+ ),+ ]+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_all_schedules_details",+ return_value=mock_schedules,+ )+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_schedule_health_metrics",+ return_value=_make_mock_schedule_health(),+ )++ response = client.get("/admin/diagnostics/schedules/all?limit=50&offset=0")++ assert response.status_code == 200+ data = response.json()+ assert data["total"] == 10+ assert len(data["schedules"]) == 1+ assert data["schedules"][0]["schedule_name"] == "Daily Run"+++def test_list_orphaned_schedules(mocker: pytest_mock.MockFixture):+ mock_orphans = [+ OrphanedScheduleDetail(+ schedule_id="sched-orphan-1",+ schedule_name="Ghost Schedule",+ graph_id="graph-deleted",+ graph_version=1,+ user_id="user-1",+ orphan_reason="deleted_graph",+ error_detail=None,+ next_run_time=datetime.now(timezone.utc).isoformat(),+ ),+ ]+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_orphaned_schedules_details",+ return_value=mock_orphans,+ )++ response = client.get("/admin/diagnostics/schedules/orphaned")++ assert response.status_code == 200+ data = response.json()+ assert data["total"] == 1+ assert data["schedules"][0]["orphan_reason"] == "deleted_graph"+++# ---------------------------------------------------------------------------+# POST endpoints: bulk stop, cleanup, requeue+# ---------------------------------------------------------------------------+++def test_stop_multiple_executions(mocker: pytest_mock.MockFixture):+ mock_exec_metas = [+ GraphExecutionMeta(+ id=f"exec-{i}",+ user_id=f"user-{i}",+ graph_id="graph-123",+ graph_version=1,+ inputs=None,+ credential_inputs=None,+ nodes_input_masks=None,+ preset_id=None,+ status=AgentExecutionStatus.RUNNING,+ started_at=datetime.now(timezone.utc),+ ended_at=None,+ stats=None,+ )+ for i in range(2)+ ]+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_graph_executions",+ return_value=mock_exec_metas,+ )+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.stop_graph_execution",+ return_value=AsyncMock(),+ )++ response = client.post(+ "/admin/diagnostics/executions/stop-bulk",+ json={"execution_ids": ["exec-0", "exec-1"]},+ )++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is True+ assert data["stopped_count"] == 2+++def test_stop_multiple_executions_none_found(mocker: pytest_mock.MockFixture):+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_graph_executions",+ return_value=[],+ )++ response = client.post(+ "/admin/diagnostics/executions/stop-bulk",+ json={"execution_ids": ["nonexistent"]},+ )++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is False+ assert data["stopped_count"] == 0+++def test_cleanup_orphaned_executions(mocker: pytest_mock.MockFixture):+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.cleanup_orphaned_executions_bulk",+ return_value=3,+ )++ response = client.post(+ "/admin/diagnostics/executions/cleanup-orphaned",+ json={"execution_ids": ["exec-1", "exec-2", "exec-3"]},+ )++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is True+ assert data["stopped_count"] == 3+++def test_cleanup_orphaned_schedules(mocker: pytest_mock.MockFixture):+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.cleanup_orphaned_schedules_bulk",+ return_value=2,+ )++ response = client.post(+ "/admin/diagnostics/schedules/cleanup-orphaned",+ json={"schedule_ids": ["sched-1", "sched-2"]},+ )++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is True+ assert data["deleted_count"] == 2+++def test_stop_all_long_running_executions(mocker: pytest_mock.MockFixture):+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.stop_all_long_running_executions",+ return_value=5,+ )++ response = client.post("/admin/diagnostics/executions/stop-all-long-running")++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is True+ assert data["stopped_count"] == 5+++def test_cleanup_all_orphaned_executions(mocker: pytest_mock.MockFixture):+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_all_orphaned_execution_ids",+ return_value=["exec-1", "exec-2"],+ )+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.cleanup_orphaned_executions_bulk",+ return_value=2,+ )++ response = client.post("/admin/diagnostics/executions/cleanup-all-orphaned")++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is True+ assert data["stopped_count"] == 2+++def test_cleanup_all_orphaned_executions_none(mocker: pytest_mock.MockFixture):+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_all_orphaned_execution_ids",+ return_value=[],+ )++ response = client.post("/admin/diagnostics/executions/cleanup-all-orphaned")++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is True+ assert data["stopped_count"] == 0+ assert "No orphaned" in data["message"]+++def test_cleanup_all_stuck_queued_executions(mocker: pytest_mock.MockFixture):+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.cleanup_all_stuck_queued_executions",+ return_value=4,+ )++ response = client.post("/admin/diagnostics/executions/cleanup-all-stuck-queued")++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is True+ assert data["stopped_count"] == 4+++def test_requeue_all_stuck_executions(mocker: pytest_mock.MockFixture):+ mock_exec_metas = [+ GraphExecutionMeta(+ id=f"exec-stuck-{i}",+ user_id=f"user-{i}",+ graph_id="graph-123",+ graph_version=1,+ inputs=None,+ credential_inputs=None,+ nodes_input_masks=None,+ preset_id=None,+ status=AgentExecutionStatus.QUEUED,+ started_at=None,+ ended_at=None,+ stats=None,+ )+ for i in range(3)+ ]+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_all_stuck_queued_execution_ids",+ return_value=["exec-stuck-0", "exec-stuck-1", "exec-stuck-2"],+ )+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_graph_executions",+ return_value=mock_exec_metas,+ )+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.add_graph_execution",+ return_value=AsyncMock(),+ )++ response = client.post("/admin/diagnostics/executions/requeue-all-stuck")++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is True+ assert data["requeued_count"] == 3+++def test_requeue_all_stuck_executions_none(mocker: pytest_mock.MockFixture):+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_all_stuck_queued_execution_ids",+ return_value=[],+ )++ response = client.post("/admin/diagnostics/executions/requeue-all-stuck")++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is True+ assert data["requeued_count"] == 0+ assert "No stuck" in data["message"]+++def test_requeue_bulk_none_found(mocker: pytest_mock.MockFixture):+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_graph_executions",+ return_value=[],+ )++ response = client.post(+ "/admin/diagnostics/executions/requeue-bulk",+ json={"execution_ids": ["nonexistent"]},+ )++ assert response.status_code == 200+ data = response.json()+ assert data["success"] is False+ assert data["requeued_count"] == 0+++def test_stop_single_execution_not_found(mocker: pytest_mock.MockFixture):+ mocker.patch(+ "backend.api.features.admin.diagnostics_admin_routes.get_graph_executions",+ return_value=[],+ )++ response = client.post(+ "/admin/diagnostics/executions/stop",+ json={"execution_id": "nonexistent"},+ )++ assert response.status_code == 404+ assert "not found" in response.json()["detail"]diff --git a/autogpt_platform/backend/backend/api/features/admin/model.py b/autogpt_platform/backend/backend/api/features/admin/model.pyindex 82f51e8e7a..c96c6d6433 100644--- a/autogpt_platform/backend/backend/api/features/admin/model.py+++ b/autogpt_platform/backend/backend/api/features/admin/model.py@@ -14,3 +14,70 @@ class UserHistoryResponse(BaseModel):class AddUserCreditsResponse(BaseModel):new_balance: inttransaction_key: str+++class ExecutionDiagnosticsResponse(BaseModel):+ """Response model for execution diagnostics"""++ # Current execution state+ running_executions: int+ queued_executions_db: int+ queued_executions_rabbitmq: int+ cancel_queue_depth: int++ # Orphaned execution detection+ orphaned_running: int+ orphaned_queued: int++ # Failure metrics+ failed_count_1h: int+ failed_count_24h: int+ failure_rate_24h: float++ # Long-running detection+ stuck_running_24h: int+ stuck_running_1h: int+ oldest_running_hours: float | None++ # Stuck queued detection+ stuck_queued_1h: int+ queued_never_started: int++ # Invalid state detection (data corruption - no auto-actions)+ invalid_queued_with_start: int+ invalid_running_without_start: int++ # Throughput metrics+ completed_1h: int+ completed_24h: int+ throughput_per_hour: float++ timestamp: str+++class AgentDiagnosticsResponse(BaseModel):+ """Response model for agent diagnostics"""++ agents_with_active_executions: int+ timestamp: str+++class ScheduleHealthMetrics(BaseModel):+ """Response model for schedule diagnostics"""++ total_schedules: int+ user_schedules: int+ system_schedules: int++ # Orphan detection+ orphaned_deleted_graph: int+ orphaned_no_library_access: int+ orphaned_invalid_credentials: int+ orphaned_validation_failed: int+ total_orphaned: int++ # Upcoming+ schedules_next_hour: int+ schedules_next_24h: int++ timestamp: strdiff --git a/autogpt_platform/backend/backend/api/rest_api.py b/autogpt_platform/backend/backend/api/rest_api.pyindex 2b2dba397e..b4fc2da4e9 100644--- a/autogpt_platform/backend/backend/api/rest_api.py+++ b/autogpt_platform/backend/backend/api/rest_api.py@@ -17,6 +17,7 @@ from fastapi.routing import APIRoutefrom prisma.errors import PrismaErrorimport backend.api.features.admin.credit_admin_routes+import backend.api.features.admin.diagnostics_admin_routesimport backend.api.features.admin.execution_analytics_routesimport backend.api.features.admin.platform_cost_routesimport backend.api.features.admin.rate_limit_admin_routes@@ -320,6 +321,11 @@ app.include_router(tags=["v2", "admin"],prefix="/api/credits",)+app.include_router(+ backend.api.features.admin.diagnostics_admin_routes.router,+ tags=["v2", "admin"],+ prefix="/api",+)app.include_router(backend.api.features.admin.execution_analytics_routes.router,tags=["v2", "admin"],diff --git a/autogpt_platform/backend/backend/data/diagnostics.py b/autogpt_platform/backend/backend/data/diagnostics.pynew file mode 100644index 0000000000..933f6c2a8a--- /dev/null+++ b/autogpt_platform/backend/backend/data/diagnostics.py@@ -0,0 +1,1215 @@+"""+Diagnostics data layer for admin operations.+Provides functions to query and manage system diagnostics including executions and agents.+"""++import asyncio+import logging+from datetime import datetime, timedelta, timezone+from typing import List, Optional++from croniter import croniter+from prisma.enums import AgentExecutionStatus+from prisma.models import AgentGraph, AgentGraphExecution, LibraryAgent, User+from pydantic import BaseModel++from backend.data.db import query_raw_with_schema+from backend.data.execution import get_graph_executions, get_graph_executions_count+from backend.data.rabbitmq import SyncRabbitMQ+from backend.executor.utils import (+ GRAPH_EXECUTION_CANCEL_EXCHANGE,+ GRAPH_EXECUTION_CANCEL_QUEUE_NAME,+ GRAPH_EXECUTION_QUEUE_NAME,+ CancelExecutionEvent,+ create_execution_queue_config,+)+from backend.util.clients import get_async_execution_queue, get_scheduler_client++logger = logging.getLogger(__name__)+++# System job IDs (exclude from user schedule counts)+SYSTEM_JOB_IDS = {+ "cleanup_expired_files",+ "report_late_executions",+ "report_block_error_rates",+ "process_existing_batches",+ "process_weekly_summary",+}+++class RunningExecutionDetail(BaseModel):+ """Details about a running execution for admin view"""++ execution_id: str+ graph_id: str+ graph_name: str # Will default to "Unknown" if not available+ graph_version: int+ user_id: str+ user_email: Optional[str]+ status: str+ created_at: datetime # When execution was created+ started_at: Optional[datetime] # When execution started running+ queue_status: Optional[str] = None+++class FailedExecutionDetail(BaseModel):+ """Details about a failed execution for admin view"""++ execution_id: str+ graph_id: str+ graph_name: str+ graph_version: int+ user_id: str+ user_email: Optional[str]+ status: str+ created_at: datetime+ started_at: Optional[datetime]+ failed_at: Optional[datetime]+ error_message: Optional[str]+++class ExecutionDiagnosticsSummary(BaseModel):+ """Summary of execution diagnostics"""++ # Current execution state+ running_count: int+ queued_db_count: int+ rabbitmq_queue_depth: int+ cancel_queue_depth: int++ # Orphaned execution detection (old DB records not in executor)+ orphaned_running: int # Running but created >24h ago (likely orphaned)+ orphaned_queued: int # Queued but created >24h ago (likely orphaned)++ # Failure metrics+ failed_count_1h: int+ failed_count_24h: int+ failure_rate_24h: float # failures per hour over last 24h++ # Long-running detection (active executions)+ stuck_running_24h: int # Running for more than 24 hours+ stuck_running_1h: int # Running for more than 1 hour+ oldest_running_hours: Optional[float] # Age of oldest running execution++ # Stuck queued detection+ stuck_queued_1h: int # Queued for more than 1 hour+ queued_never_started: int # Queued but started_at is null++ # Invalid state detection (data corruption - no auto-actions)+ invalid_queued_with_start: int # QUEUED but has startedAt (impossible state)+ invalid_running_without_start: int # RUNNING but no startedAt (impossible state)++ # Throughput metrics+ completed_1h: int+ completed_24h: int+ throughput_per_hour: float # completions per hour over last 24h++ timestamp: str+++class AgentDiagnosticsSummary(BaseModel):+ """Summary of agent diagnostics"""++ agents_with_active_executions: int+ timestamp: str+++class ScheduleDetail(BaseModel):+ """Details about a schedule for admin view"""++ schedule_id: str+ schedule_name: str+ graph_id: str+ graph_name: str+ graph_version: int+ user_id: str+ user_email: Optional[str]+ cron: str+ timezone: str+ next_run_time: str+ created_at: Optional[datetime] = None # Not available from APScheduler+++class ScheduleHealthMetrics(BaseModel):+ """Summary of schedule health diagnostics"""++ total_schedules: int+ user_schedules: int # Excludes system monitoring jobs+ system_schedules: int++ # Orphan detection+ orphaned_deleted_graph: int+ orphaned_no_library_access: int+ orphaned_invalid_credentials: int+ orphaned_validation_failed: int+ total_orphaned: int++ # Upcoming schedules (unique count)+ schedules_next_hour: int+ schedules_next_24h: int++ # Upcoming execution runs (total count)+ total_runs_next_hour: int+ total_runs_next_24h: int++ timestamp: str+++class OrphanedScheduleDetail(BaseModel):+ """Details about an orphaned schedule"""++ schedule_id: str+ schedule_name: str+ graph_id: str+ graph_version: int+ user_id: str+ orphan_reason: (+ str # deleted_graph, no_library_access, invalid_credentials, validation_failed+ )+ error_detail: Optional[str]+ next_run_time: str+++def _to_running_execution_detail(+ exec: AgentGraphExecution,+) -> RunningExecutionDetail:+ """Convert a Prisma AgentGraphExecution (with includes) to RunningExecutionDetail."""+ return RunningExecutionDetail(+ execution_id=exec.id,+ graph_id=exec.agentGraphId,+ graph_name=(+ exec.AgentGraph.name+ if exec.AgentGraph and exec.AgentGraph.name+ else "Unknown"+ ),+ graph_version=exec.agentGraphVersion,+ user_id=exec.userId,+ user_email=exec.User.email if exec.User else None,+ status=exec.executionStatus,+ created_at=exec.createdAt,+ started_at=exec.startedAt,+ )+++_EXECUTION_ADMIN_INCLUDE = {+ "AgentGraph": True,+ "User": True,+}+++async def get_execution_diagnostics() -> ExecutionDiagnosticsSummary:+ """+ Get comprehensive execution diagnostics including database and queue metrics.+ Uses a single batched SQL query for all count metrics to minimize DB round-trips.++ Returns:+ ExecutionDiagnosticsSummary with current execution state+ """+ now = datetime.now(timezone.utc)+ one_hour_ago = now - timedelta(hours=1)+ twenty_four_hours_ago = now - timedelta(hours=24)++ # Single SQL query to get all count metrics at once+ counts = await query_raw_with_schema(+ """+ SELECT+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'RUNNING'+ ) AS running_count,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'QUEUED'+ ) AS queued_db_count,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'RUNNING'+ AND "createdAt" < $1::timestamp+ ) AS orphaned_running,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'QUEUED'+ AND "createdAt" < $1::timestamp+ ) AS orphaned_queued,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'FAILED'+ AND "updatedAt" >= $2::timestamp+ ) AS failed_count_1h,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'FAILED'+ AND "updatedAt" >= $1::timestamp+ ) AS failed_count_24h,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'RUNNING'+ AND "startedAt" IS NOT NULL+ AND "startedAt" < $1::timestamp+ ) AS stuck_running_24h,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'RUNNING'+ AND "startedAt" IS NOT NULL+ AND "startedAt" < $2::timestamp+ ) AS stuck_running_1h,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'QUEUED'+ AND "createdAt" < $2::timestamp+ ) AS stuck_queued_1h,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'QUEUED'+ AND "startedAt" IS NULL+ ) AS queued_never_started,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'QUEUED'+ AND "startedAt" IS NOT NULL+ ) AS invalid_queued_with_start,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'RUNNING'+ AND "startedAt" IS NULL+ ) AS invalid_running_without_start,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'COMPLETED'+ AND "updatedAt" >= $2::timestamp+ ) AS completed_1h,+ COUNT(*) FILTER (+ WHERE "executionStatus" = 'COMPLETED'+ AND "updatedAt" >= $1::timestamp+ ) AS completed_24h+ FROM {schema_prefix}"AgentGraphExecution"+ WHERE "isDeleted" = false+ """,+ twenty_four_hours_ago,+ one_hour_ago,+ )++ row = counts[0] if counts else {}++ running_count = row.get("running_count", 0)+ queued_db_count = row.get("queued_db_count", 0)+ orphaned_running = row.get("orphaned_running", 0)+ orphaned_queued = row.get("orphaned_queued", 0)+ failed_count_1h = row.get("failed_count_1h", 0)+ failed_count_24h = row.get("failed_count_24h", 0)+ stuck_running_24h = row.get("stuck_running_24h", 0)+ stuck_running_1h = row.get("stuck_running_1h", 0)+ stuck_queued_1h = row.get("stuck_queued_1h", 0)+ queued_never_started = row.get("queued_never_started", 0)+ invalid_queued_with_start = row.get("invalid_queued_with_start", 0)+ invalid_running_without_start = row.get("invalid_running_without_start", 0)+ completed_1h = row.get("completed_1h", 0)+ completed_24h = row.get("completed_24h", 0)++ failure_rate_24h = failed_count_24h / 24.0 if failed_count_24h > 0 else 0.0+ throughput_per_hour = completed_24h / 24.0 if completed_24h > 0 else 0.0++ # RabbitMQ queue depths (blocking sync calls, run in thread pool)+ rabbitmq_queue_depth, cancel_queue_depth = await asyncio.gather(+ asyncio.to_thread(get_rabbitmq_queue_depth),+ asyncio.to_thread(get_rabbitmq_cancel_queue_depth),+ )++ # Find oldest running execution (single query)+ oldest_running_list = await get_graph_executions(+ statuses=[AgentExecutionStatus.RUNNING],+ order_by="startedAt",+ order_direction="asc",+ limit=1,+ )++ oldest_running_hours = None+ if oldest_running_list and oldest_running_list[0].started_at:+ age_seconds = (now - oldest_running_list[0].started_at).total_seconds()+ oldest_running_hours = age_seconds / 3600.0++ return ExecutionDiagnosticsSummary(+ running_count=running_count,+ queued_db_count=queued_db_count,+ rabbitmq_queue_depth=rabbitmq_queue_depth,+ cancel_queue_depth=cancel_queue_depth,+ orphaned_running=orphaned_running,+ orphaned_queued=orphaned_queued,+ failed_count_1h=failed_count_1h,+ failed_count_24h=failed_count_24h,+ failure_rate_24h=failure_rate_24h,+ stuck_running_24h=stuck_running_24h,+ stuck_running_1h=stuck_running_1h,+ oldest_running_hours=oldest_running_hours,+ stuck_queued_1h=stuck_queued_1h,+ queued_never_started=queued_never_started,+ invalid_queued_with_start=invalid_queued_with_start,+ invalid_running_without_start=invalid_running_without_start,+ completed_1h=completed_1h,+ completed_24h=completed_24h,+ throughput_per_hour=throughput_per_hour,+ timestamp=now.isoformat(),+ )+++async def get_agent_diagnostics() -> AgentDiagnosticsSummary:+ """+ Get comprehensive agent diagnostics.++ Returns:+ AgentDiagnosticsSummary with agent metrics+ """+ # Single query to count distinct agents with active executions+ result = await query_raw_with_schema(+ """+ SELECT COUNT(DISTINCT "agentGraphId") AS active_agents+ FROM {schema_prefix}"AgentGraphExecution"+ WHERE "executionStatus" IN ('RUNNING', 'QUEUED')+ AND "isDeleted" = false+ """+ )++ active_agents = result[0].get("active_agents", 0) if result else 0++ return AgentDiagnosticsSummary(+ agents_with_active_executions=active_agents,+ timestamp=datetime.now(timezone.utc).isoformat(),+ )+++async def get_schedule_health_metrics() -> ScheduleHealthMetrics:+ """+ Get comprehensive schedule diagnostics via Scheduler service.++ Returns:+ ScheduleHealthMetrics with schedule health info+ """+ scheduler = get_scheduler_client()++ # Get all schedules from scheduler service+ all_schedules = await scheduler.get_execution_schedules()++ # Filter user vs system schedules+ user_schedules = [s for s in all_schedules if s.id not in SYSTEM_JOB_IDS]+ system_schedules_count = len(all_schedules) - len(user_schedules)++ # Detect orphaned schedules+ orphans = await _detect_orphaned_schedules(user_schedules)++ # Count schedules by next run time (exclude orphaned schedules)+ now = datetime.now(timezone.utc)+ one_hour_from_now = now + timedelta(hours=1)+ twenty_four_hours_from_now = now + timedelta(hours=24)++ orphaned_ids = set()+ for category_ids in orphans.values():+ orphaned_ids.update(category_ids)++ healthy_schedules = [s for s in user_schedules if s.id not in orphaned_ids]++ schedules_next_hour = sum(+ 1+ for s in healthy_schedules+ if s.next_run_time+ and datetime.fromisoformat(s.next_run_time.replace("Z", "+00:00"))+ <= one_hour_from_now+ )++ schedules_next_24h = sum(+ 1+ for s in healthy_schedules+ if s.next_run_time+ and datetime.fromisoformat(s.next_run_time.replace("Z", "+00:00"))+ <= twenty_four_hours_from_now+ )++ # Calculate total execution runs (not just unique schedules, exclude orphaned)+ total_runs_next_hour = _calculate_total_runs(+ healthy_schedules, now, one_hour_from_now+ )+ total_runs_next_24h = _calculate_total_runs(+ healthy_schedules, now, twenty_four_hours_from_now+ )++ return ScheduleHealthMetrics(+ total_schedules=len(all_schedules),+ user_schedules=len(user_schedules),+ system_schedules=system_schedules_count,+ orphaned_deleted_graph=len(orphans["deleted_graph"]),+ orphaned_no_library_access=len(orphans["no_library_access"]),+ orphaned_invalid_credentials=len(orphans["invalid_credentials"]),+ orphaned_validation_failed=len(orphans["validation_failed"]),+ total_orphaned=sum(len(v) for v in orphans.values()),+ schedules_next_hour=schedules_next_hour,+ schedules_next_24h=schedules_next_24h,+ total_runs_next_hour=total_runs_next_hour,+ total_runs_next_24h=total_runs_next_24h,+ timestamp=now.isoformat(),+ )+++def _calculate_total_runs(+ schedules: list, start_time: datetime, end_time: datetime+) -> int:+ """+ Calculate total number of scheduled executions in time window.++ Args:+ schedules: List of GraphExecutionJobInfo with cron expressions+ start_time: Start of time window+ end_time: End of time window++ Returns:+ Total number of execution runs across all schedules+ """+ total_runs = 0++ for schedule in schedules:+ try:+ # Create cron iterator+ iter = croniter(schedule.cron, start_time)++ # Count occurrences in window (with safety limit)+ count = 0+ max_iterations = 2000 # Safety limit (e.g., every-minute for 24h = 1440)++ while count < max_iterations:+ try:+ next_run = iter.get_next(datetime)+ if next_run > end_time:+ break+ count += 1+ except Exception:+ # Handle edge cases like invalid cron progression+ break++ total_runs += count++ except Exception as e:+ logger.warning(f"Failed to parse cron expression '{schedule.cron}': {e}")+ # Skip this schedule if cron is invalid+ continue++ return total_runs+++async def _detect_orphaned_schedules(schedules: list) -> dict:+ """+ Detect orphaned schedules by validating graph, library access, and credentials.++ Args:+ schedules: List of GraphExecutionJobInfo from scheduler service++ Returns:+ Dict categorizing orphans by type+ """+ orphans = {+ "deleted_graph": [],+ "no_library_access": [],+ "invalid_credentials": [],+ "validation_failed": [],+ }++ for schedule in schedules:+ try:+ # Check 1: Graph exists+ graph = await AgentGraph.prisma().find_unique(+ where={+ "graphVersionId": {+ "id": schedule.graph_id,+ "version": schedule.graph_version,+ }+ }+ )++ if not graph:+ orphans["deleted_graph"].append(schedule.id)+ continue++ # Check 2: User has library access (not deleted/archived)+ library_agent = await LibraryAgent.prisma().find_first(+ where={+ "userId": schedule.user_id,+ "agentGraphId": schedule.graph_id,+ "isDeleted": False,+ "isArchived": False,+ }+ )++ if not library_agent:+ orphans["no_library_access"].append(schedule.id)+ continue++ # Check 3: Credentials exist (if any)+ # Note: Full credential validation would require integration_creds_manager+ # For now, skip credential validation to avoid complexity+ # Orphaned credentials will be caught during execution attempt++ except Exception as e:+ logger.error(f"Error validating schedule {schedule.id}: {e}")+ orphans["validation_failed"].append(schedule.id)++ return orphans+++def get_rabbitmq_queue_depth() -> int:+ """+ Get the number of messages in the RabbitMQ execution queue.++ Returns:+ Number of messages in queue, or -1 if error+ """+ try:+ # Create a temporary connection to query the queue+ config = create_execution_queue_config()+ rabbitmq = SyncRabbitMQ(config)+ rabbitmq.connect()++ try:+ # Use passive queue_declare to get queue info without modifying it+ if rabbitmq._channel:+ method_frame = rabbitmq._channel.queue_declare(+ queue=GRAPH_EXECUTION_QUEUE_NAME, passive=True+ )+ else:+ raise RuntimeError("RabbitMQ channel not initialized")++ return method_frame.method.message_count+ finally:+ # Always clean up connection, even on error+ try:+ rabbitmq.disconnect()+ except Exception as disconnect_err:+ logger.warning(+ f"Failed to close RabbitMQ connection after queue depth check: {disconnect_err}"+ )+ except Exception as e:+ logger.error(f"Error getting RabbitMQ queue depth: {e}")+ # Return -1 to indicate an error state rather than failing the entire request+ return -1+++def get_rabbitmq_cancel_queue_depth() -> int:+ """+ Get the number of messages in the RabbitMQ cancel queue.++ Returns:+ Number of messages in cancel queue, or -1 if error+ """+ try:+ # Create a temporary connection to query the queue+ config = create_execution_queue_config()+ rabbitmq = SyncRabbitMQ(config)+ rabbitmq.connect()++ try:+ # Use passive queue_declare to get queue info without modifying it+ if rabbitmq._channel:+ method_frame = rabbitmq._channel.queue_declare(+ queue=GRAPH_EXECUTION_CANCEL_QUEUE_NAME, passive=True+ )+ else:+ raise RuntimeError("RabbitMQ channel not initialized")++ return method_frame.method.message_count+ finally:+ # Always clean up connection, even on error+ try:+ rabbitmq.disconnect()+ except Exception as disconnect_err:+ logger.warning(+ f"Failed to close RabbitMQ connection after cancel queue check: {disconnect_err}"+ )+ except Exception as e:+ logger.error(f"Error getting RabbitMQ cancel queue depth: {e}")+ # Return -1 to indicate an error state rather than failing the entire request+ return -1+++async def get_all_schedules_details(+ limit: int = 100, offset: int = 0+) -> List[ScheduleDetail]:+ """+ Get detailed information about all user schedules via Scheduler service.++ Args:+ limit: Maximum number of schedules to return+ offset: Number of schedules to skip++ Returns:+ List of ScheduleDetail objects+ """+ scheduler = get_scheduler_client()++ # Get all schedules from scheduler+ all_schedules = await scheduler.get_execution_schedules()++ # Filter to user schedules only+ user_schedules = [s for s in all_schedules if s.id not in SYSTEM_JOB_IDS]++ # Apply pagination+ paginated_schedules = user_schedules[offset : offset + limit]++ # Enrich with graph and user details+ results = []+ for schedule in paginated_schedules:+ # Get graph name+ graph = await AgentGraph.prisma().find_unique(+ where={+ "graphVersionId": {+ "id": schedule.graph_id,+ "version": schedule.graph_version,+ }+ },+ )++ graph_name = graph.name if graph and graph.name else "Unknown"++ # Fetch user by schedule creator's user_id (not graph owner)+ schedule_user = await User.prisma().find_unique(where={"id": schedule.user_id})+ user_email = schedule_user.email if schedule_user else None++ results.append(+ ScheduleDetail(+ schedule_id=schedule.id,+ schedule_name=schedule.name,+ graph_id=schedule.graph_id,+ graph_name=graph_name,+ graph_version=schedule.graph_version,+ user_id=schedule.user_id,+ user_email=user_email,+ cron=schedule.cron,+ timezone=schedule.timezone,+ next_run_time=schedule.next_run_time,+ )+ )++ return results+++async def get_orphaned_schedules_details() -> List[OrphanedScheduleDetail]:+ """+ Get detailed list of orphaned schedules with orphan reasons.++ Returns:+ List of OrphanedScheduleDetail objects+ """+ scheduler = get_scheduler_client()++ # Get all schedules+ all_schedules = await scheduler.get_execution_schedules()+ user_schedules = [s for s in all_schedules if s.id not in SYSTEM_JOB_IDS]++ # Detect orphans with categorization+ orphan_categories = await _detect_orphaned_schedules(user_schedules)++ # Build detailed orphan list+ results = []+ for orphan_type, schedule_ids in orphan_categories.items():+ for schedule_id in schedule_ids:+ # Find the schedule+ schedule = next((s for s in user_schedules if s.id == schedule_id), None)+ if not schedule:+ continue++ results.append(+ OrphanedScheduleDetail(+ schedule_id=schedule.id,+ schedule_name=schedule.name,+ graph_id=schedule.graph_id,+ graph_version=schedule.graph_version,+ user_id=schedule.user_id,+ orphan_reason=orphan_type,+ error_detail=None, # Could add more detail in future+ next_run_time=schedule.next_run_time,+ )+ )++ return results+++async def cleanup_orphaned_schedules_bulk(+ schedule_ids: List[str], admin_user_id: str+) -> int:+ """+ Cleanup multiple orphaned schedules by deleting from scheduler.++ Args:+ schedule_ids: List of schedule IDs to delete+ admin_user_id: ID of the admin user performing the operation++ Returns:+ Number of schedules successfully deleted+ """+ logger.info(+ f"Admin user {admin_user_id} cleaning up {len(schedule_ids)} orphaned schedules"+ )++ scheduler = get_scheduler_client()++ # Fetch all schedules once to avoid N+1 queries+ all_schedules = await scheduler.get_execution_schedules()+ schedule_map = {s.id: s for s in all_schedules}++ # Delete schedules in parallel+ async def delete_schedule(schedule_id: str) -> bool:+ schedule = schedule_map.get(schedule_id)+ if not schedule:+ logger.warning(f"Schedule {schedule_id} not found")+ return False++ try:+ await scheduler.delete_schedule(+ schedule_id=schedule_id, user_id=schedule.user_id+ )+ return True+ except Exception as e:+ logger.error(f"Failed to delete schedule {schedule_id}: {e}")+ return False++ results = await asyncio.gather(+ *[delete_schedule(schedule_id) for schedule_id in schedule_ids],+ return_exceptions=False,+ )++ deleted_count = sum(1 for success in results if success)++ logger.info(+ f"Admin {admin_user_id} deleted {deleted_count}/{len(schedule_ids)} orphaned schedules"+ )++ return deleted_count+++async def get_running_executions_details(+ limit: int = 100, offset: int = 0+) -> List[RunningExecutionDetail]:+ """+ Get detailed information about running and queued executions.++ Args:+ limit: Maximum number of executions to return+ offset: Number of executions to skip++ Returns:+ List of RunningExecutionDetail objects+ """+ executions = await AgentGraphExecution.prisma().find_many(+ where={+ "executionStatus": {+ "in": [AgentExecutionStatus.RUNNING, AgentExecutionStatus.QUEUED] # type: ignore+ },+ "isDeleted": False,+ },+ include=_EXECUTION_ADMIN_INCLUDE,+ take=limit,+ skip=offset,+ order={"createdAt": "desc"},+ )++ return [_to_running_execution_detail(e) for e in executions]+++async def get_orphaned_executions_details(+ limit: int = 100, offset: int = 0+) -> List[RunningExecutionDetail]:+ """+ Get detailed information about orphaned executions (>24h old, likely not in executor).++ Args:+ limit: Maximum number of executions to return+ offset: Number of executions to skip++ Returns:+ List of orphaned RunningExecutionDetail objects+ """+ cutoff = datetime.now(timezone.utc) - timedelta(hours=24)++ executions = await AgentGraphExecution.prisma().find_many(+ where={+ "executionStatus": {+ "in": [AgentExecutionStatus.RUNNING, AgentExecutionStatus.QUEUED] # type: ignore+ },+ "createdAt": {"lt": cutoff},+ "isDeleted": False,+ },+ include=_EXECUTION_ADMIN_INCLUDE,+ take=limit,+ skip=offset,+ order={"createdAt": "asc"},+ )++ return [_to_running_execution_detail(e) for e in executions]+++async def get_long_running_executions_details(+ limit: int = 100, offset: int = 0+) -> List[RunningExecutionDetail]:+ """+ Get detailed information about long-running executions (RUNNING status >24h).++ Args:+ limit: Maximum number of executions to return+ offset: Number of executions to skip++ Returns:+ List of long-running RunningExecutionDetail objects+ """+ cutoff = datetime.now(timezone.utc) - timedelta(hours=24)++ executions = await AgentGraphExecution.prisma().find_many(+ where={+ "executionStatus": AgentExecutionStatus.RUNNING,+ "startedAt": {"lt": cutoff},+ "isDeleted": False,+ },+ include=_EXECUTION_ADMIN_INCLUDE,+ take=limit,+ skip=offset,+ order={"startedAt": "asc"},+ )++ return [_to_running_execution_detail(e) for e in executions]+++async def get_stuck_queued_executions_details(+ limit: int = 100, offset: int = 0+) -> List[RunningExecutionDetail]:+ """+ Get detailed information about stuck queued executions (QUEUED >1h, never started).++ Args:+ limit: Maximum number of executions to return+ offset: Number of executions to skip++ Returns:+ List of stuck queued RunningExecutionDetail objects+ """+ one_hour_ago = datetime.now(timezone.utc) - timedelta(hours=1)++ executions = await AgentGraphExecution.prisma().find_many(+ where={+ "executionStatus": AgentExecutionStatus.QUEUED,+ "createdAt": {"lt": one_hour_ago},+ "isDeleted": False,+ },+ include=_EXECUTION_ADMIN_INCLUDE,+ take=limit,+ skip=offset,+ order={"createdAt": "asc"},+ )++ return [_to_running_execution_detail(e) for e in executions]+++async def get_invalid_executions_details(+ limit: int = 100, offset: int = 0+) -> List[RunningExecutionDetail]:+ """+ Get detailed information about executions in invalid states.++ Invalid states are data corruption issues that require manual investigation:+ - QUEUED but has startedAt (impossible - can't start while queued)+ - RUNNING but no startedAt (impossible - can't run without starting)++ NO bulk actions provided - these need case-by-case investigation.++ Args:+ limit: Maximum number of executions to return+ offset: Number of executions to skip++ Returns:+ List of invalid RunningExecutionDetail objects+ """+ executions = await AgentGraphExecution.prisma().find_many(+ where={+ "isDeleted": False,+ "OR": [ # type: ignore+ {+ "executionStatus": AgentExecutionStatus.QUEUED,+ "startedAt": {"not": None}, # type: ignore+ },+ {+ "executionStatus": AgentExecutionStatus.RUNNING,+ "startedAt": None,+ },+ ],+ },+ include=_EXECUTION_ADMIN_INCLUDE,+ take=limit,+ skip=offset,+ order={"createdAt": "desc"},+ )++ return [_to_running_execution_detail(e) for e in executions]+++async def get_failed_executions_count(hours: int = 24) -> int:+ """+ Get count of failed executions within the specified time window.++ Args:+ hours: Number of hours to look back (default 24)++ Returns:+ Count of failed executions+ """+ cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)+ count = await get_graph_executions_count(+ statuses=[AgentExecutionStatus.FAILED],+ updated_time_gte=cutoff,+ )+ return count+++async def get_failed_executions_details(+ limit: int = 100, offset: int = 0, hours: int = 24+) -> List[FailedExecutionDetail]:+ """+ Get detailed information about failed executions.++ Args:+ limit: Maximum number of executions to return+ offset: Number of executions to skip+ hours: Number of hours to look back (default 24)++ Returns:+ List of FailedExecutionDetail objects+ """+ cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)++ executions = await AgentGraphExecution.prisma().find_many(+ where={+ "executionStatus": AgentExecutionStatus.FAILED,+ "updatedAt": {"gte": cutoff},+ "isDeleted": False,+ },+ include=_EXECUTION_ADMIN_INCLUDE,+ take=limit,+ skip=offset,+ order={"updatedAt": "desc"}, # Most recent failures first+ )++ results = []+ for exec in executions:+ # Extract error from stats JSON field+ error_message = None+ if exec.stats and isinstance(exec.stats, dict):+ error_message = exec.stats.get("error")++ results.append(+ FailedExecutionDetail(+ execution_id=exec.id,+ graph_id=exec.agentGraphId,+ graph_name=(+ exec.AgentGraph.name+ if exec.AgentGraph and exec.AgentGraph.name+ else "Unknown"+ ),+ graph_version=exec.agentGraphVersion,+ user_id=exec.userId,+ user_email=exec.User.email if exec.User else None,+ status=exec.executionStatus,+ created_at=exec.createdAt,+ started_at=exec.startedAt,+ failed_at=exec.updatedAt,+ error_message=error_message,+ )+ )++ return results+++async def cleanup_orphaned_execution(execution_id: str, admin_user_id: str) -> bool:+ """+ Cleanup orphaned execution by directly updating DB status.+ For executions that are in DB but not actually running in executor.++ Args:+ execution_id: ID of the execution to cleanup+ admin_user_id: ID of the admin user performing the operation++ Returns:+ True if execution was cleaned up, False otherwise+ """+ logger.info(+ f"Admin user {admin_user_id} cleaning up orphaned execution {execution_id}"+ )++ # Update DB status directly without sending cancel signal+ result = await AgentGraphExecution.prisma().update(+ where={"id": execution_id},+ data={+ "executionStatus": AgentExecutionStatus.FAILED,+ "updatedAt": datetime.now(timezone.utc),+ },+ )++ logger.info(+ f"Admin {admin_user_id} marked orphaned execution {execution_id} as FAILED"+ )+ return result is not None+++async def stop_all_long_running_executions(admin_user_id: str) -> int:+ """+ Stop ALL long-running executions (RUNNING >24h) by sending cancel signals.++ Args:+ admin_user_id: ID of the admin user performing the operation++ Returns:+ Number of executions for which cancel signals were sent+ """+ logger.info(f"Admin user {admin_user_id} stopping ALL long-running executions")++ # Find all long-running executions (started running >24h ago)+ cutoff = datetime.now(timezone.utc) - timedelta(hours=24)+ executions = await get_graph_executions(+ statuses=[AgentExecutionStatus.RUNNING],+ started_time_lte=cutoff,+ )++ if not executions:+ logger.info("No long-running executions to stop")+ return 0++ queue_client = await get_async_execution_queue()++ # Send cancel signals in parallel+ async def send_cancel_signal(exec_id: str) -> bool:+ try:+ await queue_client.publish_message(+ routing_key="",+ message=CancelExecutionEvent(graph_exec_id=exec_id).model_dump_json(),+ exchange=GRAPH_EXECUTION_CANCEL_EXCHANGE,+ )+ return True+ except Exception as e:+ logger.error(f"Failed to send cancel for {exec_id}: {e}")+ return False++ # Send cancel signals in parallel+ await asyncio.gather(+ *[send_cancel_signal(exec.id) for exec in executions],+ return_exceptions=True, # Don't fail if some signals fail+ )++ # ALSO update DB status directly (don't rely on executor)+ # This ensures executions are marked FAILED even if executor restarted+ result = await AgentGraphExecution.prisma().update_many(+ where={+ "executionStatus": AgentExecutionStatus.RUNNING,+ "startedAt": {"lt": cutoff},+ "isDeleted": False,+ },+ data={+ "executionStatus": AgentExecutionStatus.FAILED,+ "updatedAt": datetime.now(timezone.utc),+ },+ )++ logger.info(+ f"Admin {admin_user_id} stopped {result} long-running executions (sent cancel signals + updated DB)"+ )++ return result+++async def get_all_orphaned_execution_ids() -> List[str]:+ """+ Get all orphaned execution IDs (>24h old, RUNNING or QUEUED).++ Returns:+ List of execution IDs that are orphaned+ """+ cutoff = datetime.now(timezone.utc) - timedelta(hours=24)++ executions = await get_graph_executions(+ statuses=[AgentExecutionStatus.RUNNING, AgentExecutionStatus.QUEUED],+ created_time_lte=cutoff,+ )++ return [e.id for e in executions]+++async def cleanup_orphaned_executions_bulk(+ execution_ids: List[str], admin_user_id: str+) -> int:+ """+ Cleanup multiple orphaned executions by directly updating DB status.+ For executions in DB but not actually running in executor (old/orphaned).++ Args:+ execution_ids: List of execution IDs to cleanup+ admin_user_id: ID of the admin user performing the operation++ Returns:+ Number of executions successfully cleaned up+ """+ logger.info(+ f"Admin user {admin_user_id} cleaning up {len(execution_ids)} orphaned executions"+ )++ # Update all executions in DB directly (no cancel signals)+ # Only update executions still in RUNNING/QUEUED status to avoid+ # overwriting a legitimately COMPLETED execution (TOCTOU guard)+ result = await AgentGraphExecution.prisma().update_many(+ where={+ "id": {"in": execution_ids},+ "isDeleted": False,+ "executionStatus": {+ "in": [AgentExecutionStatus.RUNNING, AgentExecutionStatus.QUEUED]+ },+ },+ data={+ "executionStatus": AgentExecutionStatus.FAILED,+ "updatedAt": datetime.now(timezone.utc),+ },+ )++ logger.info(+ f"Admin {admin_user_id} marked {result} orphaned executions as FAILED in DB"+ )++ return result+++async def get_all_stuck_queued_execution_ids() -> List[str]:+ """+ Get all stuck queued execution IDs (QUEUED >1h).++ Returns:+ List of execution IDs that are stuck in QUEUED status+ """+ one_hour_ago = datetime.now(timezone.utc) - timedelta(hours=1)++ executions = await get_graph_executions(+ statuses=[AgentExecutionStatus.QUEUED],+ created_time_lte=one_hour_ago,+ )++ return [e.id for e in executions]+++async def cleanup_all_stuck_queued_executions(admin_user_id: str) -> int:+ """+ Cleanup ALL stuck queued executions (QUEUED >1h) by updating DB status.+ Operates on all stuck queued executions, not just paginated results.++ Args:+ admin_user_id: ID of the admin user performing the operation++ Returns:+ Number of executions successfully cleaned up+ """+ logger.info(f"Admin user {admin_user_id} cleaning up ALL stuck queued executions")++ # Find all stuck queued executions (>1h old)+ one_hour_ago = datetime.now(timezone.utc) - timedelta(hours=1)++ result = await AgentGraphExecution.prisma().update_many(+ where={+ "executionStatus": AgentExecutionStatus.QUEUED,+ "createdAt": {"lt": one_hour_ago},+ "isDeleted": False,+ },+ data={+ "executionStatus": AgentExecutionStatus.FAILED,+ "updatedAt": datetime.now(timezone.utc),+ },+ )++ logger.info(+ f"Admin {admin_user_id} marked {result} stuck queued executions as FAILED in DB"+ )++ return resultdiff --git a/autogpt_platform/backend/backend/data/diagnostics_test.py b/autogpt_platform/backend/backend/data/diagnostics_test.pynew file mode 100644index 0000000000..fc52070411--- /dev/null+++ b/autogpt_platform/backend/backend/data/diagnostics_test.py@@ -0,0 +1,464 @@+"""Unit tests for diagnostics data layer functions."""++from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from backend.data.diagnostics import (+ _calculate_total_runs,+ _detect_orphaned_schedules,+ get_execution_diagnostics,+ get_rabbitmq_cancel_queue_depth,+ get_rabbitmq_queue_depth,+)++# ---------------------------------------------------------------------------+# get_execution_diagnostics tests+# ---------------------------------------------------------------------------+++async def test_get_execution_diagnostics_full():+ """Test get_execution_diagnostics aggregates all data correctly."""+ mock_row = {+ "running_count": 10,+ "queued_db_count": 5,+ "orphaned_running": 2,+ "orphaned_queued": 1,+ "failed_count_1h": 3,+ "failed_count_24h": 12,+ "stuck_running_24h": 1,+ "stuck_running_1h": 2,+ "stuck_queued_1h": 4,+ "queued_never_started": 3,+ "invalid_queued_with_start": 1,+ "invalid_running_without_start": 0,+ "completed_1h": 50,+ "completed_24h": 600,+ }++ mock_exec = MagicMock()+ mock_exec.started_at = datetime.now(timezone.utc) - timedelta(hours=48)++ with (+ patch(+ "backend.data.diagnostics.query_raw_with_schema",+ new_callable=AsyncMock,+ return_value=[mock_row],+ ),+ patch(+ "backend.data.diagnostics.get_rabbitmq_queue_depth",+ return_value=7,+ ),+ patch(+ "backend.data.diagnostics.get_rabbitmq_cancel_queue_depth",+ return_value=2,+ ),+ patch(+ "backend.data.diagnostics.get_graph_executions",+ new_callable=AsyncMock,+ return_value=[mock_exec],+ ),+ ):+ result = await get_execution_diagnostics()++ assert result.running_count == 10+ assert result.queued_db_count == 5+ assert result.orphaned_running == 2+ assert result.orphaned_queued == 1+ assert result.failed_count_1h == 3+ assert result.failed_count_24h == 12+ assert result.failure_rate_24h == 12 / 24.0+ assert result.stuck_running_24h == 1+ assert result.stuck_running_1h == 2+ assert result.stuck_queued_1h == 4+ assert result.queued_never_started == 3+ assert result.invalid_queued_with_start == 1+ assert result.invalid_running_without_start == 0+ assert result.completed_1h == 50+ assert result.completed_24h == 600+ assert result.throughput_per_hour == 600 / 24.0+ assert result.rabbitmq_queue_depth == 7+ assert result.cancel_queue_depth == 2+ assert result.oldest_running_hours is not None+ assert result.oldest_running_hours > 47.0+++async def test_get_execution_diagnostics_empty_db():+ """Test get_execution_diagnostics with empty database."""+ with (+ patch(+ "backend.data.diagnostics.query_raw_with_schema",+ new_callable=AsyncMock,+ return_value=[{}],+ ),+ patch(+ "backend.data.diagnostics.get_rabbitmq_queue_depth",+ return_value=-1,+ ),+ patch(+ "backend.data.diagnostics.get_rabbitmq_cancel_queue_depth",+ return_value=-1,+ ),+ patch(+ "backend.data.diagnostics.get_graph_executions",+ new_callable=AsyncMock,+ return_value=[],+ ),+ ):+ result = await get_execution_diagnostics()++ assert result.running_count == 0+ assert result.queued_db_count == 0+ assert result.failure_rate_24h == 0.0+ assert result.throughput_per_hour == 0.0+ assert result.oldest_running_hours is None+ assert result.rabbitmq_queue_depth == -1+ assert result.cancel_queue_depth == -1+++async def test_get_execution_diagnostics_no_started_at():+ """Test oldest_running_hours when oldest execution has no started_at."""+ mock_row = {+ "running_count": 1,+ "queued_db_count": 0,+ "orphaned_running": 0,+ "orphaned_queued": 0,+ "failed_count_1h": 0,+ "failed_count_24h": 0,+ "stuck_running_24h": 0,+ "stuck_running_1h": 0,+ "stuck_queued_1h": 0,+ "queued_never_started": 0,+ "invalid_queued_with_start": 0,+ "invalid_running_without_start": 1,+ "completed_1h": 0,+ "completed_24h": 0,+ }++ mock_exec = MagicMock()+ mock_exec.started_at = None++ with (+ patch(+ "backend.data.diagnostics.query_raw_with_schema",+ new_callable=AsyncMock,+ return_value=[mock_row],+ ),+ patch(+ "backend.data.diagnostics.get_rabbitmq_queue_depth",+ return_value=0,+ ),+ patch(+ "backend.data.diagnostics.get_rabbitmq_cancel_queue_depth",+ return_value=0,+ ),+ patch(+ "backend.data.diagnostics.get_graph_executions",+ new_callable=AsyncMock,+ return_value=[mock_exec],+ ),+ ):+ result = await get_execution_diagnostics()++ assert result.oldest_running_hours is None+++# ---------------------------------------------------------------------------+# RabbitMQ queue depth tests+# ---------------------------------------------------------------------------+++def test_rabbitmq_queue_depth_success():+ """Test successful RabbitMQ queue depth retrieval."""+ mock_method_frame = MagicMock()+ mock_method_frame.method.message_count = 42++ mock_channel = MagicMock()+ mock_channel.queue_declare.return_value = mock_method_frame++ mock_rabbitmq = MagicMock()+ mock_rabbitmq._channel = mock_channel++ with (+ patch(+ "backend.data.diagnostics.create_execution_queue_config",+ return_value=MagicMock(),+ ),+ patch(+ "backend.data.diagnostics.SyncRabbitMQ",+ return_value=mock_rabbitmq,+ ),+ ):+ result = get_rabbitmq_queue_depth()++ assert result == 42+ mock_rabbitmq.connect.assert_called_once()+ mock_rabbitmq.disconnect.assert_called_once()+++def test_rabbitmq_queue_depth_connection_error():+ """Test RabbitMQ queue depth returns -1 on connection error."""+ mock_rabbitmq = MagicMock()+ mock_rabbitmq.connect.side_effect = Exception("Connection refused")++ with (+ patch(+ "backend.data.diagnostics.create_execution_queue_config",+ return_value=MagicMock(),+ ),+ patch(+ "backend.data.diagnostics.SyncRabbitMQ",+ return_value=mock_rabbitmq,+ ),+ ):+ result = get_rabbitmq_queue_depth()++ assert result == -1+++def test_rabbitmq_queue_depth_no_channel():+ """Test RabbitMQ queue depth when channel is None."""+ mock_rabbitmq = MagicMock()+ mock_rabbitmq._channel = None++ with (+ patch(+ "backend.data.diagnostics.create_execution_queue_config",+ return_value=MagicMock(),+ ),+ patch(+ "backend.data.diagnostics.SyncRabbitMQ",+ return_value=mock_rabbitmq,+ ),+ ):+ result = get_rabbitmq_queue_depth()++ # Should return -1 because RuntimeError is caught+ assert result == -1+++def test_rabbitmq_cancel_queue_depth_success():+ """Test successful RabbitMQ cancel queue depth retrieval."""+ mock_method_frame = MagicMock()+ mock_method_frame.method.message_count = 5++ mock_channel = MagicMock()+ mock_channel.queue_declare.return_value = mock_method_frame++ mock_rabbitmq = MagicMock()+ mock_rabbitmq._channel = mock_channel++ with (+ patch(+ "backend.data.diagnostics.create_execution_queue_config",+ return_value=MagicMock(),+ ),+ patch(+ "backend.data.diagnostics.SyncRabbitMQ",+ return_value=mock_rabbitmq,+ ),+ ):+ result = get_rabbitmq_cancel_queue_depth()++ assert result == 5+++def test_rabbitmq_cancel_queue_depth_error():+ """Test RabbitMQ cancel queue depth returns -1 on error."""+ mock_rabbitmq = MagicMock()+ mock_rabbitmq.connect.side_effect = Exception("Connection refused")++ with (+ patch(+ "backend.data.diagnostics.create_execution_queue_config",+ return_value=MagicMock(),+ ),+ patch(+ "backend.data.diagnostics.SyncRabbitMQ",+ return_value=mock_rabbitmq,+ ),+ ):+ result = get_rabbitmq_cancel_queue_depth()++ assert result == -1+++def test_rabbitmq_disconnect_error_handled():+ """Test that disconnect errors are handled gracefully."""+ mock_method_frame = MagicMock()+ mock_method_frame.method.message_count = 10++ mock_channel = MagicMock()+ mock_channel.queue_declare.return_value = mock_method_frame++ mock_rabbitmq = MagicMock()+ mock_rabbitmq._channel = mock_channel+ mock_rabbitmq.disconnect.side_effect = Exception("Disconnect failed")++ with (+ patch(+ "backend.data.diagnostics.create_execution_queue_config",+ return_value=MagicMock(),+ ),+ patch(+ "backend.data.diagnostics.SyncRabbitMQ",+ return_value=mock_rabbitmq,+ ),+ ):+ # Should still return the count even if disconnect fails+ result = get_rabbitmq_queue_depth()++ assert result == 10+++# ---------------------------------------------------------------------------+# _calculate_total_runs tests+# ---------------------------------------------------------------------------+++def test_calculate_total_runs_basic():+ """Test calculating total runs with a simple cron (every hour)."""+ now = datetime(2026, 4, 17, 0, 0, 0, tzinfo=timezone.utc)+ end = now + timedelta(hours=3)++ schedule = MagicMock()+ schedule.cron = "0 * * * *" # Every hour++ result = _calculate_total_runs([schedule], now, end)+ assert result == 3 # 01:00, 02:00, 03:00+++def test_calculate_total_runs_invalid_cron():+ """Test that invalid cron expressions are skipped."""+ now = datetime(2026, 4, 17, 0, 0, 0, tzinfo=timezone.utc)+ end = now + timedelta(hours=1)++ schedule = MagicMock()+ schedule.cron = "invalid cron expression"++ result = _calculate_total_runs([schedule], now, end)+ assert result == 0+++def test_calculate_total_runs_multiple_schedules():+ """Test total runs across multiple schedules."""+ now = datetime(2026, 4, 17, 0, 0, 0, tzinfo=timezone.utc)+ end = now + timedelta(hours=2)++ sched1 = MagicMock()+ sched1.cron = "0 * * * *" # Every hour++ sched2 = MagicMock()+ sched2.cron = "*/30 * * * *" # Every 30 min++ result = _calculate_total_runs([sched1, sched2], now, end)+ # sched1: 01:00, 02:00 = 2+ # sched2: 00:30, 01:00, 01:30, 02:00 = 4+ assert result == 6+++def test_calculate_total_runs_empty():+ """Test with no schedules."""+ now = datetime(2026, 4, 17, 0, 0, 0, tzinfo=timezone.utc)+ end = now + timedelta(hours=1)++ result = _calculate_total_runs([], now, end)+ assert result == 0+++# ---------------------------------------------------------------------------+# _detect_orphaned_schedules tests+# ---------------------------------------------------------------------------+++async def test_detect_orphaned_schedules_deleted_graph():+ """Test detection of schedules with deleted graphs."""+ schedule = MagicMock()+ schedule.id = "sched-1"+ schedule.graph_id = "graph-deleted"+ schedule.graph_version = 1+ schedule.user_id = "user-1"++ with patch("backend.data.diagnostics.AgentGraph.prisma") as mock_graph_prisma:+ mock_graph_prisma.return_value.find_unique = AsyncMock(return_value=None)++ result = await _detect_orphaned_schedules([schedule])++ assert "sched-1" in result["deleted_graph"]+ assert len(result["no_library_access"]) == 0+++async def test_detect_orphaned_schedules_no_library_access():+ """Test detection of schedules where user lost library access."""+ schedule = MagicMock()+ schedule.id = "sched-2"+ schedule.graph_id = "graph-1"+ schedule.graph_version = 1+ schedule.user_id = "user-2"++ mock_graph = MagicMock()++ with (+ patch("backend.data.diagnostics.AgentGraph.prisma") as mock_graph_prisma,+ patch("backend.data.diagnostics.LibraryAgent.prisma") as mock_lib_prisma,+ ):+ mock_graph_prisma.return_value.find_unique = AsyncMock(return_value=mock_graph)+ mock_lib_prisma.return_value.find_first = AsyncMock(return_value=None)++ result = await _detect_orphaned_schedules([schedule])++ assert "sched-2" in result["no_library_access"]+ assert len(result["deleted_graph"]) == 0+++async def test_detect_orphaned_schedules_validation_error():+ """Test detection of schedules that fail validation."""+ schedule = MagicMock()+ schedule.id = "sched-3"+ schedule.graph_id = "graph-1"+ schedule.graph_version = 1+ schedule.user_id = "user-3"++ with patch("backend.data.diagnostics.AgentGraph.prisma") as mock_graph_prisma:+ mock_graph_prisma.return_value.find_unique = AsyncMock(+ side_effect=Exception("DB connection error")+ )++ result = await _detect_orphaned_schedules([schedule])++ assert "sched-3" in result["validation_failed"]+++async def test_detect_orphaned_schedules_healthy():+ """Test that healthy schedules are not flagged."""+ schedule = MagicMock()+ schedule.id = "sched-ok"+ schedule.graph_id = "graph-1"+ schedule.graph_version = 1+ schedule.user_id = "user-1"++ mock_graph = MagicMock()+ mock_library_agent = MagicMock()++ with (+ patch("backend.data.diagnostics.AgentGraph.prisma") as mock_graph_prisma,+ patch("backend.data.diagnostics.LibraryAgent.prisma") as mock_lib_prisma,+ ):+ mock_graph_prisma.return_value.find_unique = AsyncMock(return_value=mock_graph)+ mock_lib_prisma.return_value.find_first = AsyncMock(+ return_value=mock_library_agent+ )++ result = await _detect_orphaned_schedules([schedule])++ assert len(result["deleted_graph"]) == 0+ assert len(result["no_library_access"]) == 0+ assert len(result["validation_failed"]) == 0diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.pyindex f4b341291b..4403a59080 100644--- a/autogpt_platform/backend/backend/data/execution.py+++ b/autogpt_platform/backend/backend/data/execution.py@@ -26,6 +26,7 @@ from prisma.models import (AgentNodeExecutionKeyValueData,)from prisma.types import (+ AgentGraphExecutionOrderByInput,AgentGraphExecutionUpdateManyMutationInput,AgentGraphExecutionWhereInput,AgentNodeExecutionCreateInput,@@ -510,20 +511,39 @@ class NodeExecutionResult(BaseModel):async def get_graph_executions(graph_exec_id: Optional[str] = None,+ execution_ids: Optional[list[str]] = None,graph_id: Optional[str] = None,graph_version: Optional[int] = None,user_id: Optional[str] = None,statuses: Optional[list[ExecutionStatus]] = None,created_time_gte: Optional[datetime] = None,created_time_lte: Optional[datetime] = None,+ started_time_gte: Optional[datetime] = None,+ started_time_lte: Optional[datetime] = None,limit: Optional[int] = None,+ offset: Optional[int] = None,+ order_by: Literal["createdAt", "startedAt", "updatedAt"] = "createdAt",+ order_direction: Literal["asc", "desc"] = "desc",) -> list[GraphExecutionMeta]:- """⚠️ **Optional `user_id` check**: MUST USE check in user-facing endpoints."""+ """+ Get graph executions with optional filters and ordering.++ ⚠️ **Optional `user_id` check**: MUST USE check in user-facing endpoints.++ Args:+ graph_exec_id: Filter by single execution ID (mutually exclusive with execution_ids)+ execution_ids: Filter by list of execution IDs (mutually exclusive with graph_exec_id)+ order_by: Field to order by. Defaults to "createdAt"+ order_direction: Sort direction. Defaults to "desc"+ """where_filter: AgentGraphExecutionWhereInput = {"isDeleted": False,}if graph_exec_id:where_filter["id"] = graph_exec_id+ elif execution_ids:+ where_filter["id"] = {"in": execution_ids}+if user_id:where_filter["userId"] = user_idif graph_id:@@ -535,13 +555,36 @@ async def get_graph_executions("gte": created_time_gte or datetime.min.replace(tzinfo=timezone.utc),"lte": created_time_lte or datetime.max.replace(tzinfo=timezone.utc),}+ if started_time_gte or started_time_lte:+ where_filter["startedAt"] = {+ "gte": started_time_gte or datetime.min.replace(tzinfo=timezone.utc),+ "lte": started_time_lte or datetime.max.replace(tzinfo=timezone.utc),+ }if statuses:where_filter["OR"] = [{"executionStatus": status} for status in statuses]+ # Build properly typed order clause+ # Prisma wants specific typed dicts for each field, so we construct them explicitly+ order_clause: AgentGraphExecutionOrderByInput+ match (order_by):+ case "startedAt":+ order_clause = {+ "startedAt": order_direction,+ }+ case "updatedAt":+ order_clause = {+ "updatedAt": order_direction,+ }+ case _:+ order_clause = {+ "createdAt": order_direction,+ }+executions = await AgentGraphExecution.prisma().find_many(where=where_filter,- order={"createdAt": "desc"},+ order=order_clause,take=limit,+ skip=offset,)return [GraphExecutionMeta.from_db(execution) for execution in executions]@@ -552,6 +595,10 @@ async def get_graph_executions_count(statuses: Optional[list[ExecutionStatus]] = None,created_time_gte: Optional[datetime] = None,created_time_lte: Optional[datetime] = None,+ started_time_gte: Optional[datetime] = None,+ started_time_lte: Optional[datetime] = None,+ updated_time_gte: Optional[datetime] = None,+ updated_time_lte: Optional[datetime] = None,) -> int:"""Get count of graph executions with optional filters.@@ -562,6 +609,10 @@ async def get_graph_executions_count(statuses: Optional list of execution statuses to filter bycreated_time_gte: Optional minimum creation timecreated_time_lte: Optional maximum creation time+ started_time_gte: Optional minimum start time (when execution started running)+ started_time_lte: Optional maximum start time (when execution started running)+ updated_time_gte: Optional minimum update time+ updated_time_lte: Optional maximum update timeReturns:Count of matching graph executions@@ -581,6 +632,19 @@ async def get_graph_executions_count("gte": created_time_gte or datetime.min.replace(tzinfo=timezone.utc),"lte": created_time_lte or datetime.max.replace(tzinfo=timezone.utc),}++ if started_time_gte or started_time_lte:+ where_filter["startedAt"] = {+ "gte": started_time_gte or datetime.min.replace(tzinfo=timezone.utc),+ "lte": started_time_lte or datetime.max.replace(tzinfo=timezone.utc),+ }++ if updated_time_gte or updated_time_lte:+ where_filter["updatedAt"] = {+ "gte": updated_time_gte or datetime.min.replace(tzinfo=timezone.utc),+ "lte": updated_time_lte or datetime.max.replace(tzinfo=timezone.utc),+ }+if statuses:where_filter["OR"] = [{"executionStatus": status} for status in statuses]diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.pyindex 8774ff03ef..24da0b3c7b 100644--- a/autogpt_platform/backend/backend/executor/utils.py+++ b/autogpt_platform/backend/backend/executor/utils.py@@ -919,6 +919,10 @@ async def add_graph_execution("""Adds a graph execution to the queue and returns the execution entry.+ Supports two modes:+ 1. CREATE mode (graph_exec_id=None): Validates, creates new DB entry, and queues+ 2. REQUEUE mode (graph_exec_id provided): Fetches existing execution and re-queues it+Args:graph_id: The ID of the graph to execute.user_id: The ID of the user executing the graph.@@ -931,7 +935,7 @@ async def add_graph_execution(parent_graph_exec_id: The ID of the parent graph execution (for nested executions).graph_exec_id: If provided, resume this existing execution instead of creating a new one.Returns:- GraphExecutionEntry: The entry for the graph execution.+ GraphExecutionWithNodes: The execution entry.Raises:ValueError: If the graph is not found or if there are validation errors.NotFoundError: If graph_exec_id is provided but execution is not found.diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/__tests__/layout.test.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/__tests__/layout.test.tsxnew file mode 100644index 0000000000..d0ea04602b--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/admin/__tests__/layout.test.tsx@@ -0,0 +1,53 @@+import { render, screen } from "@/tests/integrations/test-utils";+import { describe, expect, it, vi } from "vitest";+import AdminLayout from "../layout";++vi.mock("@/components/__legacy__/Sidebar", () => ({+ Sidebar: ({+ linkGroups,+ }: {+ linkGroups: { links: { text: string }[] }[];+ }) => (+ <nav data-testid="sidebar">+ {linkGroups[0].links.map((link) => (+ <span key={link.text}>{link.text}</span>+ ))}+ </nav>+ ),+}));++describe("AdminLayout", () => {+ it("renders sidebar with System Diagnostics link", () => {+ render(+ <AdminLayout>+ <div>Child Content</div>+ </AdminLayout>,+ );+ expect(screen.getByText("System Diagnostics")).toBeDefined();+ });++ it("renders child content", () => {+ render(+ <AdminLayout>+ <div>Test Child</div>+ </AdminLayout>,+ );+ expect(screen.getByText("Test Child")).toBeDefined();+ });++ it("renders all admin navigation links", () => {+ render(+ <AdminLayout>+ <div />+ </AdminLayout>,+ );+ expect(screen.getByText("Marketplace Management")).toBeDefined();+ expect(screen.getByText("User Spending")).toBeDefined();+ expect(screen.getByText("System Diagnostics")).toBeDefined();+ expect(screen.getByText("User Impersonation")).toBeDefined();+ expect(screen.getByText("Rate Limits")).toBeDefined();+ expect(screen.getByText("Platform Costs")).toBeDefined();+ expect(screen.getByText("Execution Analytics")).toBeDefined();+ expect(screen.getByText("Admin User Management")).toBeDefined();+ });+});diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/__tests__/DiagnosticsContent.test.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/__tests__/DiagnosticsContent.test.tsxnew file mode 100644index 0000000000..b4b0b843af--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/__tests__/DiagnosticsContent.test.tsx@@ -0,0 +1,540 @@+import {+ render,+ screen,+ cleanup,+ fireEvent,+} from "@/tests/integrations/test-utils";+import { afterEach, describe, expect, it, vi } from "vitest";+import { DiagnosticsContent } from "../components/DiagnosticsContent";++// Mock the generated API hooks directly so useDiagnosticsContent code is exercised+const mockExecQuery = vi.fn();+const mockAgentQuery = vi.fn();+const mockScheduleQuery = vi.fn();++vi.mock("@/app/api/__generated__/endpoints/admin/admin", () => ({+ useGetV2GetExecutionDiagnostics: () => mockExecQuery(),+ useGetV2GetAgentDiagnostics: () => mockAgentQuery(),+ useGetV2GetScheduleDiagnostics: () => mockScheduleQuery(),+ useGetV2ListRunningExecutions: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListOrphanedExecutions: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListFailedExecutions: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListLongRunningExecutions: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListStuckQueuedExecutions: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListInvalidExecutions: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ usePostV2StopSingleExecution: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2StopMultipleExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2StopAllLongRunningExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2CleanupOrphanedExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2CleanupAllOrphanedExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2CleanupAllStuckQueuedExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2RequeueStuckExecution: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2RequeueMultipleStuckExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2RequeueAllStuckQueuedExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ useGetV2ListAllUserSchedules: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListOrphanedSchedules: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ usePostV2CleanupOrphanedSchedules: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+}));++afterEach(() => {+ cleanup();+ mockExecQuery.mockReset();+ mockAgentQuery.mockReset();+ mockScheduleQuery.mockReset();+});++const executionData = {+ running_executions: 10,+ queued_executions_db: 5,+ queued_executions_rabbitmq: 3,+ cancel_queue_depth: 0,+ orphaned_running: 2,+ orphaned_queued: 1,+ failed_count_1h: 5,+ failed_count_24h: 20,+ failure_rate_24h: 0.83,+ stuck_running_24h: 3,+ stuck_running_1h: 5,+ oldest_running_hours: 26.5,+ stuck_queued_1h: 2,+ queued_never_started: 1,+ invalid_queued_with_start: 1,+ invalid_running_without_start: 1,+ completed_1h: 50,+ completed_24h: 1200,+ throughput_per_hour: 50.0,+ timestamp: "2026-04-17T00:00:00Z",+};++const agentData = {+ agents_with_active_executions: 7,+ timestamp: "2026-04-17T00:00:00Z",+};++const scheduleData = {+ total_schedules: 15,+ user_schedules: 10,+ system_schedules: 5,+ orphaned_deleted_graph: 2,+ orphaned_no_library_access: 1,+ orphaned_invalid_credentials: 0,+ orphaned_validation_failed: 0,+ total_orphaned: 3,+ schedules_next_hour: 4,+ schedules_next_24h: 8,+ total_runs_next_hour: 12,+ total_runs_next_24h: 48,+ timestamp: "2026-04-17T00:00:00Z",+};++function setupLoadedMocks() {+ mockExecQuery.mockReturnValue({+ data: { data: executionData },+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockAgentQuery.mockReturnValue({+ data: { data: agentData },+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockScheduleQuery.mockReturnValue({+ data: { data: scheduleData },+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+}++function setupLoadingMocks() {+ mockExecQuery.mockReturnValue({+ data: undefined,+ isLoading: true,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockAgentQuery.mockReturnValue({+ data: undefined,+ isLoading: true,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockScheduleQuery.mockReturnValue({+ data: undefined,+ isLoading: true,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+}++function setupErrorMocks() {+ mockExecQuery.mockReturnValue({+ data: undefined,+ isLoading: false,+ isError: true,+ error: { status: 500, message: "Server error" },+ refetch: vi.fn(),+ });+ mockAgentQuery.mockReturnValue({+ data: undefined,+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockScheduleQuery.mockReturnValue({+ data: undefined,+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+}++describe("DiagnosticsContent", () => {+ it("shows loading state", () => {+ setupLoadingMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText("Loading diagnostics...")).toBeDefined();+ });++ it("shows error state with retry", () => {+ setupErrorMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText("Try Again")).toBeDefined();+ });++ it("renders system diagnostics heading with data", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText("System Diagnostics")).toBeDefined();+ expect(screen.getByText("Refresh")).toBeDefined();+ });++ it("renders execution queue status cards", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText("Execution Queue Status")).toBeDefined();+ expect(screen.getByText("Running Executions")).toBeDefined();+ expect(screen.getByText("Queued in Database")).toBeDefined();+ expect(screen.getByText("Queued in RabbitMQ")).toBeDefined();+ });++ it("renders throughput metrics", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText("System Throughput")).toBeDefined();+ expect(screen.getByText("Completed (24h)")).toBeDefined();+ expect(screen.getByText("Throughput Rate")).toBeDefined();+ expect(screen.getByText("50.0")).toBeDefined();+ });++ it("renders schedule summary card", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText("User Schedules")).toBeDefined();+ expect(screen.getByText("Upcoming Runs (1h)")).toBeDefined();+ expect(screen.getByText("Upcoming Runs (24h)")).toBeDefined();+ });++ it("renders alert cards for critical issues", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText("Orphaned Executions")).toBeDefined();+ expect(screen.getByText("Failed Executions (24h)")).toBeDefined();+ expect(screen.getByText("Long-Running Executions")).toBeDefined();+ expect(screen.getByText("Orphaned Schedules")).toBeDefined();+ expect(screen.getByText("Invalid States (Data Corruption)")).toBeDefined();+ });++ it("hides alert cards when counts are zero", () => {+ mockExecQuery.mockReturnValue({+ data: {+ data: {+ ...executionData,+ orphaned_running: 0,+ orphaned_queued: 0,+ failed_count_24h: 0,+ stuck_running_24h: 0,+ invalid_queued_with_start: 0,+ invalid_running_without_start: 0,+ },+ },+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockAgentQuery.mockReturnValue({+ data: { data: agentData },+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockScheduleQuery.mockReturnValue({+ data: { data: { ...scheduleData, total_orphaned: 0 } },+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ render(<DiagnosticsContent />);+ expect(screen.queryByText("Orphaned Executions")).toBeNull();+ expect(screen.queryByText("Failed Executions (24h)")).toBeNull();+ expect(screen.queryByText("Long-Running Executions")).toBeNull();+ expect(screen.queryByText("Orphaned Schedules")).toBeNull();+ expect(screen.queryByText("Invalid States (Data Corruption)")).toBeNull();+ });++ it("renders diagnostic information section", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText("Diagnostic Information")).toBeDefined();+ expect(screen.getByText("Throughput Metrics:")).toBeDefined();+ expect(screen.getByText("Queue Health:")).toBeDefined();+ });++ it("shows no data message when execution data is null", () => {+ mockExecQuery.mockReturnValue({+ data: undefined,+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockAgentQuery.mockReturnValue({+ data: undefined,+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockScheduleQuery.mockReturnValue({+ data: undefined,+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ render(<DiagnosticsContent />);+ const noDataMessages = screen.getAllByText("No data available");+ expect(noDataMessages.length).toBeGreaterThanOrEqual(1);+ });++ it("shows RabbitMQ error state when depth is -1", () => {+ mockExecQuery.mockReturnValue({+ data: {+ data: { ...executionData, queued_executions_rabbitmq: -1 },+ },+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockAgentQuery.mockReturnValue({+ data: { data: agentData },+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockScheduleQuery.mockReturnValue({+ data: { data: scheduleData },+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ render(<DiagnosticsContent />);+ const errorTexts = screen.getAllByText("Error");+ expect(errorTexts.length).toBeGreaterThanOrEqual(1);+ });++ it("renders completed 24h and 1h values", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText("1200")).toBeDefined();+ expect(screen.getByText("50 in last hour")).toBeDefined();+ });++ it("renders schedule metric values", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText("12")).toBeDefined();+ expect(screen.getByText("48")).toBeDefined();+ });++ it("renders oldest running hours in alert card", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText(/oldest:.*26h/)).toBeDefined();+ });++ it("renders cancel queue depth error when -1", () => {+ mockExecQuery.mockReturnValue({+ data: {+ data: { ...executionData, cancel_queue_depth: -1 },+ },+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockAgentQuery.mockReturnValue({+ data: { data: agentData },+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ mockScheduleQuery.mockReturnValue({+ data: { data: scheduleData },+ isLoading: false,+ isError: false,+ error: null,+ refetch: vi.fn(),+ });+ render(<DiagnosticsContent />);+ const errorTexts = screen.getAllByText("Error");+ expect(errorTexts.length).toBeGreaterThanOrEqual(1);+ });++ it("renders stuck queued count in queue status card", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText(/2 stuck/)).toBeDefined();+ });++ it("renders schedule orphaned count in card", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText(/3 orphaned/)).toBeDefined();+ });++ it("clicking orphaned alert card does not crash", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ fireEvent.click(screen.getByText("Orphaned Executions"));+ });++ it("clicking failed alert card does not crash", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ fireEvent.click(screen.getByText("Failed Executions (24h)"));+ });++ it("clicking long-running alert card does not crash", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ fireEvent.click(screen.getByText("Long-Running Executions"));+ });++ it("clicking orphaned schedules alert card does not crash", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ fireEvent.click(screen.getByText("Orphaned Schedules"));+ });++ it("clicking invalid states alert card does not crash", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ fireEvent.click(screen.getByText("Invalid States (Data Corruption)"));+ });++ it("renders orphan detail text in schedule alert", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText(/2 deleted graph/)).toBeDefined();+ expect(screen.getByText(/1 no access/)).toBeDefined();+ });++ it("renders failure rate in failed alert card", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText(/0.8\/hr rate/)).toBeDefined();+ });++ it("renders click to view text on alert cards", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ const clickTexts = screen.getAllByText(/Click to view/);+ expect(clickTexts.length).toBeGreaterThanOrEqual(3);+ });++ it("renders schedule next hour count", () => {+ setupLoadedMocks();+ render(<DiagnosticsContent />);+ expect(screen.getByText(/from 4 schedules/)).toBeDefined();+ });++ it("clicking Refresh button calls all refetch functions", () => {+ const refetchExec = vi.fn();+ const refetchAgent = vi.fn();+ const refetchSchedule = vi.fn();+ mockExecQuery.mockReturnValue({+ data: { data: executionData },+ isLoading: false,+ isError: false,+ error: null,+ refetch: refetchExec,+ });+ mockAgentQuery.mockReturnValue({+ data: { data: agentData },+ isLoading: false,+ isError: false,+ error: null,+ refetch: refetchAgent,+ });+ mockScheduleQuery.mockReturnValue({+ data: { data: scheduleData },+ isLoading: false,+ isError: false,+ error: null,+ refetch: refetchSchedule,+ });+ render(<DiagnosticsContent />);+ fireEvent.click(screen.getByText("Refresh"));+ expect(refetchExec).toHaveBeenCalled();+ expect(refetchAgent).toHaveBeenCalled();+ expect(refetchSchedule).toHaveBeenCalled();+ });+});diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/__tests__/ExecutionsTable.test.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/__tests__/ExecutionsTable.test.tsxnew file mode 100644index 0000000000..e116d220e2--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/__tests__/ExecutionsTable.test.tsx@@ -0,0 +1,1258 @@+import {+ render,+ screen,+ cleanup,+ fireEvent,+ waitFor,+} from "@/tests/integrations/test-utils";+import { afterEach, describe, expect, it, vi } from "vitest";+import { ExecutionsTable } from "../components/ExecutionsTable";++const mockRunningQuery = vi.fn();+const mockOrphanedQuery = vi.fn();+const mockFailedQuery = vi.fn();+const mockLongRunningQuery = vi.fn();+const mockStuckQueuedQuery = vi.fn();+const mockInvalidQuery = vi.fn();+const mockStopSingle = vi.fn();+const mockStopMultiple = vi.fn();+const mockStopAllLongRunning = vi.fn();+const mockCleanupOrphaned = vi.fn();+const mockCleanupAllOrphaned = vi.fn();+const mockCleanupAllStuckQueued = vi.fn();+const mockRequeueSingle = vi.fn();+const mockRequeueMultiple = vi.fn();+const mockRequeueAllStuck = vi.fn();++vi.mock("@/app/api/__generated__/endpoints/admin/admin", () => ({+ useGetV2ListRunningExecutions: (...args: unknown[]) =>+ mockRunningQuery(...args),+ useGetV2ListOrphanedExecutions: (...args: unknown[]) =>+ mockOrphanedQuery(...args),+ useGetV2ListFailedExecutions: (...args: unknown[]) =>+ mockFailedQuery(...args),+ useGetV2ListLongRunningExecutions: (...args: unknown[]) =>+ mockLongRunningQuery(...args),+ useGetV2ListStuckQueuedExecutions: (...args: unknown[]) =>+ mockStuckQueuedQuery(...args),+ useGetV2ListInvalidExecutions: (...args: unknown[]) =>+ mockInvalidQuery(...args),+ usePostV2StopSingleExecution: () => ({+ mutateAsync: mockStopSingle,+ isPending: false,+ }),+ usePostV2StopMultipleExecutions: () => ({+ mutateAsync: mockStopMultiple,+ isPending: false,+ }),+ usePostV2StopAllLongRunningExecutions: () => ({+ mutateAsync: mockStopAllLongRunning,+ isPending: false,+ }),+ usePostV2CleanupOrphanedExecutions: () => ({+ mutateAsync: mockCleanupOrphaned,+ isPending: false,+ }),+ usePostV2CleanupAllOrphanedExecutions: () => ({+ mutateAsync: mockCleanupAllOrphaned,+ isPending: false,+ }),+ usePostV2CleanupAllStuckQueuedExecutions: () => ({+ mutateAsync: mockCleanupAllStuckQueued,+ isPending: false,+ }),+ usePostV2RequeueStuckExecution: () => ({+ mutateAsync: mockRequeueSingle,+ isPending: false,+ }),+ usePostV2RequeueMultipleStuckExecutions: () => ({+ mutateAsync: mockRequeueMultiple,+ isPending: false,+ }),+ usePostV2RequeueAllStuckQueuedExecutions: () => ({+ mutateAsync: mockRequeueAllStuck,+ isPending: false,+ }),+}));++function defaultQueryReturn(overrides = {}) {+ return {+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ ...overrides,+ };+}++function withExecutions(+ executions: Record<string, unknown>[],+ total: number,+ overrides = {},+) {+ return defaultQueryReturn({+ data: { data: { executions, total } },+ ...overrides,+ });+}++const sampleExecution = {+ execution_id: "exec-001",+ graph_id: "graph-123",+ graph_name: "Test Agent",+ graph_version: 1,+ user_id: "user-abc",+ user_email: "[email protected]",+ status: "RUNNING",+ created_at: "2026-04-16T10:00:00Z",+ started_at: "2026-04-16T10:01:00Z",+ queue_status: null,+};++const diagnosticsData = {+ orphaned_running: 2,+ orphaned_queued: 1,+ failed_count_24h: 5,+ stuck_running_24h: 3,+ stuck_queued_1h: 2,+ invalid_queued_with_start: 1,+ invalid_running_without_start: 1,+};++function setupDefaultMocks() {+ mockRunningQuery.mockReturnValue(defaultQueryReturn());+ mockOrphanedQuery.mockReturnValue(defaultQueryReturn());+ mockFailedQuery.mockReturnValue(defaultQueryReturn());+ mockLongRunningQuery.mockReturnValue(defaultQueryReturn());+ mockStuckQueuedQuery.mockReturnValue(defaultQueryReturn());+ mockInvalidQuery.mockReturnValue(defaultQueryReturn());+}++afterEach(() => {+ cleanup();+ mockRunningQuery.mockReset();+ mockOrphanedQuery.mockReset();+ mockFailedQuery.mockReset();+ mockLongRunningQuery.mockReset();+ mockStuckQueuedQuery.mockReset();+ mockInvalidQuery.mockReset();+});++describe("ExecutionsTable", () => {+ it("shows empty state when no executions", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([], 0));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("No running executions")).toBeDefined();+ });++ it("renders execution rows in all tab", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([sampleExecution], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("Test Agent")).toBeDefined();+ expect(screen.getByText("[email protected]")).toBeDefined();+ expect(screen.getByText("RUNNING")).toBeDefined();+ });++ it("shows loading spinner", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(defaultQueryReturn({ isLoading: true }));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(document.querySelector(".animate-spin")).toBeDefined();+ });++ it("renders tab triggers with counts from diagnostics data", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([], 0));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText(/Orphaned/)).toBeDefined();+ expect(screen.getByText(/Failed/)).toBeDefined();+ expect(screen.getByText(/Long-Running/)).toBeDefined();+ expect(screen.getByText(/Stuck Queued/)).toBeDefined();+ expect(screen.getByText(/Invalid/)).toBeDefined();+ });++ it("renders error state", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(+ defaultQueryReturn({ error: { status: 500, message: "Server down" } }),+ );+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("Try Again")).toBeDefined();+ });++ it("renders failed execution with error message", () => {+ setupDefaultMocks();+ const failedExec = {+ ...sampleExecution,+ execution_id: "exec-fail-1",+ status: "FAILED",+ failed_at: "2026-04-16T12:00:00Z",+ error_message: "Out of memory",+ };+ mockRunningQuery.mockReturnValue(withExecutions([], 0));+ mockFailedQuery.mockReturnValue(withExecutions([failedExec], 1));+ render(+ <ExecutionsTable diagnosticsData={diagnosticsData} initialTab="failed" />,+ );+ expect(screen.getByText("Out of memory")).toBeDefined();+ });++ it("renders pagination when total exceeds page size", () => {+ setupDefaultMocks();+ const executions = Array.from({ length: 10 }, (_, i) => ({+ ...sampleExecution,+ execution_id: `exec-${i}`,+ }));+ mockRunningQuery.mockReturnValue(withExecutions(executions, 25));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText(/Page 1 of 3/)).toBeDefined();+ expect(screen.getByText("Previous")).toBeDefined();+ expect(screen.getByText("Next")).toBeDefined();+ });++ it("shows unknown for null user email", () => {+ setupDefaultMocks();+ const noEmailExec = {+ ...sampleExecution,+ user_email: null,+ };+ mockRunningQuery.mockReturnValue(withExecutions([noEmailExec], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("Unknown")).toBeDefined();+ });++ it("copies execution ID to clipboard on click", () => {+ const writeText = vi.fn().mockResolvedValue(undefined);+ vi.stubGlobal("navigator", { ...navigator, clipboard: { writeText } });+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([sampleExecution], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ fireEvent.click(screen.getByText("exec-001".substring(0, 8) + "..."));+ expect(writeText).toHaveBeenCalledWith("exec-001");+ vi.unstubAllGlobals();+ });++ it("copies user ID to clipboard on click", () => {+ const writeText = vi.fn().mockResolvedValue(undefined);+ vi.stubGlobal("navigator", { ...navigator, clipboard: { writeText } });+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([sampleExecution], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ fireEvent.click(screen.getByText("user-abc".substring(0, 8) + "..."));+ expect(writeText).toHaveBeenCalledWith("user-abc");+ vi.unstubAllGlobals();+ });++ it("shows never started for null started_at", () => {+ setupDefaultMocks();+ const neverStarted = {+ ...sampleExecution,+ started_at: null,+ };+ mockRunningQuery.mockReturnValue(withExecutions([neverStarted], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("Never started")).toBeDefined();+ });++ it("renders stuck-queued tab with requeue buttons", () => {+ setupDefaultMocks();+ const stuckExec = {+ ...sampleExecution,+ execution_id: "exec-stuck-1",+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ expect(screen.getByTitle("Cleanup (mark as FAILED)")).toBeDefined();+ expect(screen.getByTitle("Requeue (send to RabbitMQ)")).toBeDefined();+ });++ it("renders orphaned tab executions", () => {+ setupDefaultMocks();+ const orphanedExec = {+ ...sampleExecution,+ execution_id: "exec-orphan-1",+ created_at: "2026-04-10T10:00:00Z",+ };+ mockOrphanedQuery.mockReturnValue(withExecutions([orphanedExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="orphaned"+ />,+ );+ expect(screen.getByText("Test Agent")).toBeDefined();+ });++ it("renders long-running tab executions", () => {+ setupDefaultMocks();+ mockLongRunningQuery.mockReturnValue(withExecutions([sampleExecution], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="long-running"+ />,+ );+ expect(screen.getByText("Test Agent")).toBeDefined();+ });++ it("renders invalid tab executions", () => {+ setupDefaultMocks();+ const invalidExec = {+ ...sampleExecution,+ execution_id: "exec-invalid-1",+ status: "QUEUED",+ started_at: "2026-04-16T10:01:00Z",+ };+ mockInvalidQuery.mockReturnValue(withExecutions([invalidExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="invalid"+ />,+ );+ expect(screen.getByText("QUEUED")).toBeDefined();+ });++ it("renders all tab trigger labels with correct counts", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([], 0));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText(/Orphaned.*3/)).toBeDefined();+ expect(screen.getByText(/Failed.*5/)).toBeDefined();+ expect(screen.getByText(/Stuck Queued.*2/)).toBeDefined();+ expect(screen.getByText(/Long-Running.*3/)).toBeDefined();+ expect(screen.getByText(/Invalid States.*2/)).toBeDefined();+ });++ it("shows graph version number", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([sampleExecution], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("1")).toBeDefined();+ });++ it("renders QUEUED status badge", () => {+ setupDefaultMocks();+ const queuedExec = { ...sampleExecution, status: "QUEUED" };+ mockRunningQuery.mockReturnValue(withExecutions([queuedExec], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("QUEUED")).toBeDefined();+ });++ it("renders without diagnosticsData", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([], 0));+ render(<ExecutionsTable />);+ expect(screen.getByText(/All/)).toBeDefined();+ });++ it("renders stuck-queued bulk action buttons when total > 0", () => {+ setupDefaultMocks();+ const stuckExec = {+ ...sampleExecution,+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 5));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ expect(screen.getByText(/Cleanup All \(5\)/)).toBeDefined();+ expect(screen.getByText(/Requeue All \(5\)/)).toBeDefined();+ });++ it("renders long-running stop all button when total > 0", () => {+ setupDefaultMocks();+ mockLongRunningQuery.mockReturnValue(withExecutions([sampleExecution], 3));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="long-running"+ />,+ );+ expect(screen.getByText(/Stop All Long-Running \(3\)/)).toBeDefined();+ });++ it("shows invalid state read-only banner", () => {+ setupDefaultMocks();+ mockInvalidQuery.mockReturnValue(withExecutions([], 0));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="invalid"+ />,+ );+ expect(+ screen.getByText(+ /Read-only: Invalid states require manual investigation/,+ ),+ ).toBeDefined();+ });++ it("shows view-only message in failed tab with no selection", () => {+ setupDefaultMocks();+ const failedExec = {+ ...sampleExecution,+ status: "FAILED",+ error_message: "err",+ };+ mockFailedQuery.mockReturnValue(withExecutions([failedExec], 1));+ render(+ <ExecutionsTable diagnosticsData={diagnosticsData} initialTab="failed" />,+ );+ expect(screen.getByText("View-only (select to delete)")).toBeDefined();+ });++ it("renders table column headers", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([sampleExecution], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("Execution ID")).toBeDefined();+ expect(screen.getByText("Agent Name")).toBeDefined();+ expect(screen.getByText("Version")).toBeDefined();+ expect(screen.getByText("User")).toBeDefined();+ expect(screen.getByText("Status")).toBeDefined();+ expect(screen.getByText("Age")).toBeDefined();+ });++ it("renders failed tab with error column header", () => {+ setupDefaultMocks();+ const failedExec = {+ ...sampleExecution,+ status: "FAILED",+ failed_at: "2026-04-16T12:00:00Z",+ error_message: "Timeout",+ };+ mockFailedQuery.mockReturnValue(withExecutions([failedExec], 1));+ render(+ <ExecutionsTable diagnosticsData={diagnosticsData} initialTab="failed" />,+ );+ expect(screen.getByText("Error Message")).toBeDefined();+ expect(screen.getByText("Timeout")).toBeDefined();+ });++ it("renders no error message text when error_message is null", () => {+ setupDefaultMocks();+ const failedNoMsg = {+ ...sampleExecution,+ status: "FAILED",+ failed_at: "2026-04-16T12:00:00Z",+ error_message: null,+ };+ mockFailedQuery.mockReturnValue(withExecutions([failedNoMsg], 1));+ render(+ <ExecutionsTable diagnosticsData={diagnosticsData} initialTab="failed" />,+ );+ expect(screen.getByText("No error message")).toBeDefined();+ });++ it("renders started_at as dash when null in non-failed tab", () => {+ setupDefaultMocks();+ const noStart = { ...sampleExecution, started_at: null };+ mockRunningQuery.mockReturnValue(withExecutions([noStart], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ const dashes = screen.getAllByText("-");+ expect(dashes.length).toBeGreaterThanOrEqual(1);+ });++ it("renders failed_at as dash when null in failed tab", () => {+ setupDefaultMocks();+ const failedNoDate = {+ ...sampleExecution,+ status: "FAILED",+ failed_at: null,+ error_message: "err",+ };+ mockFailedQuery.mockReturnValue(withExecutions([failedNoDate], 1));+ render(+ <ExecutionsTable diagnosticsData={diagnosticsData} initialTab="failed" />,+ );+ const dashes = screen.getAllByText("-");+ expect(dashes.length).toBeGreaterThanOrEqual(1);+ });++ it("renders Executions card title", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([], 0));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("Executions")).toBeDefined();+ });++ it("opens stop dialog when clicking cleanup button on stuck-queued row", async () => {+ setupDefaultMocks();+ const stuckExec = {+ ...sampleExecution,+ execution_id: "exec-stuck-dialog",+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ fireEvent.click(screen.getByTitle("Cleanup (mark as FAILED)"));+ await waitFor(() => {+ expect(+ screen.getByText("Confirm Cleanup Orphaned Executions"),+ ).toBeDefined();+ expect(screen.getByText("Cancel")).toBeDefined();+ expect(screen.getByText("Cleanup Orphaned")).toBeDefined();+ });+ });++ it("calls cleanupOrphanedExecutions when confirming single cleanup", async () => {+ setupDefaultMocks();+ mockCleanupOrphaned.mockResolvedValue({+ data: { success: true, stopped_count: 1, message: "Cleaned" },+ });+ const stuckExec = {+ ...sampleExecution,+ execution_id: "exec-stuck-confirm",+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ fireEvent.click(screen.getByTitle("Cleanup (mark as FAILED)"));+ await waitFor(() => {+ expect(screen.getByText("Cleanup Orphaned")).toBeDefined();+ });+ fireEvent.click(screen.getByText("Cleanup Orphaned"));+ await waitFor(() => {+ expect(mockCleanupOrphaned).toHaveBeenCalled();+ });+ });++ it("opens cleanup dialog for stuck-queued execution", async () => {+ setupDefaultMocks();+ const stuckExec = {+ ...sampleExecution,+ execution_id: "exec-stuck-1",+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ fireEvent.click(screen.getByTitle("Cleanup (mark as FAILED)"));+ await waitFor(() => {+ expect(+ screen.getByText("Confirm Cleanup Orphaned Executions"),+ ).toBeDefined();+ expect(screen.getByText("Cleanup Orphaned")).toBeDefined();+ });+ });++ it("calls cleanupOrphanedExecutions when confirming cleanup", async () => {+ setupDefaultMocks();+ mockCleanupOrphaned.mockResolvedValue({+ data: { success: true, stopped_count: 1, message: "Cleaned" },+ });+ const stuckExec = {+ ...sampleExecution,+ execution_id: "exec-stuck-1",+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ fireEvent.click(screen.getByTitle("Cleanup (mark as FAILED)"));+ await waitFor(() => {+ expect(screen.getByText("Cleanup Orphaned")).toBeDefined();+ });+ fireEvent.click(screen.getByText("Cleanup Orphaned"));+ await waitFor(() => {+ expect(mockCleanupOrphaned).toHaveBeenCalled();+ });+ });++ it("opens requeue dialog for stuck-queued execution", async () => {+ setupDefaultMocks();+ const stuckExec = {+ ...sampleExecution,+ execution_id: "exec-stuck-1",+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ fireEvent.click(screen.getByTitle("Requeue (send to RabbitMQ)"));+ await waitFor(() => {+ expect(+ screen.getByText("Confirm Requeue Stuck Executions"),+ ).toBeDefined();+ expect(screen.getByText("Requeue Executions")).toBeDefined();+ });+ });++ it("calls requeueSingleExecution when confirming requeue", async () => {+ setupDefaultMocks();+ mockRequeueSingle.mockResolvedValue({+ data: { success: true, requeued_count: 1, message: "Requeued" },+ });+ const stuckExec = {+ ...sampleExecution,+ execution_id: "exec-stuck-1",+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ fireEvent.click(screen.getByTitle("Requeue (send to RabbitMQ)"));+ await waitFor(() => {+ expect(screen.getByText("Requeue Executions")).toBeDefined();+ });+ fireEvent.click(screen.getByText("Requeue Executions"));+ await waitFor(() => {+ expect(mockRequeueSingle).toHaveBeenCalled();+ });+ });++ it("closes dialog when cancel is clicked", async () => {+ setupDefaultMocks();+ const stuckExec = {+ ...sampleExecution,+ execution_id: "exec-cancel-test",+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ fireEvent.click(screen.getByTitle("Cleanup (mark as FAILED)"));+ await waitFor(() => {+ expect(+ screen.getByText("Confirm Cleanup Orphaned Executions"),+ ).toBeDefined();+ });+ fireEvent.click(screen.getByText("Cancel"));+ await waitFor(() => {+ expect(+ screen.queryByText("Confirm Cleanup Orphaned Executions"),+ ).toBeNull();+ });+ });++ it("handles cleanup mutation error gracefully", async () => {+ setupDefaultMocks();+ mockCleanupOrphaned.mockRejectedValue(new Error("Network error"));+ const stuckExec = {+ ...sampleExecution,+ execution_id: "exec-error-test",+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ fireEvent.click(screen.getByTitle("Cleanup (mark as FAILED)"));+ await waitFor(() => {+ expect(screen.getByText("Cleanup Orphaned")).toBeDefined();+ });+ fireEvent.click(screen.getByText("Cleanup Orphaned"));+ await waitFor(() => {+ expect(mockCleanupOrphaned).toHaveBeenCalled();+ });+ });++ it("calls requeueAllStuck when clicking Requeue All button and confirming", async () => {+ setupDefaultMocks();+ mockRequeueAllStuck.mockResolvedValue({+ data: { success: true, requeued_count: 5, message: "Requeued 5" },+ });+ const stuckExecs = Array.from({ length: 3 }, (_, i) => ({+ ...sampleExecution,+ execution_id: `exec-stuck-${i}`,+ status: "QUEUED",+ started_at: null,+ }));+ mockStuckQueuedQuery.mockReturnValue(withExecutions(stuckExecs, 5));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ fireEvent.click(screen.getByText(/Requeue All \(5\)/));+ await waitFor(() => {+ expect(+ screen.getByText("Confirm Requeue Stuck Executions"),+ ).toBeDefined();+ });+ fireEvent.click(screen.getByText("Requeue Executions"));+ await waitFor(() => {+ expect(mockRequeueAllStuck).toHaveBeenCalled();+ });+ });++ it("calls cleanupAllStuckQueued when clicking Cleanup All on stuck-queued tab", async () => {+ setupDefaultMocks();+ mockCleanupAllStuckQueued.mockResolvedValue({+ data: { success: true, stopped_count: 5, message: "Cleaned 5" },+ });+ const stuckExecs = Array.from({ length: 3 }, (_, i) => ({+ ...sampleExecution,+ execution_id: `exec-stuck-${i}`,+ status: "QUEUED",+ started_at: null,+ }));+ mockStuckQueuedQuery.mockReturnValue(withExecutions(stuckExecs, 5));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ fireEvent.click(screen.getByText(/Cleanup All \(5\)/));+ await waitFor(() => {+ expect(+ screen.getByText("Confirm Cleanup Orphaned Executions"),+ ).toBeDefined();+ });+ fireEvent.click(screen.getByText("Cleanup Orphaned"));+ await waitFor(() => {+ expect(mockCleanupAllStuckQueued).toHaveBeenCalled();+ });+ });++ it("calls stopAllLongRunning when clicking Stop All Long-Running", async () => {+ setupDefaultMocks();+ mockStopAllLongRunning.mockResolvedValue({+ data: { success: true, stopped_count: 3, message: "Stopped 3" },+ });+ mockLongRunningQuery.mockReturnValue(withExecutions([sampleExecution], 3));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="long-running"+ />,+ );+ fireEvent.click(screen.getByText(/Stop All Long-Running \(3\)/));+ await waitFor(() => {+ expect(screen.getByText("Confirm Stop Executions")).toBeDefined();+ });+ fireEvent.click(screen.getByText("Stop Executions"));+ await waitFor(() => {+ expect(mockStopAllLongRunning).toHaveBeenCalled();+ });+ });++ it("shows requeue warning text in dialog", async () => {+ setupDefaultMocks();+ const stuckExec = {+ ...sampleExecution,+ execution_id: "exec-stuck-warn",+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ fireEvent.click(screen.getByTitle("Requeue (send to RabbitMQ)"));+ await waitFor(() => {+ expect(screen.getByText(/will cost credits/)).toBeDefined();+ });+ });++ it("shows cleanup description in dialog", async () => {+ setupDefaultMocks();+ const stuckExec = {+ ...sampleExecution,+ execution_id: "exec-stuck-desc",+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ fireEvent.click(screen.getByTitle("Cleanup (mark as FAILED)"));+ await waitFor(() => {+ expect(screen.getByText(/cleanup this orphaned execution/)).toBeDefined();+ });+ });++ it("renders age in days format for old executions", () => {+ setupDefaultMocks();+ const oldExec = {+ ...sampleExecution,+ started_at: new Date(Date.now() - 3 * 24 * 60 * 60 * 1000).toISOString(),+ };+ mockRunningQuery.mockReturnValue(withExecutions([oldExec], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText(/3d/)).toBeDefined();+ });++ it("shows stop selected button after selecting a checkbox", async () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([sampleExecution], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ if (checkboxes[1]) fireEvent.click(checkboxes[1]);+ await waitFor(() => {+ expect(screen.getByText(/Stop Selected/)).toBeDefined();+ });+ });++ it("shows stop selected button with count after selection", async () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([sampleExecution], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ if (checkboxes[1]) fireEvent.click(checkboxes[1]);+ await waitFor(() => {+ expect(screen.getByText(/Stop Selected \(1\)/)).toBeDefined();+ });+ });++ it("renders select-all checkbox", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([sampleExecution], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ expect(checkboxes.length).toBeGreaterThanOrEqual(2);+ });++ it("selects all checkboxes with select-all", async () => {+ setupDefaultMocks();+ const execs = [+ { ...sampleExecution, execution_id: "exec-a" },+ { ...sampleExecution, execution_id: "exec-b" },+ ];+ mockRunningQuery.mockReturnValue(withExecutions(execs, 2));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ // First checkbox is select-all+ if (checkboxes[0]) fireEvent.click(checkboxes[0]);+ await waitFor(() => {+ expect(screen.getByText(/Stop Selected \(2\)/)).toBeDefined();+ });+ });++ it("renders hours format for recent execution age", () => {+ setupDefaultMocks();+ const recentExec = {+ ...sampleExecution,+ started_at: new Date(Date.now() - 5 * 60 * 60 * 1000).toISOString(),+ };+ mockRunningQuery.mockReturnValue(withExecutions([recentExec], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText(/5h/)).toBeDefined();+ });++ it("calls onRefresh when provided", async () => {+ setupDefaultMocks();+ const onRefresh = vi.fn();+ mockStopSingle.mockResolvedValue({+ data: { success: true, stopped_count: 1, message: "Stopped" },+ });+ const stuckExec = {+ ...sampleExecution,+ execution_id: "exec-refresh-test",+ status: "QUEUED",+ started_at: null,+ };+ mockStuckQueuedQuery.mockReturnValue(withExecutions([stuckExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ onRefresh={onRefresh}+ />,+ );+ fireEvent.click(screen.getByTitle("Cleanup (mark as FAILED)"));+ await waitFor(() => {+ expect(screen.getByText("Cleanup Orphaned")).toBeDefined();+ });+ mockCleanupOrphaned.mockResolvedValue({+ data: { success: true, stopped_count: 1, message: "OK" },+ });+ fireEvent.click(screen.getByText("Cleanup Orphaned"));+ await waitFor(() => {+ expect(onRefresh).toHaveBeenCalled();+ });+ });++ it("renders showing count text in pagination", () => {+ setupDefaultMocks();+ const executions = Array.from({ length: 10 }, (_, i) => ({+ ...sampleExecution,+ execution_id: `exec-page-${i}`,+ }));+ mockRunningQuery.mockReturnValue(withExecutions(executions, 30));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText(/Showing 1 to 10 of 30/)).toBeDefined();+ });++ it("disables Previous button on first page", () => {+ setupDefaultMocks();+ const executions = Array.from({ length: 10 }, (_, i) => ({+ ...sampleExecution,+ execution_id: `exec-dis-${i}`,+ }));+ mockRunningQuery.mockReturnValue(withExecutions(executions, 25));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ const prevBtn = screen.getByText("Previous").closest("button");+ expect(prevBtn?.disabled).toBe(true);+ });++ it("enables Next button when more pages exist", () => {+ setupDefaultMocks();+ const executions = Array.from({ length: 10 }, (_, i) => ({+ ...sampleExecution,+ execution_id: `exec-next-${i}`,+ }));+ mockRunningQuery.mockReturnValue(withExecutions(executions, 25));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ const nextBtn = screen.getByText("Next").closest("button");+ expect(nextBtn?.disabled).toBe(false);+ });++ it("renders orphaned execution with orange background", () => {+ setupDefaultMocks();+ const orphanedExec = {+ ...sampleExecution,+ execution_id: "exec-orange",+ created_at: "2026-04-10T10:00:00Z",+ };+ mockOrphanedQuery.mockReturnValue(withExecutions([orphanedExec], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="orphaned"+ />,+ );+ const row = screen.getByText("Test Agent").closest("tr");+ expect(row?.className).toContain("bg-orange");+ });++ it("renders initialTab syncs with useEffect", () => {+ setupDefaultMocks();+ mockFailedQuery.mockReturnValue(+ withExecutions(+ [+ {+ ...sampleExecution,+ execution_id: "exec-sync",+ status: "FAILED",+ error_message: "sync test",+ },+ ],+ 1,+ ),+ );+ const { rerender } = render(+ <ExecutionsTable diagnosticsData={diagnosticsData} initialTab="all" />,+ );+ // Rerender with new initialTab to trigger useEffect sync+ rerender(+ <ExecutionsTable diagnosticsData={diagnosticsData} initialTab="failed" />,+ );+ expect(screen.getByText("sync test")).toBeDefined();+ });++ it("renders the all tab total count", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([sampleExecution], 7));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ // "All (7)" in the tab trigger+ expect(screen.getByText(/All.*7/)).toBeDefined();+ });++ it("opens stop dialog and calls mutations for selected executions", async () => {+ setupDefaultMocks();+ mockStopMultiple.mockResolvedValue({+ data: { success: true, stopped_count: 1, message: "Stopped 1" },+ });+ mockCleanupOrphaned.mockResolvedValue({+ data: { success: true, stopped_count: 0, message: "OK" },+ });+ // Use a recent execution that won't be classified as orphaned+ const recentExec = {+ ...sampleExecution,+ execution_id: "exec-recent-stop",+ created_at: new Date().toISOString(),+ };+ mockRunningQuery.mockReturnValue(withExecutions([recentExec], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ // Select execution+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ if (checkboxes[1]) fireEvent.click(checkboxes[1]);+ await waitFor(() => {+ expect(screen.getByText(/Stop Selected/)).toBeDefined();+ });+ // Click stop selected+ fireEvent.click(screen.getByText(/Stop Selected/));+ // Dialog should open+ await waitFor(() => {+ expect(screen.getByText("Confirm Stop Executions")).toBeDefined();+ });+ // Confirm+ fireEvent.click(screen.getByText("Stop Executions"));+ await waitFor(() => {+ expect(mockStopMultiple).toHaveBeenCalled();+ });+ });++ it("calls requeueMultiple for selected stuck-queued executions", async () => {+ setupDefaultMocks();+ mockRequeueMultiple.mockResolvedValue({+ data: { success: true, requeued_count: 2, message: "Requeued 2" },+ });+ const stuckExecs = [+ {+ ...sampleExecution,+ execution_id: "stuck-a",+ status: "QUEUED",+ started_at: null,+ },+ {+ ...sampleExecution,+ execution_id: "stuck-b",+ status: "QUEUED",+ started_at: null,+ },+ ];+ mockStuckQueuedQuery.mockReturnValue(withExecutions(stuckExecs, 2));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="stuck-queued"+ />,+ );+ // Select all via select-all checkbox+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ if (checkboxes[0]) fireEvent.click(checkboxes[0]);+ // In stuck-queued tab, no "Stop Selected" button - only Cleanup All / Requeue All+ // Use Requeue All button instead+ await waitFor(() => {+ expect(screen.getByText(/Requeue All \(2\)/)).toBeDefined();+ });+ fireEvent.click(screen.getByText(/Requeue All \(2\)/));+ await waitFor(() => {+ expect(screen.getByText("Requeue Executions")).toBeDefined();+ });+ fireEvent.click(screen.getByText("Requeue Executions"));+ await waitFor(() => {+ expect(mockRequeueAllStuck).toHaveBeenCalled();+ });+ });++ it("shows dialog description for stop all on long-running tab", async () => {+ setupDefaultMocks();+ mockLongRunningQuery.mockReturnValue(withExecutions([sampleExecution], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="long-running"+ />,+ );+ fireEvent.click(screen.getByText(/Stop All Long-Running/));+ await waitFor(() => {+ expect(screen.getByText(/stop ALL 1 execution/)).toBeDefined();+ });+ });++ it("shows stop dialog description listing what it does", async () => {+ setupDefaultMocks();+ mockLongRunningQuery.mockReturnValue(withExecutions([sampleExecution], 1));+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ initialTab="long-running"+ />,+ );+ fireEvent.click(screen.getByText(/Stop All Long-Running/));+ await waitFor(() => {+ expect(+ screen.getByText(/Send cancel signals for active executions/),+ ).toBeDefined();+ expect(screen.getByText(/Mark all as FAILED/)).toBeDefined();+ });+ });++ it("clicking refresh button calls refetch and onRefresh", () => {+ setupDefaultMocks();+ const onRefresh = vi.fn();+ const refetch = vi.fn();+ mockRunningQuery.mockReturnValue({+ data: { data: { executions: [sampleExecution], total: 1 } },+ isLoading: false,+ error: null,+ refetch,+ });+ render(+ <ExecutionsTable+ diagnosticsData={diagnosticsData}+ onRefresh={onRefresh}+ />,+ );+ // The refresh button is the last button with ArrowClockwise icon in the header+ const buttons = document.querySelectorAll("button");+ // Find the standalone refresh button (no text, just icon)+ const refreshBtn = Array.from(buttons).find(+ (b) => b.querySelector("svg") && b.textContent?.trim() === "",+ );+ if (refreshBtn) {+ fireEvent.click(refreshBtn);+ expect(refetch).toHaveBeenCalled();+ expect(onRefresh).toHaveBeenCalled();+ }+ });++ it("renders executions text label in Showing pagination", () => {+ setupDefaultMocks();+ const executions = Array.from({ length: 10 }, (_, i) => ({+ ...sampleExecution,+ execution_id: `exec-label-${i}`,+ }));+ mockRunningQuery.mockReturnValue(withExecutions(executions, 20));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText(/executions/)).toBeDefined();+ });++ it("renders status badge with green for RUNNING", () => {+ setupDefaultMocks();+ mockRunningQuery.mockReturnValue(withExecutions([sampleExecution], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ const badge = screen.getByText("RUNNING");+ expect(badge.className).toContain("bg-green");+ });++ it("renders status badge with yellow for QUEUED", () => {+ setupDefaultMocks();+ const queuedExec = { ...sampleExecution, status: "QUEUED" };+ mockRunningQuery.mockReturnValue(withExecutions([queuedExec], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ const badge = screen.getByText("QUEUED");+ expect(badge.className).toContain("bg-yellow");+ });++ it("clicking Next advances pagination page", () => {+ setupDefaultMocks();+ const executions = Array.from({ length: 10 }, (_, i) => ({+ ...sampleExecution,+ execution_id: `exec-pagnext-${i}`,+ }));+ mockRunningQuery.mockReturnValue(withExecutions(executions, 25));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText(/Page 1 of 3/)).toBeDefined();+ fireEvent.click(screen.getByText("Next"));+ expect(screen.getByText(/Page 2 of 3/)).toBeDefined();+ });++ it("clicking Previous goes back a page", () => {+ setupDefaultMocks();+ const executions = Array.from({ length: 10 }, (_, i) => ({+ ...sampleExecution,+ execution_id: `exec-pagprev-${i}`,+ }));+ mockRunningQuery.mockReturnValue(withExecutions(executions, 25));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ fireEvent.click(screen.getByText("Next"));+ expect(screen.getByText(/Page 2 of 3/)).toBeDefined();+ fireEvent.click(screen.getByText("Previous"));+ expect(screen.getByText(/Page 1 of 3/)).toBeDefined();+ });++ it("splits orphaned and active IDs when stopping selected with old execution", async () => {+ setupDefaultMocks();+ mockStopMultiple.mockResolvedValue({+ data: { success: true, stopped_count: 0, message: "OK" },+ });+ mockCleanupOrphaned.mockResolvedValue({+ data: { success: true, stopped_count: 1, message: "Cleaned 1" },+ });+ // Use an OLD execution (>24h) so it's classified as orphaned+ const oldExec = {+ ...sampleExecution,+ execution_id: "exec-old-orphan",+ created_at: new Date(Date.now() - 48 * 60 * 60 * 1000).toISOString(),+ };+ mockRunningQuery.mockReturnValue(withExecutions([oldExec], 1));+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ // Select the old execution+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ if (checkboxes[1]) fireEvent.click(checkboxes[1]);+ await waitFor(() => {+ expect(screen.getByText(/Stop Selected/)).toBeDefined();+ });+ fireEvent.click(screen.getByText(/Stop Selected/));+ await waitFor(() => {+ expect(screen.getByText("Stop Executions")).toBeDefined();+ });+ fireEvent.click(screen.getByText("Stop Executions"));+ await waitFor(() => {+ // Should call cleanupOrphaned for the old execution+ expect(mockCleanupOrphaned).toHaveBeenCalled();+ });+ });++ it("clicking Try Again on error state calls refetch", () => {+ setupDefaultMocks();+ const refetch = vi.fn();+ mockRunningQuery.mockReturnValue({+ data: undefined,+ isLoading: false,+ error: { status: 500, message: "Server error" },+ refetch,+ });+ render(<ExecutionsTable diagnosticsData={diagnosticsData} />);+ fireEvent.click(screen.getByText("Try Again"));+ expect(refetch).toHaveBeenCalled();+ });+});diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/__tests__/SchedulesTable.test.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/__tests__/SchedulesTable.test.tsxnew file mode 100644index 0000000000..a377fafe3c--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/__tests__/SchedulesTable.test.tsx@@ -0,0 +1,413 @@+import {+ render,+ screen,+ cleanup,+ fireEvent,+ waitFor,+} from "@/tests/integrations/test-utils";+import { afterEach, describe, expect, it, vi } from "vitest";+import { SchedulesTable } from "../components/SchedulesTable";++const mockAllSchedulesQuery = vi.fn();+const mockOrphanedSchedulesQuery = vi.fn();+const mockCleanupOrphaned = vi.fn();++vi.mock("@/app/api/__generated__/endpoints/admin/admin", () => ({+ useGetV2ListAllUserSchedules: (...args: unknown[]) =>+ mockAllSchedulesQuery(...args),+ useGetV2ListOrphanedSchedules: (...args: unknown[]) =>+ mockOrphanedSchedulesQuery(...args),+ usePostV2CleanupOrphanedSchedules: () => ({+ mutateAsync: mockCleanupOrphaned,+ isPending: false,+ }),+}));++function defaultQueryReturn(overrides = {}) {+ return {+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ ...overrides,+ };+}++function withSchedules(+ schedules: Record<string, unknown>[],+ total: number,+ overrides = {},+) {+ return defaultQueryReturn({+ data: { data: { schedules, total } },+ ...overrides,+ });+}++const sampleSchedule = {+ schedule_id: "sched-001",+ schedule_name: "Daily Agent Run",+ graph_id: "graph-123",+ graph_name: "My Agent",+ graph_version: 1,+ user_id: "user-abc",+ user_email: "[email protected]",+ cron: "0 9 * * *",+ timezone: "America/New_York",+ next_run_time: "2026-04-17T13:00:00Z",+};++const diagnosticsData = {+ total_orphaned: 3,+ user_schedules: 10,+};++function setupDefaultMocks() {+ mockAllSchedulesQuery.mockReturnValue(defaultQueryReturn());+ mockOrphanedSchedulesQuery.mockReturnValue(defaultQueryReturn());+}++afterEach(() => {+ cleanup();+ mockAllSchedulesQuery.mockReset();+ mockOrphanedSchedulesQuery.mockReset();+ mockCleanupOrphaned.mockReset();+});++describe("SchedulesTable", () => {+ it("shows empty state when no schedules", () => {+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(withSchedules([], 0));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("No schedules found")).toBeDefined();+ });++ it("renders schedule rows", () => {+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(withSchedules([sampleSchedule], 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("Daily Agent Run")).toBeDefined();+ expect(screen.getByText("[email protected]")).toBeDefined();+ expect(screen.getByText("0 9 * * *")).toBeDefined();+ expect(screen.getByText("America/New_York")).toBeDefined();+ });++ it("renders tab triggers with counts", () => {+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(withSchedules([], 0));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("All Schedules (10)")).toBeDefined();+ expect(screen.getByText("Orphaned (3)")).toBeDefined();+ });++ it("shows loading spinner", () => {+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(+ defaultQueryReturn({ isLoading: true }),+ );+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(document.querySelector(".animate-spin")).toBeDefined();+ });++ it("renders graph version", () => {+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(withSchedules([sampleSchedule], 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("v1")).toBeDefined();+ });++ it("shows unknown for missing graph name", () => {+ setupDefaultMocks();+ const noGraphSchedule = { ...sampleSchedule, graph_name: undefined };+ mockAllSchedulesQuery.mockReturnValue(withSchedules([noGraphSchedule], 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("Unknown")).toBeDefined();+ });++ it("renders without diagnostics data", () => {+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(withSchedules([], 0));+ render(<SchedulesTable />);+ expect(screen.getByText("All Schedules")).toBeDefined();+ expect(screen.getByText("Orphaned")).toBeDefined();+ });++ it("renders pagination for many schedules", () => {+ setupDefaultMocks();+ const schedules = Array.from({ length: 10 }, (_, i) => ({+ ...sampleSchedule,+ schedule_id: `sched-${i}`,+ }));+ mockAllSchedulesQuery.mockReturnValue(withSchedules(schedules, 25));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText(/Page 1 of 3/)).toBeDefined();+ expect(screen.getByText("Previous")).toBeDefined();+ expect(screen.getByText("Next")).toBeDefined();+ });++ it("copies user ID to clipboard on click", () => {+ const writeText = vi.fn().mockResolvedValue(undefined);+ vi.stubGlobal("navigator", { ...navigator, clipboard: { writeText } });+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(withSchedules([sampleSchedule], 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ fireEvent.click(screen.getByText("user-abc".substring(0, 8) + "..."));+ expect(writeText).toHaveBeenCalledWith("user-abc");+ vi.unstubAllGlobals();+ });++ it("shows unknown for null user email", () => {+ setupDefaultMocks();+ const noEmailSchedule = { ...sampleSchedule, user_email: null };+ mockAllSchedulesQuery.mockReturnValue(withSchedules([noEmailSchedule], 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("Unknown")).toBeDefined();+ });++ it("renders cron expression in code block", () => {+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(withSchedules([sampleSchedule], 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ const codeEl = screen.getByText("0 9 * * *");+ expect(codeEl.tagName.toLowerCase()).toBe("code");+ });++ it("renders next run time as date string", () => {+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(withSchedules([sampleSchedule], 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ const dateStr = new Date("2026-04-17T13:00:00Z").toLocaleString();+ expect(screen.getByText(dateStr)).toBeDefined();+ });++ it("shows not scheduled for missing next run time", () => {+ setupDefaultMocks();+ const noRunTime = { ...sampleSchedule, next_run_time: null };+ mockAllSchedulesQuery.mockReturnValue(withSchedules([noRunTime], 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("Not scheduled")).toBeDefined();+ });++ it("renders table headers", () => {+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(withSchedules([sampleSchedule], 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("Name")).toBeDefined();+ expect(screen.getByText("Graph")).toBeDefined();+ expect(screen.getByText("User")).toBeDefined();+ expect(screen.getByText("Cron")).toBeDefined();+ expect(screen.getByText("Next Run")).toBeDefined();+ });++ it("renders Schedules card title", () => {+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(withSchedules([], 0));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("Schedules")).toBeDefined();+ });++ it("renders multiple schedule rows", () => {+ setupDefaultMocks();+ const schedules = [+ { ...sampleSchedule, schedule_id: "sched-1", schedule_name: "First" },+ { ...sampleSchedule, schedule_id: "sched-2", schedule_name: "Second" },+ ];+ mockAllSchedulesQuery.mockReturnValue(withSchedules(schedules, 2));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText("First")).toBeDefined();+ expect(screen.getByText("Second")).toBeDefined();+ });++ it("shows delete all button on orphaned tab", async () => {+ setupDefaultMocks();+ const orphanedSchedule = {+ ...sampleSchedule,+ schedule_id: "sched-orphan-1",+ orphan_reason: "deleted_graph",+ };+ mockOrphanedSchedulesQuery.mockReturnValue(+ withSchedules([orphanedSchedule], 1),+ );+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ // Switch to orphaned tab by rendering with initial state+ // The "Delete All Orphaned" button only shows in orphaned tab+ // We can't switch tabs programmatically, but we can test the orphaned tab directly+ });++ it("renders refresh button", () => {+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(withSchedules([], 0));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ // The refresh button has an ArrowClockwise icon+ const buttons = document.querySelectorAll("button");+ expect(buttons.length).toBeGreaterThan(0);+ });++ it("renders showing count text with pagination", () => {+ setupDefaultMocks();+ const schedules = Array.from({ length: 10 }, (_, i) => ({+ ...sampleSchedule,+ schedule_id: `sched-${i}`,+ }));+ mockAllSchedulesQuery.mockReturnValue(withSchedules(schedules, 15));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText(/Showing 1 to 10 of 15/)).toBeDefined();+ });++ it("renders delete selected button when schedules are selected via checkbox", async () => {+ setupDefaultMocks();+ const schedules = [+ { ...sampleSchedule, schedule_id: "sched-sel-1" },+ { ...sampleSchedule, schedule_id: "sched-sel-2" },+ ];+ mockAllSchedulesQuery.mockReturnValue(withSchedules(schedules, 2));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ // Click the first checkbox (individual schedule)+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ // First checkbox is select-all, subsequent are individual+ if (checkboxes[1]) fireEvent.click(checkboxes[1]);+ await waitFor(() => {+ expect(screen.getByText(/Delete Selected/)).toBeDefined();+ });+ });++ it("shows select-all checkbox in header", () => {+ setupDefaultMocks();+ mockAllSchedulesQuery.mockReturnValue(withSchedules([sampleSchedule], 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ expect(checkboxes.length).toBeGreaterThanOrEqual(2);+ });++ it("opens delete dialog and calls cleanup mutation", async () => {+ setupDefaultMocks();+ mockCleanupOrphaned.mockResolvedValue({+ data: { success: true, deleted_count: 1, message: "Deleted 1" },+ });+ const schedules = [{ ...sampleSchedule, schedule_id: "sched-del-1" }];+ mockAllSchedulesQuery.mockReturnValue(withSchedules(schedules, 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ // Select a schedule via checkbox+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ if (checkboxes[1]) fireEvent.click(checkboxes[1]);+ await waitFor(() => {+ expect(screen.getByText(/Delete Selected/)).toBeDefined();+ });+ // Click delete selected+ fireEvent.click(screen.getByText(/Delete Selected/));+ // Dialog should open+ await waitFor(() => {+ expect(screen.getByText("Confirm Delete Schedules")).toBeDefined();+ });+ // Confirm deletion+ fireEvent.click(screen.getByText("Delete Schedules"));+ await waitFor(() => {+ expect(mockCleanupOrphaned).toHaveBeenCalled();+ });+ });++ it("shows cancel button in delete dialog", async () => {+ setupDefaultMocks();+ const schedules = [{ ...sampleSchedule, schedule_id: "sched-cancel-1" }];+ mockAllSchedulesQuery.mockReturnValue(withSchedules(schedules, 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ if (checkboxes[1]) fireEvent.click(checkboxes[1]);+ await waitFor(() => {+ expect(screen.getByText(/Delete Selected/)).toBeDefined();+ });+ fireEvent.click(screen.getByText(/Delete Selected/));+ await waitFor(() => {+ expect(screen.getByText("Cancel")).toBeDefined();+ expect(screen.getByText("Delete Schedules")).toBeDefined();+ });+ });++ it("shows dialog description text about permanent removal", async () => {+ setupDefaultMocks();+ const schedules = [{ ...sampleSchedule, schedule_id: "sched-desc-1" }];+ mockAllSchedulesQuery.mockReturnValue(withSchedules(schedules, 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ if (checkboxes[1]) fireEvent.click(checkboxes[1]);+ await waitFor(() => {+ expect(screen.getByText(/Delete Selected/)).toBeDefined();+ });+ fireEvent.click(screen.getByText(/Delete Selected/));+ await waitFor(() => {+ expect(+ screen.getByText(/permanently remove the schedules/),+ ).toBeDefined();+ });+ });++ it("closes dialog when cancel is clicked", async () => {+ setupDefaultMocks();+ const schedules = [{ ...sampleSchedule, schedule_id: "sched-close-1" }];+ mockAllSchedulesQuery.mockReturnValue(withSchedules(schedules, 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ if (checkboxes[1]) fireEvent.click(checkboxes[1]);+ await waitFor(() => {+ expect(screen.getByText(/Delete Selected/)).toBeDefined();+ });+ fireEvent.click(screen.getByText(/Delete Selected/));+ await waitFor(() => {+ expect(screen.getByText("Cancel")).toBeDefined();+ });+ fireEvent.click(screen.getByText("Cancel"));+ await waitFor(() => {+ expect(screen.queryByText("Confirm Delete Schedules")).toBeNull();+ });+ });++ it("handles delete error gracefully", async () => {+ setupDefaultMocks();+ mockCleanupOrphaned.mockRejectedValue(new Error("Delete failed"));+ const schedules = [{ ...sampleSchedule, schedule_id: "sched-err-1" }];+ mockAllSchedulesQuery.mockReturnValue(withSchedules(schedules, 1));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ const checkboxes = document.querySelectorAll('[role="checkbox"]');+ if (checkboxes[1]) fireEvent.click(checkboxes[1]);+ await waitFor(() => {+ expect(screen.getByText(/Delete Selected/)).toBeDefined();+ });+ fireEvent.click(screen.getByText(/Delete Selected/));+ await waitFor(() => {+ expect(screen.getByText("Delete Schedules")).toBeDefined();+ });+ fireEvent.click(screen.getByText("Delete Schedules"));+ await waitFor(() => {+ expect(mockCleanupOrphaned).toHaveBeenCalled();+ });+ });++ it("clicking Next button advances page", () => {+ setupDefaultMocks();+ const schedules = Array.from({ length: 10 }, (_, i) => ({+ ...sampleSchedule,+ schedule_id: `sched-pag-${i}`,+ }));+ mockAllSchedulesQuery.mockReturnValue(withSchedules(schedules, 25));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ expect(screen.getByText(/Page 1 of 3/)).toBeDefined();+ fireEvent.click(screen.getByText("Next"));+ expect(screen.getByText(/Page 2 of 3/)).toBeDefined();+ });++ it("clicking Previous button goes back a page", () => {+ setupDefaultMocks();+ const schedules = Array.from({ length: 10 }, (_, i) => ({+ ...sampleSchedule,+ schedule_id: `sched-back-${i}`,+ }));+ mockAllSchedulesQuery.mockReturnValue(withSchedules(schedules, 25));+ render(<SchedulesTable diagnosticsData={diagnosticsData} />);+ // Go to page 2 first+ fireEvent.click(screen.getByText("Next"));+ expect(screen.getByText(/Page 2 of 3/)).toBeDefined();+ // Go back+ fireEvent.click(screen.getByText("Previous"));+ expect(screen.getByText(/Page 1 of 3/)).toBeDefined();+ });+});diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/__tests__/page.test.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/__tests__/page.test.tsxnew file mode 100644index 0000000000..310c238dfc--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/__tests__/page.test.tsx@@ -0,0 +1,133 @@+import { render, screen } from "@/tests/integrations/test-utils";+import { describe, expect, it, vi } from "vitest";++// Mock withRoleAccess to bypass server-side auth+vi.mock("@/lib/withRoleAccess", () => ({+ withRoleAccess: () =>+ Promise.resolve((Component: React.ComponentType) =>+ Promise.resolve(Component),+ ),+}));++// Mock the generated API hooks used by DiagnosticsContent+vi.mock("@/app/api/__generated__/endpoints/admin/admin", () => ({+ useGetV2GetExecutionDiagnostics: () => ({+ data: undefined,+ isLoading: true,+ isError: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2GetAgentDiagnostics: () => ({+ data: undefined,+ isLoading: true,+ isError: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2GetScheduleDiagnostics: () => ({+ data: undefined,+ isLoading: true,+ isError: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListRunningExecutions: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListOrphanedExecutions: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListFailedExecutions: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListLongRunningExecutions: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListStuckQueuedExecutions: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListInvalidExecutions: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ usePostV2StopSingleExecution: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2StopMultipleExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2StopAllLongRunningExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2CleanupOrphanedExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2CleanupAllOrphanedExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2CleanupAllStuckQueuedExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2RequeueStuckExecution: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2RequeueMultipleStuckExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ usePostV2RequeueAllStuckQueuedExecutions: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+ useGetV2ListAllUserSchedules: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ useGetV2ListOrphanedSchedules: () => ({+ data: undefined,+ isLoading: false,+ error: null,+ refetch: vi.fn(),+ }),+ usePostV2CleanupOrphanedSchedules: () => ({+ mutateAsync: vi.fn(),+ isPending: false,+ }),+}));++// Import the inner component directly since the page is async/server+import { DiagnosticsContent } from "../components/DiagnosticsContent";++describe("AdminDiagnosticsPage", () => {+ it("renders DiagnosticsContent in loading state", () => {+ render(<DiagnosticsContent />);+ expect(screen.getByText("Loading diagnostics...")).toBeDefined();+ });+});diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/components/DiagnosticsContent.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/components/DiagnosticsContent.tsxnew file mode 100644index 0000000000..2cf9da5f2d--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/components/DiagnosticsContent.tsx@@ -0,0 +1,579 @@+"use client";++import { useState } from "react";+import { Button } from "@/components/atoms/Button/Button";+import { Card } from "@/components/atoms/Card/Card";+import {+ CardContent,+ CardDescription,+ CardHeader,+ CardTitle,+} from "@/components/__legacy__/ui/card";+import { ArrowClockwise } from "@phosphor-icons/react";+import { ErrorCard } from "@/components/molecules/ErrorCard/ErrorCard";+import { useDiagnosticsContent } from "./useDiagnosticsContent";+import { ExecutionsTable } from "./ExecutionsTable";+import { SchedulesTable } from "./SchedulesTable";++export function DiagnosticsContent() {+ const {+ executionData,+ agentData,+ scheduleData,+ isLoading,+ isError,+ error,+ refresh,+ } = useDiagnosticsContent();++ const [activeTab, setActiveTab] = useState<+ "all" | "orphaned" | "failed" | "long-running" | "stuck-queued" | "invalid"+ >("all");++ if (isLoading && !executionData && !agentData) {+ return (+ <div className="flex h-64 items-center justify-center">+ <div className="text-center">+ <ArrowClockwise className="mx-auto h-8 w-8 animate-spin text-gray-400" />+ <p className="mt-2 text-gray-500">Loading diagnostics...</p>+ </div>+ </div>+ );+ }++ if (isError) {+ return (+ <ErrorCard+ httpError={error as { status?: number; message?: string }}+ onRetry={refresh}+ context="diagnostics"+ />+ );+ }++ return (+ <div className="space-y-6">+ <div className="flex items-center justify-between">+ <div>+ <h1 className="text-3xl font-bold">System Diagnostics</h1>+ <p className="text-gray-500">+ Monitor execution and agent system health+ </p>+ </div>+ <Button+ onClick={refresh}+ disabled={isLoading}+ variant="outline"+ size="small"+ >+ <ArrowClockwise+ className={`mr-2 h-4 w-4 ${isLoading ? "animate-spin" : ""}`}+ />+ Refresh+ </Button>+ </div>++ {/* Alert Cards for Critical Issues */}+ <div className="grid gap-4 md:grid-cols-3">+ {executionData && (+ <>+ {/* Orphaned Executions Alert */}+ {(executionData.orphaned_running > 0 ||+ executionData.orphaned_queued > 0) && (+ <div+ className="cursor-pointer transition-all hover:scale-105"+ onClick={() => setActiveTab("orphaned")}+ >+ <Card className="border-orange-300 bg-orange-50">+ <CardHeader className="pb-3">+ <CardTitle className="text-orange-800">+ Orphaned Executions+ </CardTitle>+ </CardHeader>+ <CardContent>+ <p className="text-3xl font-bold text-orange-900">+ {executionData.orphaned_running ++ executionData.orphaned_queued}+ </p>+ <p className="text-sm text-orange-700">+ {executionData.orphaned_running} running,{" "}+ {executionData.orphaned_queued} queued ({">"}24h old)+ </p>+ <p className="mt-2 text-xs text-orange-600">+ Click to view →+ </p>+ </CardContent>+ </Card>+ </div>+ )}++ {/* Failed Executions Alert */}+ {executionData.failed_count_24h > 0 && (+ <div+ className="cursor-pointer transition-all hover:scale-105"+ onClick={() => setActiveTab("failed")}+ >+ <Card className="border-red-300 bg-red-50">+ <CardHeader className="pb-3">+ <CardTitle className="text-red-800">+ Failed Executions (24h)+ </CardTitle>+ </CardHeader>+ <CardContent>+ <p className="text-3xl font-bold text-red-900">+ {executionData.failed_count_24h}+ </p>+ <p className="text-sm text-red-700">+ {executionData.failed_count_1h} in last hour (+ {executionData.failure_rate_24h.toFixed(1)}/hr rate)+ </p>+ <p className="mt-2 text-xs text-red-600">Click to view →</p>+ </CardContent>+ </Card>+ </div>+ )}++ {/* Long-Running Alert */}+ {executionData.stuck_running_24h > 0 && (+ <>+ <div+ className="cursor-pointer transition-all hover:scale-105"+ onClick={() => setActiveTab("long-running")}+ >+ <Card className="border-yellow-300 bg-yellow-50">+ <CardHeader className="pb-3">+ <CardTitle className="text-yellow-800">+ Long-Running Executions+ </CardTitle>+ </CardHeader>+ <CardContent>+ <p className="text-3xl font-bold text-yellow-900">+ {executionData.stuck_running_24h}+ </p>+ <p className="text-sm text-yellow-700">+ Running {">"}24h (oldest:{" "}+ {executionData.oldest_running_hours+ ? `${Math.floor(executionData.oldest_running_hours)}h`+ : "N/A"}+ )+ </p>+ <p className="mt-2 text-xs text-yellow-600">+ Click to view →+ </p>+ </CardContent>+ </Card>+ </div>+ </>+ )}++ {/* Orphaned Schedules Alert */}+ {scheduleData && scheduleData.total_orphaned > 0 && (+ <div+ className="cursor-pointer transition-all hover:scale-105"+ onClick={() => setActiveTab("all")}+ >+ <Card className="border-purple-300 bg-purple-50">+ <CardHeader className="pb-3">+ <CardTitle className="text-purple-800">+ Orphaned Schedules+ </CardTitle>+ </CardHeader>+ <CardContent>+ <p className="text-3xl font-bold text-purple-900">+ {scheduleData.total_orphaned}+ </p>+ <p className="text-sm text-purple-700">+ {scheduleData.orphaned_deleted_graph > 0 &&+ `${scheduleData.orphaned_deleted_graph} deleted graph, `}+ {scheduleData.orphaned_no_library_access > 0 &&+ `${scheduleData.orphaned_no_library_access} no access`}+ </p>+ <p className="mt-2 text-xs text-purple-600">+ Click to view schedules →+ </p>+ </CardContent>+ </Card>+ </div>+ )}++ {/* Invalid State Alert */}+ {(executionData.invalid_queued_with_start > 0 ||+ executionData.invalid_running_without_start > 0) && (+ <div+ className="cursor-pointer transition-all hover:scale-105"+ onClick={() => setActiveTab("invalid")}+ >+ <Card className="border-pink-300 bg-pink-50">+ <CardHeader className="pb-3">+ <CardTitle className="text-pink-800">+ Invalid States (Data Corruption)+ </CardTitle>+ </CardHeader>+ <CardContent>+ <p className="text-3xl font-bold text-pink-900">+ {executionData.invalid_queued_with_start ++ executionData.invalid_running_without_start}+ </p>+ <p className="text-sm text-pink-700">+ Requires manual investigation+ </p>+ <p className="mt-2 text-xs text-pink-600">+ Click to view (read-only) →+ </p>+ </CardContent>+ </Card>+ </div>+ )}+ </>+ )}+ </div>++ <div className="grid gap-6 md:grid-cols-3">+ <Card>+ <CardHeader>+ <CardTitle>Execution Queue Status</CardTitle>+ <CardDescription>+ Current execution and queue metrics+ </CardDescription>+ </CardHeader>+ <CardContent>+ {executionData ? (+ <div className="space-y-4">+ <div className="flex items-center justify-between rounded-lg border p-4">+ <div>+ <p className="text-sm font-medium text-gray-500">+ Running Executions+ </p>+ <p className="text-3xl font-bold">+ {executionData.running_executions}+ </p>+ </div>+ <div className="flex h-12 w-12 items-center justify-center rounded-full bg-green-100">+ <div className="h-6 w-6 rounded-full bg-green-500"></div>+ </div>+ </div>++ <div className="flex items-center justify-between rounded-lg border p-4">+ <div>+ <p className="text-sm font-medium text-gray-500">+ Queued in Database+ </p>+ <p className="text-3xl font-bold">+ {executionData.queued_executions_db}+ </p>+ {executionData.stuck_queued_1h > 0 && (+ <p className="text-xs text-orange-600">+ {executionData.stuck_queued_1h} stuck {">"}1h+ </p>+ )}+ </div>+ <div className="flex h-12 w-12 items-center justify-center rounded-full bg-blue-100">+ <div className="h-6 w-6 rounded-full bg-blue-500"></div>+ </div>+ </div>++ <div className="flex items-center justify-between rounded-lg border p-4">+ <div>+ <p className="text-sm font-medium text-gray-500">+ Queued in RabbitMQ+ </p>+ <p className="text-3xl font-bold">+ {executionData.queued_executions_rabbitmq === -1 ? (+ <span className="text-xl text-red-500">Error</span>+ ) : (+ executionData.queued_executions_rabbitmq+ )}+ </p>+ </div>+ <div+ className={`flex h-12 w-12 items-center justify-center rounded-full ${+ executionData.queued_executions_rabbitmq === -1+ ? "bg-red-100"+ : "bg-yellow-100"+ }`}+ >+ <div+ className={`h-6 w-6 rounded-full ${+ executionData.queued_executions_rabbitmq === -1+ ? "bg-red-500"+ : "bg-yellow-500"+ }`}+ ></div>+ </div>+ </div>++ <div className="text-xs text-gray-400">+ Last updated:{" "}+ {new Date(executionData.timestamp).toLocaleString()}+ </div>+ </div>+ ) : (+ <p className="text-gray-500">No data available</p>+ )}+ </CardContent>+ </Card>++ <Card>+ <CardHeader>+ <CardTitle>System Throughput</CardTitle>+ <CardDescription>+ Execution completion and processing rates+ </CardDescription>+ </CardHeader>+ <CardContent>+ {executionData ? (+ <div className="space-y-4">+ <div className="flex items-center justify-between rounded-lg border p-4">+ <div>+ <p className="text-sm font-medium text-gray-500">+ Completed (24h)+ </p>+ <p className="text-3xl font-bold">+ {executionData.completed_24h}+ </p>+ <p className="text-xs text-gray-600">+ {executionData.completed_1h} in last hour+ </p>+ </div>+ <div className="flex h-12 w-12 items-center justify-center rounded-full bg-green-100">+ <div className="h-6 w-6 rounded-full bg-green-500"></div>+ </div>+ </div>++ <div className="flex items-center justify-between rounded-lg border p-4">+ <div>+ <p className="text-sm font-medium text-gray-500">+ Throughput Rate+ </p>+ <p className="text-3xl font-bold">+ {executionData.throughput_per_hour.toFixed(1)}+ </p>+ <p className="text-xs text-gray-600">+ completions per hour+ </p>+ </div>+ <div className="flex h-12 w-12 items-center justify-center rounded-full bg-blue-100">+ <div className="h-6 w-6 rounded-full bg-blue-500"></div>+ </div>+ </div>++ <div className="flex items-center justify-between rounded-lg border p-4">+ <div>+ <p className="text-sm font-medium text-gray-500">+ Cancel Queue Depth+ </p>+ <p className="text-3xl font-bold">+ {executionData.cancel_queue_depth === -1 ? (+ <span className="text-xl text-red-500">Error</span>+ ) : (+ executionData.cancel_queue_depth+ )}+ </p>+ </div>+ <div className="flex h-12 w-12 items-center justify-center rounded-full bg-purple-100">+ <div className="h-6 w-6 rounded-full bg-purple-500"></div>+ </div>+ </div>++ <div className="text-xs text-gray-400">+ Last updated:{" "}+ {new Date(executionData.timestamp).toLocaleString()}+ </div>+ </div>+ ) : (+ <p className="text-gray-500">No data available</p>+ )}+ </CardContent>+ </Card>++ <Card>+ <CardHeader>+ <CardTitle>Schedules</CardTitle>+ <CardDescription>+ Scheduled agent executions and health+ </CardDescription>+ </CardHeader>+ <CardContent>+ {scheduleData ? (+ <div className="space-y-4">+ <div className="flex items-center justify-between rounded-lg border p-4">+ <div>+ <p className="text-sm font-medium text-gray-500">+ User Schedules+ </p>+ <p className="text-3xl font-bold">+ {scheduleData.user_schedules}+ </p>+ {scheduleData.total_orphaned > 0 && (+ <p className="text-xs text-orange-600">+ {scheduleData.total_orphaned} orphaned+ </p>+ )}+ </div>+ <div className="flex h-12 w-12 items-center justify-center rounded-full bg-purple-100">+ <div className="h-6 w-6 rounded-full bg-purple-500"></div>+ </div>+ </div>++ <div className="flex items-center justify-between rounded-lg border p-4">+ <div>+ <p className="text-sm font-medium text-gray-500">+ Upcoming Runs (1h)+ </p>+ <p className="text-3xl font-bold">+ {scheduleData.total_runs_next_hour}+ </p>+ <p className="text-xs text-gray-600">+ from {scheduleData.schedules_next_hour} schedule+ {scheduleData.schedules_next_hour !== 1 ? "s" : ""}+ </p>+ </div>+ <div className="flex h-12 w-12 items-center justify-center rounded-full bg-blue-100">+ <div className="h-6 w-6 rounded-full bg-blue-500"></div>+ </div>+ </div>++ <div className="flex items-center justify-between rounded-lg border p-4">+ <div>+ <p className="text-sm font-medium text-gray-500">+ Upcoming Runs (24h)+ </p>+ <p className="text-3xl font-bold">+ {scheduleData.total_runs_next_24h}+ </p>+ <p className="text-xs text-gray-600">+ from {scheduleData.schedules_next_24h} schedule+ {scheduleData.schedules_next_24h !== 1 ? "s" : ""}+ </p>+ </div>+ <div className="flex h-12 w-12 items-center justify-center rounded-full bg-green-100">+ <div className="h-6 w-6 rounded-full bg-green-500"></div>+ </div>+ </div>++ <div className="text-xs text-gray-400">+ Last updated:{" "}+ {new Date(scheduleData.timestamp).toLocaleString()}+ </div>+ </div>+ ) : (+ <p className="text-gray-500">No data available</p>+ )}+ </CardContent>+ </Card>+ </div>++ <Card>+ <CardHeader>+ <CardTitle>Diagnostic Information</CardTitle>+ <CardDescription>+ Understanding metrics and tabs for on-call diagnostics+ </CardDescription>+ </CardHeader>+ <CardContent>+ <div className="space-y-3 text-sm">+ <div>+ <p className="font-semibold text-orange-700">+ 🟠 Orphaned Executions:+ </p>+ <p className="text-gray-600">+ Executions {">"}24h old in database but not actually running in+ executor. Usually from executor restarts/crashes. Safe to+ cleanup (marks as FAILED in DB).+ </p>+ </div>+ <div>+ <p className="font-semibold text-blue-700">+ 🔵 Stuck Queued Executions:+ </p>+ <p className="text-gray-600">+ QUEUED {">"}1h but never started. Not in RabbitMQ queue. Can+ cleanup (safe) or requeue (⚠️ costs credits - only if temporary+ issue like RabbitMQ purge).+ </p>+ </div>+ <div>+ <p className="font-semibold text-yellow-700">+ 🟡 Long-Running Executions:+ </p>+ <p className="text-gray-600">+ RUNNING status {">"}24h. May be legitimately long jobs or stuck.+ Review before stopping. Sends cancel signal to executor.+ </p>+ </div>+ <div>+ <p className="font-semibold text-red-700">+ 🔴 Failed Executions:+ </p>+ <p className="text-gray-600">+ Executions that failed in last 24h. View error messages to+ identify patterns. Spike in failures indicates system issues.+ </p>+ </div>+ <div>+ <p className="font-semibold text-pink-700">+ 🩷 Invalid States (Data Corruption):+ </p>+ <p className="text-gray-600">+ Executions in impossible states (QUEUED with startedAt, RUNNING+ without startedAt). Indicates DB corruption, race conditions, or+ crashes. Each requires manual investigation - no bulk actions+ provided.+ </p>+ </div>+ <div>+ <p className="font-semibold">Throughput Metrics:</p>+ <p className="text-gray-600">+ Completions per hour shows system productivity. Declining+ throughput indicates performance degradation or executor issues.+ </p>+ </div>+ <div>+ <p className="font-semibold">Queue Health:</p>+ <p className="text-gray-600">+ RabbitMQ depths should be low ({"<"}100). High queues indicate+ executor can't keep up. Cancel queue backlog indicates+ executor processing issues.+ </p>+ </div>+ </div>+ </CardContent>+ </Card>++ {/* Add Executions Table with tab counts */}+ <ExecutionsTable+ onRefresh={refresh}+ initialTab={activeTab}+ onTabChange={setActiveTab}+ diagnosticsData={+ executionData+ ? {+ orphaned_running: executionData.orphaned_running,+ orphaned_queued: executionData.orphaned_queued,+ failed_count_24h: executionData.failed_count_24h,+ stuck_running_24h: executionData.stuck_running_24h,+ stuck_queued_1h: executionData.stuck_queued_1h,+ invalid_queued_with_start:+ executionData.invalid_queued_with_start,+ invalid_running_without_start:+ executionData.invalid_running_without_start,+ }+ : undefined+ }+ />++ {/* Add Schedules Table */}+ <SchedulesTable+ onRefresh={refresh}+ diagnosticsData={+ scheduleData+ ? {+ total_orphaned: scheduleData.total_orphaned,+ user_schedules: scheduleData.user_schedules,+ }+ : undefined+ }+ />+ </div>+ );+}diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/components/ExecutionsTable.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/components/ExecutionsTable.tsxnew file mode 100644index 0000000000..6c27256845--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/components/ExecutionsTable.tsx@@ -0,0 +1,1079 @@+"use client";++import { Button } from "@/components/atoms/Button/Button";+import { Card } from "@/components/atoms/Card/Card";+import { ErrorCard } from "@/components/molecules/ErrorCard/ErrorCard";+import {+ Dialog,+ DialogContent,+ DialogDescription,+ DialogFooter,+ DialogHeader,+ DialogTitle,+} from "@/components/__legacy__/ui/dialog";+import { toast } from "@/components/molecules/Toast/use-toast";+import {+ StopCircleIcon,+ ArrowClockwise,+ Stop,+ CaretLeft,+ CaretRight,+ Copy,+} from "@phosphor-icons/react";+import React, { useState } from "react";+import {+ Table,+ TableHeader,+ TableBody,+ TableHead,+ TableRow,+ TableCell,+} from "@/components/__legacy__/ui/table";+import { Checkbox } from "@/components/__legacy__/ui/checkbox";+import {+ CardHeader,+ CardTitle,+ CardContent,+} from "@/components/__legacy__/ui/card";+import {+ useGetV2ListRunningExecutions,+ useGetV2ListOrphanedExecutions,+ useGetV2ListFailedExecutions,+ useGetV2ListLongRunningExecutions,+ useGetV2ListStuckQueuedExecutions,+ useGetV2ListInvalidExecutions,+ usePostV2StopSingleExecution,+ usePostV2StopMultipleExecutions,+ usePostV2StopAllLongRunningExecutions,+ usePostV2CleanupOrphanedExecutions,+ usePostV2CleanupAllOrphanedExecutions,+ usePostV2CleanupAllStuckQueuedExecutions,+ usePostV2RequeueStuckExecution,+ usePostV2RequeueMultipleStuckExecutions,+ usePostV2RequeueAllStuckQueuedExecutions,+} from "@/app/api/__generated__/endpoints/admin/admin";+import {+ TabsLine,+ TabsLineContent,+ TabsLineList,+ TabsLineTrigger,+} from "@/components/molecules/TabsLine/TabsLine";++interface RunningExecutionDetail {+ execution_id: string;+ graph_id: string;+ graph_name: string;+ graph_version: number;+ user_id: string;+ user_email: string | null;+ status: string;+ created_at: string;+ started_at: string | null;+ queue_status: string | null;+ failed_at?: string | null;+ error_message?: string | null;+}++interface MutationResponseData {+ success: boolean;+ message: string;+ stopped_count?: number;+ requeued_count?: number;+}++interface ExecutionsTableProps {+ onRefresh?: () => void;+ initialTab?:+ | "all"+ | "orphaned"+ | "failed"+ | "long-running"+ | "stuck-queued"+ | "invalid";+ onTabChange?: (+ tab:+ | "all"+ | "orphaned"+ | "failed"+ | "long-running"+ | "stuck-queued"+ | "invalid",+ ) => void;+ diagnosticsData?: {+ orphaned_running: number;+ orphaned_queued: number;+ failed_count_24h: number;+ stuck_running_24h: number;+ stuck_queued_1h: number;+ invalid_queued_with_start: number;+ invalid_running_without_start: number;+ };+}++export function ExecutionsTable({+ onRefresh,+ initialTab = "all",+ onTabChange,+ diagnosticsData,+}: ExecutionsTableProps) {+ const [activeTab, setActiveTab] = useState<+ "all" | "orphaned" | "failed" | "long-running" | "stuck-queued" | "invalid"+ >(initialTab);+ const [selectedIds, setSelectedIds] = useState<Set<string>>(new Set());+ const [showStopDialog, setShowStopDialog] = useState(false);+ const [stopTarget, setStopTarget] = useState<"single" | "selected" | "all">(+ "single",+ );+ const [stopMode, setStopMode] = useState<"stop" | "cleanup" | "requeue">(+ "stop",+ );+ const [singleStopId, setSingleStopId] = useState<string | null>(null);+ const [currentPage, setCurrentPage] = useState(1);+ const [pageSize] = useState(10);++ type ExecutionTab =+ | "all"+ | "orphaned"+ | "failed"+ | "long-running"+ | "stuck-queued"+ | "invalid";++ function handleTabChange(newTab: string) {+ const tab = newTab as ExecutionTab;+ setActiveTab(tab);+ setCurrentPage(1);+ setSelectedIds(new Set());+ if (onTabChange) onTabChange(tab);+ }++ // Sync with external tab changes (from clicking alert cards)+ React.useEffect(() => {+ if (initialTab !== activeTab) {+ setActiveTab(initialTab);+ setCurrentPage(1);+ setSelectedIds(new Set());+ }+ }, [initialTab]);++ // Fetch data based on active tab+ const runningQuery = useGetV2ListRunningExecutions(+ {+ limit: pageSize,+ offset: (currentPage - 1) * pageSize,+ },+ { query: { enabled: activeTab === "all" } },+ );++ const orphanedQuery = useGetV2ListOrphanedExecutions(+ {+ limit: pageSize,+ offset: (currentPage - 1) * pageSize,+ },+ { query: { enabled: activeTab === "orphaned" } },+ );++ const failedQuery = useGetV2ListFailedExecutions(+ {+ limit: pageSize,+ offset: (currentPage - 1) * pageSize,+ hours: 24,+ },+ { query: { enabled: activeTab === "failed" } },+ );++ // Long-running has dedicated endpoint (RUNNING status >24h only)+ const longRunningQuery = useGetV2ListLongRunningExecutions(+ {+ limit: pageSize,+ offset: (currentPage - 1) * pageSize,+ },+ { query: { enabled: activeTab === "long-running" } },+ );++ // Stuck queued has dedicated endpoint (QUEUED >1h)+ const stuckQueuedQuery = useGetV2ListStuckQueuedExecutions(+ {+ limit: pageSize,+ offset: (currentPage - 1) * pageSize,+ },+ { query: { enabled: activeTab === "stuck-queued" } },+ );++ // Invalid states endpoint (read-only, data corruption cases)+ const invalidQuery = useGetV2ListInvalidExecutions(+ {+ limit: pageSize,+ offset: (currentPage - 1) * pageSize,+ },+ { query: { enabled: activeTab === "invalid" } },+ );++ // Select active query based on tab+ const activeQuery =+ activeTab === "orphaned"+ ? orphanedQuery+ : activeTab === "failed"+ ? failedQuery+ : activeTab === "long-running"+ ? longRunningQuery+ : activeTab === "stuck-queued"+ ? stuckQueuedQuery+ : activeTab === "invalid"+ ? invalidQuery+ : runningQuery;++ const { data: executionsResponse, isLoading, error, refetch } = activeQuery;++ const responseData = executionsResponse?.data as+ | { executions: RunningExecutionDetail[]; total: number }+ | undefined;+ const executions = responseData?.executions || [];+ const total = responseData?.total || 0;++ // Stop single execution mutation+ const { mutateAsync: stopSingleExecution, isPending: isStoppingSingle } =+ usePostV2StopSingleExecution();++ // Stop multiple executions mutation+ const { mutateAsync: stopMultipleExecutions, isPending: isStoppingMultiple } =+ usePostV2StopMultipleExecutions();++ // Cleanup orphaned executions mutation+ const { mutateAsync: cleanupOrphanedExecutions, isPending: isCleaningUp } =+ usePostV2CleanupOrphanedExecutions();++ // Requeue stuck queued executions mutation+ const { mutateAsync: requeueSingleExecution, isPending: isRequeuingSingle } =+ usePostV2RequeueStuckExecution();++ const {+ mutateAsync: requeueMultipleExecutions,+ isPending: isRequeueingMultiple,+ } = usePostV2RequeueMultipleStuckExecutions();++ const { mutateAsync: requeueAllStuck, isPending: isRequeueingAll } =+ usePostV2RequeueAllStuckQueuedExecutions();++ const { mutateAsync: cleanupAllOrphaned, isPending: isCleaningUpAll } =+ usePostV2CleanupAllOrphanedExecutions();++ const {+ mutateAsync: cleanupAllStuckQueued,+ isPending: isCleaningUpAllStuckQueued,+ } = usePostV2CleanupAllStuckQueuedExecutions();++ const {+ mutateAsync: stopAllLongRunning,+ isPending: isStoppingAllLongRunning,+ } = usePostV2StopAllLongRunningExecutions();++ const isStopping =+ isStoppingSingle ||+ isStoppingMultiple ||+ isCleaningUp ||+ isRequeuingSingle ||+ isRequeueingMultiple ||+ isRequeueingAll ||+ isCleaningUpAll ||+ isCleaningUpAllStuckQueued ||+ isStoppingAllLongRunning;++ const now = new Date();++ // Determine which executions are orphaned+ // If viewing the "orphaned" tab, trust backend filtering - all executions are orphaned+ // Otherwise, calculate based on created_at > 24h+ const orphanedIds = new Set(+ activeTab === "orphaned"+ ? executions.map((e: RunningExecutionDetail) => e.execution_id)+ : executions+ .filter((e: RunningExecutionDetail) => {+ const createdDate = new Date(e.created_at);+ const ageHours =+ (now.getTime() - createdDate.getTime()) / (1000 * 60 * 60);+ return ageHours > 24;+ })+ .map((e: RunningExecutionDetail) => e.execution_id),+ );++ const selectedOrphanedIds = Array.from(selectedIds).filter((id) =>+ orphanedIds.has(id),+ );+ const hasOrphanedSelected = selectedOrphanedIds.length > 0;++ // Show error toast if fetching fails (in useEffect to avoid render side-effects)+ React.useEffect(() => {+ if (error) {+ toast({+ title: "Error",+ description: "Failed to fetch executions",+ variant: "destructive",+ });+ }+ }, [error]);++ const handleSelectAll = (checked: boolean) => {+ if (checked) {+ setSelectedIds(+ new Set(executions.map((e: RunningExecutionDetail) => e.execution_id)),+ );+ } else {+ setSelectedIds(new Set());+ }+ };++ const handleSelectExecution = (id: string, checked: boolean) => {+ const newSelected = new Set(selectedIds);+ if (checked) {+ newSelected.add(id);+ } else {+ newSelected.delete(id);+ }+ setSelectedIds(newSelected);+ };++ const confirmStop = (+ target: "single" | "selected" | "all",+ mode: "stop" | "cleanup" | "requeue",+ singleId?: string,+ ) => {+ setStopTarget(target);+ setStopMode(mode);+ setSingleStopId(singleId || null);+ setShowStopDialog(true);+ };++ const handleStop = async () => {+ setShowStopDialog(false);++ try {+ if (stopTarget === "single" && singleStopId) {+ // Single execution - use appropriate method+ const result =+ stopMode === "cleanup"+ ? await cleanupOrphanedExecutions({+ data: { execution_ids: [singleStopId] },+ })+ : stopMode === "requeue"+ ? await requeueSingleExecution({+ data: { execution_id: singleStopId },+ })+ : await stopSingleExecution({+ data: { execution_id: singleStopId },+ });++ toast({+ title: "Success",+ description:+ (result.data as MutationResponseData)?.message ||+ (stopMode === "cleanup"+ ? "Orphaned execution cleaned up"+ : stopMode === "requeue"+ ? "Execution requeued"+ : "Execution stopped"),+ });+ } else {+ // Multiple executions+ if (stopMode === "requeue") {+ // Requeue stuck queued executions+ if (stopTarget === "all") {+ // Use ALL endpoint for entire dataset+ const result = await requeueAllStuck();++ toast({+ title: "Success",+ description:+ (result.data as MutationResponseData)?.message ||+ `Requeued ${(result.data as MutationResponseData)?.requeued_count || 0} stuck executions`,+ });+ } else {+ // Selected only+ const allIds = Array.from(selectedIds);+ const result = await requeueMultipleExecutions({+ data: { execution_ids: allIds },+ });++ toast({+ title: "Success",+ description:+ (result.data as MutationResponseData)?.message ||+ `Requeued ${(result.data as MutationResponseData)?.requeued_count || 0} execution(s)`,+ });+ }+ } else if (stopMode === "cleanup") {+ // Cleanup executions+ if (stopTarget === "all" && activeTab === "orphaned") {+ // Use ALL endpoint for orphaned tab (>24h old)+ const result = await cleanupAllOrphaned();++ toast({+ title: "Success",+ description:+ (result.data as MutationResponseData)?.message ||+ `Cleaned up ${(result.data as MutationResponseData)?.stopped_count || 0} orphaned executions`,+ });+ } else if (stopTarget === "all" && activeTab === "stuck-queued") {+ // Use ALL endpoint for stuck-queued tab (>1h old)+ const result = await cleanupAllStuckQueued();++ toast({+ title: "Success",+ description:+ (result.data as MutationResponseData)?.message ||+ `Cleaned up ${(result.data as MutationResponseData)?.stopped_count || 0} stuck queued executions`,+ });+ } else {+ // Selected or other tabs+ const allIds =+ stopTarget === "selected"+ ? Array.from(selectedIds)+ : executions.map((e: RunningExecutionDetail) => e.execution_id);++ const result = await cleanupOrphanedExecutions({+ data: { execution_ids: allIds },+ });++ toast({+ title: "Success",+ description:+ (result.data as MutationResponseData)?.message ||+ `Cleaned up ${(result.data as MutationResponseData)?.stopped_count || 0} execution(s)`,+ });+ }+ } else {+ // Stop - handle long-running ALL or split active/orphaned+ if (stopTarget === "all" && activeTab === "long-running") {+ // Use ALL endpoint for long-running tab+ const result = await stopAllLongRunning();++ toast({+ title: "Success",+ description:+ (result.data as MutationResponseData)?.message ||+ `Stopped ${(result.data as MutationResponseData)?.stopped_count || 0} long-running executions`,+ });+ } else {+ // Stop selected - intelligently split between active and orphaned+ const activeIds: string[] = [];+ const orphanedIdsToCleanup: string[] = [];++ const allIds = Array.from(selectedIds);++ // Split into active vs orphaned+ allIds.forEach((id: string) => {+ if (orphanedIds.has(id)) {+ orphanedIdsToCleanup.push(id);+ } else {+ activeIds.push(id);+ }+ });++ // Execute both operations in parallel+ const results = await Promise.all([+ activeIds.length > 0+ ? stopMultipleExecutions({+ data: { execution_ids: activeIds },+ })+ : Promise.resolve(null),+ orphanedIdsToCleanup.length > 0+ ? cleanupOrphanedExecutions({+ data: { execution_ids: orphanedIdsToCleanup },+ })+ : Promise.resolve(null),+ ]);++ const stoppedCount = results[0]+ ? (results[0].data as MutationResponseData)?.stopped_count || 0+ : 0;+ const cleanedCount = results[1]+ ? (results[1].data as MutationResponseData)?.stopped_count || 0+ : 0;++ toast({+ title: "Success",+ description:+ stoppedCount > 0 && cleanedCount > 0+ ? `Stopped ${stoppedCount} active and cleaned ${cleanedCount} orphaned executions`+ : stoppedCount > 0+ ? `Stopped ${stoppedCount} execution(s)`+ : `Cleaned ${cleanedCount} orphaned execution(s)`,+ });+ }+ }+ }++ // Clear selections and refresh+ setSelectedIds(new Set());+ await refetch();+ if (onRefresh) {+ onRefresh();+ }+ } catch (err: unknown) {+ console.error("Error stopping/cleaning executions:", err);+ toast({+ title: "Error",+ description:+ err instanceof Error+ ? err.message+ : "Failed to stop/cleanup executions",+ variant: "destructive",+ });+ }+ };++ const totalPages = Math.ceil(total / pageSize);++ return (+ <>+ <Card>+ <TabsLine value={activeTab} onValueChange={handleTabChange}>+ <CardHeader>+ <div className="flex items-center justify-between">+ <CardTitle>Executions</CardTitle>+ <div className="flex gap-2">+ {/* Show Cleanup and Requeue buttons for stuck-queued tab */}+ {activeTab === "stuck-queued" && total > 0 && (+ <>+ <Button+ variant="outline"+ size="small"+ onClick={() => confirmStop("all", "cleanup")}+ disabled={isStopping}+ className="border-orange-500 text-orange-700 hover:bg-orange-50"+ >+ <StopCircleIcon className="mr-2 h-4 w-4" />+ Cleanup All ({total})+ </Button>+ <Button+ variant="outline"+ size="small"+ onClick={() => confirmStop("all", "requeue")}+ disabled={isStopping}+ className="border-blue-500 text-blue-700 hover:bg-blue-50"+ >+ <ArrowClockwise className="mr-2 h-4 w-4" />+ Requeue All ({total})+ </Button>+ </>+ )}+ {selectedIds.size > 0 &&+ activeTab !== "stuck-queued" &&+ activeTab !== "invalid" && (+ <Button+ variant="destructive"+ size="small"+ onClick={() => confirmStop("selected", "stop")}+ disabled={isStopping}+ >+ <StopCircleIcon className="mr-2 h-4 w-4" />+ Stop Selected ({selectedIds.size})+ {hasOrphanedSelected && (+ <span className="ml-1 text-xs text-orange-200">+ ({selectedOrphanedIds.length} orphaned)+ </span>+ )}+ </Button>+ )}+ {/* Only show Stop All for specific tabs, not "all" tab */}+ {activeTab === "long-running" && total > 0 && (+ <Button+ variant="destructive"+ size="small"+ onClick={() => confirmStop("all", "stop")}+ disabled={isStopping}+ >+ <StopCircleIcon className="mr-2 h-4 w-4" />+ Stop All Long-Running ({total})+ </Button>+ )}+ {activeTab === "failed" && selectedIds.size === 0 && (+ <div className="px-3 text-sm text-gray-500">+ View-only (select to delete)+ </div>+ )}+ {activeTab === "invalid" && (+ <div className="rounded-md bg-pink-50 px-3 py-2 text-sm text-pink-700">+ ⚠️ Read-only: Invalid states require manual investigation+ </div>+ )}+ <Button+ variant="outline"+ size="small"+ onClick={() => {+ refetch();+ if (onRefresh) onRefresh();+ }}+ disabled={isLoading}+ >+ <ArrowClockwise+ className={`h-4 w-4 ${isLoading ? "animate-spin" : ""}`}+ />+ </Button>+ </div>+ </div>++ {/* Tabs for filtering */}+ <TabsLineList className="px-6">+ <TabsLineTrigger value="all">+ All+ {activeTab === "all" && ` (${total})`}+ </TabsLineTrigger>+ <TabsLineTrigger value="orphaned">+ Orphaned+ {diagnosticsData &&+ ` (${diagnosticsData.orphaned_running + diagnosticsData.orphaned_queued})`}+ </TabsLineTrigger>+ <TabsLineTrigger value="stuck-queued">+ Stuck Queued+ {diagnosticsData && ` (${diagnosticsData.stuck_queued_1h})`}+ </TabsLineTrigger>+ <TabsLineTrigger value="long-running">+ Long-Running+ {diagnosticsData && ` (${diagnosticsData.stuck_running_24h})`}+ </TabsLineTrigger>+ <TabsLineTrigger value="failed">+ Failed+ {diagnosticsData && ` (${diagnosticsData.failed_count_24h})`}+ </TabsLineTrigger>+ <TabsLineTrigger value="invalid">+ Invalid States+ {diagnosticsData &&+ ` (${diagnosticsData.invalid_queued_with_start + diagnosticsData.invalid_running_without_start})`}+ </TabsLineTrigger>+ </TabsLineList>+ </CardHeader>++ <TabsLineContent value={activeTab}>+ <CardContent>+ {error ? (+ <ErrorCard+ httpError={error as { status?: number; message?: string }}+ onRetry={() => refetch()}+ context="executions"+ />+ ) : isLoading && executions.length === 0 ? (+ <div className="flex h-32 items-center justify-center">+ <ArrowClockwise className="h-6 w-6 animate-spin text-gray-400" />+ </div>+ ) : executions.length === 0 ? (+ <div className="py-8 text-center text-gray-500">+ No running executions+ </div>+ ) : (+ <>+ <Table>+ <TableHeader>+ <TableRow>+ <TableHead className="w-12">+ <Checkbox+ checked={+ selectedIds.size === executions.length &&+ executions.length > 0+ }+ onCheckedChange={handleSelectAll}+ disabled={activeTab === "invalid"}+ />+ </TableHead>+ <TableHead>Execution ID</TableHead>+ <TableHead>Agent Name</TableHead>+ <TableHead>Version</TableHead>+ <TableHead>User</TableHead>+ <TableHead>Status</TableHead>+ <TableHead>Age</TableHead>+ <TableHead>+ {activeTab === "failed" ? "Failed At" : "Started At"}+ </TableHead>+ {activeTab === "failed" && (+ <TableHead>Error Message</TableHead>+ )}+ <TableHead className="w-20">Actions</TableHead>+ </TableRow>+ </TableHeader>+ <TableBody>+ {executions.map((execution: RunningExecutionDetail) => {+ const isOrphaned = orphanedIds.has(+ execution.execution_id,+ );+ return (+ <TableRow+ key={execution.execution_id}+ className={+ isOrphaned+ ? "bg-orange-50 hover:bg-orange-100"+ : ""+ }+ >+ <TableCell>+ <Checkbox+ checked={selectedIds.has(+ execution.execution_id,+ )}+ onCheckedChange={(checked) =>+ handleSelectExecution(+ execution.execution_id,+ checked as boolean,+ )+ }+ disabled={activeTab === "invalid"}+ />+ </TableCell>+ <TableCell className="font-mono text-xs">+ <div+ className="group flex cursor-pointer items-center gap-1 hover:text-gray-700"+ onClick={() => {+ navigator.clipboard.writeText(+ execution.execution_id,+ );+ toast({+ title: "Copied",+ description:+ "Execution ID copied to clipboard",+ });+ }}+ title="Click to copy full execution ID"+ >+ {execution.execution_id.substring(0, 8)}...+ <Copy className="h-3 w-3 opacity-0 transition-opacity group-hover:opacity-100" />+ </div>+ </TableCell>+ <TableCell>{execution.graph_name}</TableCell>+ <TableCell>{execution.graph_version}</TableCell>+ <TableCell>+ <div>+ {execution.user_email || (+ <span className="text-gray-400">Unknown</span>+ )}+ </div>+ <div+ className="group flex cursor-pointer items-center gap-1 font-mono text-xs text-gray-500 hover:text-gray-700"+ onClick={() => {+ navigator.clipboard.writeText(+ execution.user_id,+ );+ toast({+ title: "Copied",+ description: "User ID copied to clipboard",+ });+ }}+ title="Click to copy full user ID"+ >+ {execution.user_id.substring(0, 8)}...+ <Copy className="h-3 w-3 opacity-0 transition-opacity group-hover:opacity-100" />+ </div>+ </TableCell>+ <TableCell>+ <span+ className={`inline-flex rounded-full px-2 py-1 text-xs font-semibold ${+ execution.status === "RUNNING"+ ? "bg-green-100 text-green-800"+ : "bg-yellow-100 text-yellow-800"+ }`}+ >+ {execution.status}+ </span>+ </TableCell>+ <TableCell>+ {(() => {+ if (!execution.started_at)+ return "Never started";+ const ageMs =+ now.getTime() -+ new Date(execution.started_at).getTime();+ const ageHours = ageMs / (1000 * 60 * 60);+ const ageDays = Math.floor(ageHours / 24);+ const remainingHours = Math.floor(+ ageHours % 24,+ );++ if (ageDays > 0) {+ return (+ <span+ className={+ ageDays > 1+ ? "font-semibold text-orange-600"+ : ""+ }+ >+ {ageDays}d {remainingHours}h+ </span>+ );+ } else {+ return `${remainingHours}h`;+ }+ })()}+ </TableCell>+ <TableCell>+ {activeTab === "failed"+ ? execution.failed_at+ ? new Date(+ execution.failed_at,+ ).toLocaleString()+ : "-"+ : execution.started_at+ ? new Date(+ execution.started_at,+ ).toLocaleString()+ : "-"}+ </TableCell>+ {activeTab === "failed" && (+ <TableCell className="max-w-xs truncate">+ <span+ className="text-xs text-red-600"+ title={execution.error_message || ""}+ >+ {execution.error_message ||+ "No error message"}+ </span>+ </TableCell>+ )}+ <TableCell>+ <div className="flex gap-1">+ {activeTab === "stuck-queued" ? (+ <>+ <Button+ variant="ghost"+ size="small"+ onClick={() =>+ confirmStop(+ "single",+ "cleanup",+ execution.execution_id,+ )+ }+ disabled={isStopping}+ className="text-orange-600 hover:bg-orange-50"+ title="Cleanup (mark as FAILED)"+ >+ <StopCircleIcon className="h-4 w-4" />+ </Button>+ <Button+ variant="ghost"+ size="small"+ onClick={() =>+ confirmStop(+ "single",+ "requeue",+ execution.execution_id,+ )+ }+ disabled={isStopping}+ className="text-blue-600 hover:bg-blue-50"+ title="Requeue (send to RabbitMQ)"+ >+ <ArrowClockwise className="h-4 w-4" />+ </Button>+ </>+ ) : (+ <Button+ variant="ghost"+ size="small"+ onClick={() => {+ const isOrphaned = orphanedIds.has(+ execution.execution_id,+ );+ confirmStop(+ "single",+ isOrphaned ? "cleanup" : "stop",+ execution.execution_id,+ );+ }}+ disabled={isStopping}+ className={+ orphanedIds.has(execution.execution_id)+ ? "text-orange-600 hover:bg-orange-50"+ : ""+ }+ >+ <Stop className="h-4 w-4" />+ </Button>+ )}+ </div>+ </TableCell>+ </TableRow>+ );+ })}+ </TableBody>+ </Table>++ {totalPages > 1 && (+ <div className="mt-4 flex items-center justify-between">+ <div className="text-sm text-gray-600">+ Showing {(currentPage - 1) * pageSize + 1} to{" "}+ {Math.min(currentPage * pageSize, total)} of {total}{" "}+ executions+ </div>+ <div className="flex gap-2">+ <Button+ variant="outline"+ size="small"+ onClick={() => setCurrentPage(currentPage - 1)}+ disabled={currentPage === 1}+ >+ <CaretLeft className="h-4 w-4" />+ Previous+ </Button>+ <div className="flex items-center px-3">+ Page {currentPage} of {totalPages}+ </div>+ <Button+ variant="outline"+ size="small"+ onClick={() => setCurrentPage(currentPage + 1)}+ disabled={currentPage === totalPages}+ >+ Next+ <CaretRight className="h-4 w-4" />+ </Button>+ </div>+ </div>+ )}+ </>+ )}+ </CardContent>+ </TabsLineContent>+ </TabsLine>+ </Card>++ <Dialog open={showStopDialog} onOpenChange={setShowStopDialog}>+ <DialogContent>+ <DialogHeader>+ <DialogTitle>+ {stopMode === "cleanup"+ ? "Confirm Cleanup Orphaned Executions"+ : stopMode === "requeue"+ ? "Confirm Requeue Stuck Executions"+ : "Confirm Stop Executions"}+ </DialogTitle>+ <DialogDescription>+ {stopMode === "requeue" ? (+ <>+ {stopTarget === "single" && (+ <>Are you sure you want to requeue this stuck execution?</>+ )}+ {stopTarget === "selected" && (+ <>+ Are you sure you want to requeue {selectedIds.size}{" "}+ selected execution(s)?+ </>+ )}+ {stopTarget === "all" && (+ <>+ Are you sure you want to requeue ALL {total} stuck+ executions?+ </>+ )}+ <br />+ <br />+ <strong className="text-blue-700">⚠️ Warning:</strong> This+ will publish these executions to RabbitMQ to be processed+ again. This <strong>will cost credits</strong> and may fail+ again if the original issue persists.+ <br />+ <br />+ Only requeue if you believe the executions are stuck due to a+ temporary issue (executor restart, RabbitMQ purge, etc).+ </>+ ) : stopMode === "cleanup" ? (+ <>+ {stopTarget === "single" && (+ <>+ Are you sure you want to cleanup this orphaned execution?+ </>+ )}+ {stopTarget === "selected" && (+ <>+ Are you sure you want to cleanup{" "}+ {selectedOrphanedIds.length} orphaned execution(s)?+ </>+ )}+ {stopTarget === "all" && (+ <>+ Are you sure you want to cleanup ALL {orphanedIds.size}{" "}+ orphaned executions?+ </>+ )}+ <br />+ <br />+ <strong>Orphaned executions</strong> are {">"}24h old and not+ actually running in the executor. This will mark them as+ FAILED in the database only (no cancel signal sent).+ </>+ ) : (+ <>+ {stopTarget === "single" && (+ <>Are you sure you want to stop this execution?</>+ )}+ {stopTarget === "selected" && (+ <>+ Are you sure you want to stop {selectedIds.size} selected+ execution(s)?+ {hasOrphanedSelected && (+ <>+ <br />+ <br />+ <span className="text-orange-600">+ Includes {selectedOrphanedIds.length} orphaned+ execution(s) that will be cleaned up directly.+ </span>+ </>+ )}+ </>+ )}+ {stopTarget === "all" && (+ <>+ Are you sure you want to stop ALL {executions.length}{" "}+ execution(s)?+ {orphanedIds.size > 0 && (+ <>+ <br />+ <br />+ <span className="text-orange-600">+ Includes {orphanedIds.size} orphaned execution(s) (+ {">"}24h old) that will be cleaned up directly.+ </span>+ </>+ )}+ </>+ )}+ <br />+ <br />+ This will automatically:+ <ul className="mt-2 list-disc pl-5 text-sm">+ <li>Send cancel signals for active executions</li>+ <li>+ Clean up orphaned executions ({">"}24h old) directly in DB+ </li>+ <li>Mark all as FAILED</li>+ </ul>+ </>+ )}+ </DialogDescription>+ </DialogHeader>+ <DialogFooter>+ <Button variant="outline" onClick={() => setShowStopDialog(false)}>+ Cancel+ </Button>+ <Button+ variant="destructive"+ onClick={handleStop}+ className={+ stopMode === "cleanup"+ ? "bg-orange-600 hover:bg-orange-700"+ : stopMode === "requeue"+ ? "bg-blue-600 hover:bg-blue-700"+ : "bg-red-600 hover:bg-red-700"+ }+ >+ {stopMode === "cleanup"+ ? "Cleanup Orphaned"+ : stopMode === "requeue"+ ? "Requeue Executions"+ : "Stop Executions"}+ </Button>+ </DialogFooter>+ </DialogContent>+ </Dialog>+ </>+ );+}diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/components/SchedulesTable.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/components/SchedulesTable.tsxnew file mode 100644index 0000000000..4ad268995b--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/components/SchedulesTable.tsx@@ -0,0 +1,455 @@+"use client";++import { Button } from "@/components/atoms/Button/Button";+import { Card } from "@/components/atoms/Card/Card";+import {+ Dialog,+ DialogContent,+ DialogDescription,+ DialogFooter,+ DialogHeader,+ DialogTitle,+} from "@/components/__legacy__/ui/dialog";+import { toast } from "@/components/molecules/Toast/use-toast";+import { ArrowClockwise, Trash, Copy } from "@phosphor-icons/react";+import React, { useState } from "react";+import {+ Table,+ TableHeader,+ TableBody,+ TableHead,+ TableRow,+ TableCell,+} from "@/components/__legacy__/ui/table";+import { Checkbox } from "@/components/__legacy__/ui/checkbox";+import {+ CardHeader,+ CardTitle,+ CardContent,+} from "@/components/__legacy__/ui/card";+import {+ useGetV2ListAllUserSchedules,+ useGetV2ListOrphanedSchedules,+ usePostV2CleanupOrphanedSchedules,+} from "@/app/api/__generated__/endpoints/admin/admin";+import {+ TabsLine,+ TabsLineContent,+ TabsLineList,+ TabsLineTrigger,+} from "@/components/molecules/TabsLine/TabsLine";++interface ScheduleDetail {+ schedule_id: string;+ schedule_name: string;+ graph_id: string;+ graph_name: string;+ graph_version: number;+ user_id: string;+ user_email: string | null;+ cron: string;+ timezone: string;+ next_run_time: string;+}++interface OrphanedScheduleDetail {+ schedule_id: string;+ schedule_name: string;+ graph_id: string;+ graph_name?: string;+ graph_version: number;+ user_id: string;+ user_email?: string | null;+ cron?: string;+ timezone?: string;+ orphan_reason: string;+ error_detail: string | null;+ next_run_time: string;+}++interface CleanupResponseData {+ success: boolean;+ message: string;+ deleted_count?: number;+}++interface SchedulesTableProps {+ onRefresh?: () => void;+ diagnosticsData?: {+ total_orphaned: number;+ user_schedules: number;+ };+}++export function SchedulesTable({+ onRefresh,+ diagnosticsData,+}: SchedulesTableProps) {+ const [activeTab, setActiveTab] = useState<"all" | "orphaned">("all");+ const [selectedIds, setSelectedIds] = useState<Set<string>>(new Set());+ const [showDeleteDialog, setShowDeleteDialog] = useState(false);+ const [currentPage, setCurrentPage] = useState(1);+ const [pageSize] = useState(10);++ // Fetch data based on active tab+ const allSchedulesQuery = useGetV2ListAllUserSchedules(+ {+ limit: pageSize,+ offset: (currentPage - 1) * pageSize,+ },+ { query: { enabled: activeTab === "all" } },+ );++ const orphanedSchedulesQuery = useGetV2ListOrphanedSchedules({+ query: { enabled: activeTab === "orphaned" },+ });++ const activeQuery =+ activeTab === "orphaned" ? orphanedSchedulesQuery : allSchedulesQuery;++ const {+ data: schedulesResponse,+ isLoading,+ error: _error,+ refetch,+ } = activeQuery;++ const schedulesData = schedulesResponse?.data as+ | { schedules: (ScheduleDetail | OrphanedScheduleDetail)[]; total: number }+ | undefined;+ const schedules = schedulesData?.schedules || [];+ const total = schedulesData?.total || 0;++ // Cleanup mutation+ const { mutateAsync: cleanupOrphanedSchedules, isPending: isDeleting } =+ usePostV2CleanupOrphanedSchedules();++ const handleSelectAll = (checked: boolean) => {+ if (checked) {+ setSelectedIds(+ new Set(+ schedules.map(+ (s: ScheduleDetail | OrphanedScheduleDetail) => s.schedule_id,+ ),+ ),+ );+ } else {+ setSelectedIds(new Set());+ }+ };++ const handleSelectSchedule = (id: string, checked: boolean) => {+ const newSelected = new Set(selectedIds);+ if (checked) {+ newSelected.add(id);+ } else {+ newSelected.delete(id);+ }+ setSelectedIds(newSelected);+ };++ const confirmDelete = () => {+ setShowDeleteDialog(true);+ };++ const handleDelete = async () => {+ setShowDeleteDialog(false);++ try {+ const idsToDelete =+ activeTab === "orphaned" && selectedIds.size === 0+ ? schedules.map(+ (s: ScheduleDetail | OrphanedScheduleDetail) => s.schedule_id,+ )+ : Array.from(selectedIds);++ const result = await cleanupOrphanedSchedules({+ data: { schedule_ids: idsToDelete },+ });++ toast({+ title: "Success",+ description:+ (result.data as CleanupResponseData)?.message ||+ `Deleted ${(result.data as CleanupResponseData)?.deleted_count || 0} schedule(s)`,+ });++ setSelectedIds(new Set());+ await refetch();+ if (onRefresh) onRefresh();+ } catch (err: unknown) {+ console.error("Error deleting schedules:", err);+ toast({+ title: "Error",+ description:+ err instanceof Error ? err.message : "Failed to delete schedules",+ variant: "destructive",+ });+ }+ };++ const totalPages = Math.ceil(total / pageSize);++ return (+ <>+ <Card>+ <TabsLine+ value={activeTab}+ onValueChange={(v) => setActiveTab(v as "all" | "orphaned")}+ >+ <CardHeader>+ <div className="flex items-center justify-between">+ <CardTitle>Schedules</CardTitle>+ <div className="flex gap-2">+ {activeTab === "orphaned" && schedules.length > 0 && (+ <Button+ variant="destructive"+ size="small"+ onClick={confirmDelete}+ disabled={isDeleting}+ >+ <Trash className="mr-2 h-4 w-4" />+ Delete All Orphaned ({total})+ </Button>+ )}+ {selectedIds.size > 0 && (+ <Button+ variant="destructive"+ size="small"+ onClick={confirmDelete}+ disabled={isDeleting}+ >+ <Trash className="mr-2 h-4 w-4" />+ Delete Selected ({selectedIds.size})+ </Button>+ )}+ <Button+ variant="outline"+ size="small"+ onClick={() => {+ refetch();+ if (onRefresh) onRefresh();+ }}+ disabled={isLoading}+ >+ <ArrowClockwise+ className={`h-4 w-4 ${isLoading ? "animate-spin" : ""}`}+ />+ </Button>+ </div>+ </div>++ <TabsLineList className="px-6">+ <TabsLineTrigger value="all">+ All Schedules+ {diagnosticsData && ` (${diagnosticsData.user_schedules})`}+ </TabsLineTrigger>+ <TabsLineTrigger value="orphaned">+ Orphaned+ {diagnosticsData && ` (${diagnosticsData.total_orphaned})`}+ </TabsLineTrigger>+ </TabsLineList>+ </CardHeader>++ <TabsLineContent value={activeTab}>+ <CardContent>+ {isLoading && schedules.length === 0 ? (+ <div className="flex h-32 items-center justify-center">+ <ArrowClockwise className="h-6 w-6 animate-spin text-gray-400" />+ </div>+ ) : schedules.length === 0 ? (+ <div className="py-8 text-center text-gray-500">+ No schedules found+ </div>+ ) : (+ <Table>+ <TableHeader>+ <TableRow>+ <TableHead className="w-12">+ <Checkbox+ checked={+ selectedIds.size === schedules.length &&+ schedules.length > 0+ }+ onCheckedChange={handleSelectAll}+ />+ </TableHead>+ <TableHead>Name</TableHead>+ <TableHead>Graph</TableHead>+ <TableHead>User</TableHead>+ <TableHead>Cron</TableHead>+ <TableHead>Next Run</TableHead>+ {activeTab === "orphaned" && (+ <TableHead>Orphan Reason</TableHead>+ )}+ </TableRow>+ </TableHeader>+ <TableBody>+ {schedules.map(+ (schedule: ScheduleDetail | OrphanedScheduleDetail) => {+ const isOrphaned = activeTab === "orphaned";+ return (+ <TableRow+ key={schedule.schedule_id}+ className={isOrphaned ? "bg-purple-50" : ""}+ >+ <TableCell>+ <Checkbox+ checked={selectedIds.has(schedule.schedule_id)}+ onCheckedChange={(checked) =>+ handleSelectSchedule(+ schedule.schedule_id,+ checked as boolean,+ )+ }+ />+ </TableCell>+ <TableCell>{schedule.schedule_name}</TableCell>+ <TableCell>+ <div>{schedule.graph_name || "Unknown"}</div>+ <div className="font-mono text-xs text-gray-500">+ v{schedule.graph_version}+ </div>+ </TableCell>+ <TableCell>+ <div>+ {(schedule as ScheduleDetail).user_email || (+ <span className="text-gray-400">Unknown</span>+ )}+ </div>+ <div+ className="group flex cursor-pointer items-center gap-1 font-mono text-xs text-gray-500 hover:text-gray-700"+ onClick={() => {+ navigator.clipboard.writeText(+ schedule.user_id,+ );+ toast({+ title: "Copied",+ description: "User ID copied to clipboard",+ });+ }}+ title="Click to copy user ID"+ >+ {schedule.user_id.substring(0, 8)}...+ <Copy className="h-3 w-3 opacity-0 transition-opacity group-hover:opacity-100" />+ </div>+ </TableCell>+ <TableCell>+ {schedule.cron ? (+ <>+ <code className="rounded bg-gray-100 px-2 py-1 text-xs">+ {schedule.cron}+ </code>+ <div className="text-xs text-gray-500">+ {schedule.timezone}+ </div>+ </>+ ) : (+ <span className="text-gray-400">N/A</span>+ )}+ </TableCell>+ <TableCell>+ {schedule.next_run_time+ ? new Date(+ schedule.next_run_time,+ ).toLocaleString()+ : "Not scheduled"}+ </TableCell>+ {activeTab === "orphaned" && (+ <TableCell>+ <span className="text-xs text-purple-600">+ {(+ schedule as OrphanedScheduleDetail+ ).orphan_reason?.replace(/_/g, " ") ||+ "unknown"}+ </span>+ </TableCell>+ )}+ </TableRow>+ );+ },+ )}+ </TableBody>+ </Table>+ )}++ {totalPages > 1 && activeTab === "all" && (+ <div className="mt-4 flex items-center justify-between">+ <div className="text-sm text-gray-600">+ Showing {(currentPage - 1) * pageSize + 1} to{" "}+ {Math.min(currentPage * pageSize, total)} of {total}{" "}+ schedules+ </div>+ <div className="flex gap-2">+ <Button+ variant="outline"+ size="small"+ onClick={() => setCurrentPage(currentPage - 1)}+ disabled={currentPage === 1}+ >+ Previous+ </Button>+ <div className="flex items-center px-3">+ Page {currentPage} of {totalPages}+ </div>+ <Button+ variant="outline"+ size="small"+ onClick={() => setCurrentPage(currentPage + 1)}+ disabled={currentPage === totalPages}+ >+ Next+ </Button>+ </div>+ </div>+ )}+ </CardContent>+ </TabsLineContent>+ </TabsLine>+ </Card>++ <Dialog open={showDeleteDialog} onOpenChange={setShowDeleteDialog}>+ <DialogContent>+ <DialogHeader>+ <DialogTitle>Confirm Delete Schedules</DialogTitle>+ <DialogDescription>+ {activeTab === "orphaned" && selectedIds.size === 0 ? (+ <>+ Are you sure you want to delete ALL {total} orphaned+ schedules?+ <br />+ <br />+ These schedules reference deleted graphs or graphs the user no+ longer has access to. Deleting them is safe.+ </>+ ) : (+ <>+ Are you sure you want to delete {selectedIds.size} selected+ schedule(s)?+ <br />+ <br />+ This will permanently remove the schedules from the system.+ </>+ )}+ </DialogDescription>+ </DialogHeader>+ <DialogFooter>+ <Button+ variant="outline"+ onClick={() => setShowDeleteDialog(false)}+ >+ Cancel+ </Button>+ <Button+ variant="destructive"+ onClick={handleDelete}+ className="bg-red-600 hover:bg-red-700"+ >+ Delete Schedules+ </Button>+ </DialogFooter>+ </DialogContent>+ </Dialog>+ </>+ );+}diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/components/useDiagnosticsContent.ts b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/components/useDiagnosticsContent.tsnew file mode 100644index 0000000000..e2d5dbab85--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/components/useDiagnosticsContent.ts@@ -0,0 +1,63 @@+import {+ useGetV2GetExecutionDiagnostics,+ useGetV2GetAgentDiagnostics,+ useGetV2GetScheduleDiagnostics,+} from "@/app/api/__generated__/endpoints/admin/admin";+import type { ExecutionDiagnosticsResponse } from "@/app/api/__generated__/models/executionDiagnosticsResponse";+import type { AgentDiagnosticsResponse } from "@/app/api/__generated__/models/agentDiagnosticsResponse";+import type { ScheduleHealthMetrics } from "@/app/api/__generated__/models/scheduleHealthMetrics";++export function useDiagnosticsContent() {+ const {+ data: executionResponse,+ isLoading: isLoadingExecutions,+ isError: isExecutionError,+ error: executionError,+ refetch: refetchExecutions,+ } = useGetV2GetExecutionDiagnostics();++ const {+ data: agentResponse,+ isLoading: isLoadingAgents,+ isError: isAgentError,+ error: agentError,+ refetch: refetchAgents,+ } = useGetV2GetAgentDiagnostics();++ const {+ data: scheduleResponse,+ isLoading: isLoadingSchedules,+ isError: isScheduleError,+ error: scheduleError,+ refetch: refetchSchedules,+ } = useGetV2GetScheduleDiagnostics();++ const isLoading =+ isLoadingExecutions || isLoadingAgents || isLoadingSchedules;+ const isError = isExecutionError || isAgentError || isScheduleError;+ const error = executionError || agentError || scheduleError;++ const executionData = executionResponse?.data as+ | ExecutionDiagnosticsResponse+ | undefined;+ const agentData = agentResponse?.data as AgentDiagnosticsResponse | undefined;+ const scheduleData = scheduleResponse?.data as+ | ScheduleHealthMetrics+ | undefined;++ const refresh = () => {+ refetchExecutions();+ refetchAgents();+ refetchSchedules();+ };++ return {+ executionData,+ agentData,+ scheduleData,+ isLoading,+ isError,+ error,+ refresh,+ };+}diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/page.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/page.tsxnew file mode 100644index 0000000000..cbbf0065b0--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/admin/diagnostics/page.tsx@@ -0,0 +1,17 @@+import { withRoleAccess } from "@/lib/withRoleAccess";+import { DiagnosticsContent } from "./components/DiagnosticsContent";++function AdminDiagnostics() {+ return (+ <div className="mx-auto p-6">+ <DiagnosticsContent />+ </div>+ );+}++export default async function AdminDiagnosticsPage() {+ "use server";+ const withAdminAccess = await withRoleAccess(["admin"]);+ const ProtectedAdminDiagnostics = await withAdminAccess(AdminDiagnostics);+ return <ProtectedAdminDiagnostics />;+}diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/layout.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/layout.tsxindex c7483d55cd..13dd942b52 100644--- a/autogpt_platform/frontend/src/app/(platform)/admin/layout.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/admin/layout.tsx@@ -6,6 +6,7 @@ import {Gauge,Receipt,FileText,+ Heartbeat,} from "@phosphor-icons/react/dist/ssr";import { IconSliders } from "@/components/__legacy__/ui/icons";@@ -23,6 +24,11 @@ const sidebarLinkGroups = [href: "/admin/spending",icon: <CurrencyDollar className="h-6 w-6" />,},+ {+ text: "System Diagnostics",+ href: "/admin/diagnostics",+ icon: <Heartbeat className="h-6 w-6" />,+ },{text: "User Impersonation",href: "/admin/impersonation",diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.jsonindex 9103d6f475..87fc8ccace 100644--- a/autogpt_platform/frontend/src/app/api/openapi.json+++ b/autogpt_platform/frontend/src/app/api/openapi.json@@ -7,6 +7,768 @@"version": "0.1"},"paths": {+ "/api/admin/diagnostics/agents": {+ "get": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Get Agent Diagnostics",+ "description": "Get diagnostic information about agents.\n\nReturns:\n - agents_with_active_executions: Number of unique agents with running/queued executions\n - timestamp: Current timestamp",+ "operationId": "getV2Get agent diagnostics",+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/AgentDiagnosticsResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/executions": {+ "get": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Get Execution Diagnostics",+ "description": "Get comprehensive diagnostic information about execution status.\n\nReturns all execution metrics including:\n- Current state (running, queued)\n- Orphaned executions (>24h old, likely not in executor)\n- Failure metrics (1h, 24h, rate)\n- Long-running detection (stuck >1h, >24h)\n- Stuck queued detection\n- Throughput metrics (completions/hour)\n- RabbitMQ queue depths",+ "operationId": "getV2Get execution diagnostics",+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ExecutionDiagnosticsResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/executions/cleanup-all-orphaned": {+ "post": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Cleanup ALL Orphaned Executions",+ "description": "Cleanup ALL orphaned executions (>24h old) by directly updating DB status.\nOperates on all executions, not just paginated results.\n\nReturns:\n Number of executions cleaned up and success message",+ "operationId": "postV2Cleanup all orphaned executions",+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/StopExecutionResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/executions/cleanup-all-stuck-queued": {+ "post": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Cleanup ALL Stuck Queued Executions",+ "description": "Cleanup ALL stuck queued executions (QUEUED >1h) by updating DB status (admin only).\nOperates on entire dataset, not limited to pagination.\n\nReturns:\n Number of executions cleaned up and success message",+ "operationId": "postV2Cleanup all stuck queued executions",+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/StopExecutionResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/executions/cleanup-orphaned": {+ "post": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Cleanup Orphaned Executions",+ "description": "Cleanup orphaned executions by directly updating DB status (admin only).\nFor executions in DB but not actually running in executor (old/stale records).\n\nArgs:\n request: Contains list of execution_ids to cleanup\n\nReturns:\n Number of executions cleaned up and success message",+ "operationId": "postV2Cleanup orphaned executions",+ "requestBody": {+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/StopExecutionsRequest" }+ }+ },+ "required": true+ },+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/StopExecutionResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/executions/failed": {+ "get": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "List Failed Executions",+ "description": "Get detailed list of failed executions.\n\nArgs:\n limit: Maximum number of executions to return (default 100)\n offset: Number of executions to skip (default 0)\n hours: Number of hours to look back (default 24)\n\nReturns:\n List of failed executions with error details",+ "operationId": "getV2List failed executions",+ "security": [{ "HTTPBearerJWT": [] }],+ "parameters": [+ {+ "name": "limit",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 100, "title": "Limit" }+ },+ {+ "name": "offset",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 0, "title": "Offset" }+ },+ {+ "name": "hours",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 24, "title": "Hours" }+ }+ ],+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/FailedExecutionsListResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ }+ }+ },+ "/api/admin/diagnostics/executions/invalid": {+ "get": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "List Invalid Executions",+ "description": "Get detailed list of executions in invalid states (READ-ONLY).\n\nInvalid states indicate data corruption and require manual investigation:\n- QUEUED but has startedAt (impossible - can't start while queued)\n- RUNNING but no startedAt (impossible - can't run without starting)\n\n⚠️ NO BULK ACTIONS PROVIDED - These need case-by-case investigation.\n\nEach invalid execution likely has a different root cause (crashes, race conditions,\nDB corruption). Investigate the execution history and logs to determine appropriate\naction (manual cleanup, status fix, or leave as-is if system recovered).\n\nArgs:\n limit: Maximum number of executions to return (default 100)\n offset: Number of executions to skip (default 0)\n\nReturns:\n List of invalid state executions with details",+ "operationId": "getV2List invalid executions",+ "security": [{ "HTTPBearerJWT": [] }],+ "parameters": [+ {+ "name": "limit",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 100, "title": "Limit" }+ },+ {+ "name": "offset",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 0, "title": "Offset" }+ }+ ],+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/RunningExecutionsListResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ }+ }+ },+ "/api/admin/diagnostics/executions/long-running": {+ "get": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "List Long-Running Executions",+ "description": "Get detailed list of long-running executions (RUNNING status >24h).\n\nArgs:\n limit: Maximum number of executions to return (default 100)\n offset: Number of executions to skip (default 0)\n\nReturns:\n List of long-running executions with details",+ "operationId": "getV2List long-running executions",+ "security": [{ "HTTPBearerJWT": [] }],+ "parameters": [+ {+ "name": "limit",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 100, "title": "Limit" }+ },+ {+ "name": "offset",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 0, "title": "Offset" }+ }+ ],+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/RunningExecutionsListResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ }+ }+ },+ "/api/admin/diagnostics/executions/orphaned": {+ "get": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "List Orphaned Executions",+ "description": "Get detailed list of orphaned executions (>24h old, likely not in executor).\n\nArgs:\n limit: Maximum number of executions to return (default 100)\n offset: Number of executions to skip (default 0)\n\nReturns:\n List of orphaned executions with details",+ "operationId": "getV2List orphaned executions",+ "security": [{ "HTTPBearerJWT": [] }],+ "parameters": [+ {+ "name": "limit",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 100, "title": "Limit" }+ },+ {+ "name": "offset",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 0, "title": "Offset" }+ }+ ],+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/RunningExecutionsListResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ }+ }+ },+ "/api/admin/diagnostics/executions/requeue": {+ "post": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Requeue Stuck Execution",+ "description": "Requeue a stuck QUEUED execution (admin only).\n\nUses add_graph_execution with existing graph_exec_id to requeue.\n\n⚠️ WARNING: Only use for stuck executions. This will re-execute and may cost credits.\n\nArgs:\n request: Contains execution_id to requeue\n\nReturns:\n Success status and message",+ "operationId": "postV2Requeue stuck execution",+ "requestBody": {+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/StopExecutionRequest" }+ }+ },+ "required": true+ },+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/RequeueExecutionResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/executions/requeue-all-stuck": {+ "post": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Requeue ALL Stuck Queued Executions",+ "description": "Requeue ALL stuck queued executions (QUEUED >1h) by publishing to RabbitMQ.\nOperates on all executions, not just paginated results.\n\nUses add_graph_execution with existing graph_exec_id to requeue.\n\n⚠️ WARNING: This will re-execute ALL stuck executions and may cost significant credits.\n\nReturns:\n Number of executions requeued and success message",+ "operationId": "postV2Requeue all stuck queued executions",+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/RequeueExecutionResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/executions/requeue-bulk": {+ "post": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Requeue Multiple Stuck Executions",+ "description": "Requeue multiple stuck QUEUED executions (admin only).\n\nUses add_graph_execution with existing graph_exec_id to requeue.\n\n⚠️ WARNING: Only use for stuck executions. This will re-execute and may cost credits.\n\nArgs:\n request: Contains list of execution_ids to requeue\n\nReturns:\n Number of executions requeued and success message",+ "operationId": "postV2Requeue multiple stuck executions",+ "requestBody": {+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/StopExecutionsRequest" }+ }+ },+ "required": true+ },+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/RequeueExecutionResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/executions/running": {+ "get": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "List Running Executions",+ "description": "Get detailed list of running and queued executions (recent, likely active).\n\nArgs:\n limit: Maximum number of executions to return (default 100)\n offset: Number of executions to skip (default 0)\n\nReturns:\n List of running executions with details",+ "operationId": "getV2List running executions",+ "security": [{ "HTTPBearerJWT": [] }],+ "parameters": [+ {+ "name": "limit",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 100, "title": "Limit" }+ },+ {+ "name": "offset",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 0, "title": "Offset" }+ }+ ],+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/RunningExecutionsListResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ }+ }+ },+ "/api/admin/diagnostics/executions/stop": {+ "post": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Stop Single Execution",+ "description": "Stop a single execution (admin only).\n\nUses robust stop_graph_execution which cascades to children and waits for termination.\n\nArgs:\n request: Contains execution_id to stop\n\nReturns:\n Success status and message",+ "operationId": "postV2Stop single execution",+ "requestBody": {+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/StopExecutionRequest" }+ }+ },+ "required": true+ },+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/StopExecutionResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/executions/stop-all-long-running": {+ "post": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Stop ALL Long-Running Executions",+ "description": "Stop ALL long-running executions (RUNNING >24h) by sending cancel signals (admin only).\nOperates on entire dataset, not limited to pagination.\n\nReturns:\n Number of executions stopped and success message",+ "operationId": "postV2Stop all long-running executions",+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/StopExecutionResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/executions/stop-bulk": {+ "post": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Stop Multiple Executions",+ "description": "Stop multiple active executions (admin only).\n\nUses robust stop_graph_execution which cascades to children and waits for termination.\n\nArgs:\n request: Contains list of execution_ids to stop\n\nReturns:\n Number of executions stopped and success message",+ "operationId": "postV2Stop multiple executions",+ "requestBody": {+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/StopExecutionsRequest" }+ }+ },+ "required": true+ },+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/StopExecutionResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/executions/stuck-queued": {+ "get": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "List Stuck Queued Executions",+ "description": "Get detailed list of stuck queued executions (QUEUED >1h, never started).\n\nArgs:\n limit: Maximum number of executions to return (default 100)\n offset: Number of executions to skip (default 0)\n\nReturns:\n List of stuck queued executions with details",+ "operationId": "getV2List stuck queued executions",+ "security": [{ "HTTPBearerJWT": [] }],+ "parameters": [+ {+ "name": "limit",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 100, "title": "Limit" }+ },+ {+ "name": "offset",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 0, "title": "Offset" }+ }+ ],+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/RunningExecutionsListResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ }+ }+ },+ "/api/admin/diagnostics/schedules": {+ "get": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Get Schedule Diagnostics",+ "description": "Get comprehensive diagnostic information about schedule health.\n\nReturns schedule metrics including:\n- Total schedules (user vs system)\n- Orphaned schedules by category\n- Upcoming executions",+ "operationId": "getV2Get schedule diagnostics",+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ScheduleHealthMetrics"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/schedules/all": {+ "get": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "List All User Schedules",+ "description": "Get detailed list of all user schedules (excludes system monitoring jobs).\n\nArgs:\n limit: Maximum number of schedules to return (default 100)\n offset: Number of schedules to skip (default 0)\n\nReturns:\n List of schedules with details",+ "operationId": "getV2List all user schedules",+ "security": [{ "HTTPBearerJWT": [] }],+ "parameters": [+ {+ "name": "limit",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 100, "title": "Limit" }+ },+ {+ "name": "offset",+ "in": "query",+ "required": false,+ "schema": { "type": "integer", "default": 0, "title": "Offset" }+ }+ ],+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/SchedulesListResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ }+ }+ },+ "/api/admin/diagnostics/schedules/cleanup-orphaned": {+ "post": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "Cleanup Orphaned Schedules",+ "description": "Cleanup orphaned schedules by deleting from scheduler (admin only).\n\nArgs:\n request: Contains list of schedule_ids to delete\n\nReturns:\n Number of schedules deleted and success message",+ "operationId": "postV2Cleanup orphaned schedules",+ "requestBody": {+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ScheduleCleanupRequest"+ }+ }+ },+ "required": true+ },+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ScheduleCleanupResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },+ "/api/admin/diagnostics/schedules/orphaned": {+ "get": {+ "tags": ["v2", "admin", "diagnostics", "admin"],+ "summary": "List Orphaned Schedules",+ "description": "Get detailed list of orphaned schedules with orphan reasons.\n\nReturns:\n List of orphaned schedules categorized by orphan type",+ "operationId": "getV2List orphaned schedules",+ "responses": {+ "200": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/OrphanedSchedulesListResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ }+ },+ "security": [{ "HTTPBearerJWT": [] }]+ }+ },"/api/admin/platform-costs/dashboard": {"get": {"tags": ["v2", "admin", "platform-cost", "admin"],@@ -8120,6 +8882,19 @@"title": "AgentDetailsResponse","description": "Response for get_details action."},+ "AgentDiagnosticsResponse": {+ "properties": {+ "agents_with_active_executions": {+ "type": "integer",+ "title": "Agents With Active Executions"+ },+ "timestamp": { "type": "string", "title": "Timestamp" }+ },+ "type": "object",+ "required": ["agents_with_active_executions", "timestamp"],+ "title": "AgentDiagnosticsResponse",+ "description": "Response model for agent diagnostics"+ },"AgentExecutionStatus": {"type": "string","enum": [@@ -9915,6 +10690,94 @@],"title": "ExecutionAnalyticsResult"},+ "ExecutionDiagnosticsResponse": {+ "properties": {+ "running_executions": {+ "type": "integer",+ "title": "Running Executions"+ },+ "queued_executions_db": {+ "type": "integer",+ "title": "Queued Executions Db"+ },+ "queued_executions_rabbitmq": {+ "type": "integer",+ "title": "Queued Executions Rabbitmq"+ },+ "cancel_queue_depth": {+ "type": "integer",+ "title": "Cancel Queue Depth"+ },+ "orphaned_running": {+ "type": "integer",+ "title": "Orphaned Running"+ },+ "orphaned_queued": { "type": "integer", "title": "Orphaned Queued" },+ "failed_count_1h": { "type": "integer", "title": "Failed Count 1H" },+ "failed_count_24h": {+ "type": "integer",+ "title": "Failed Count 24H"+ },+ "failure_rate_24h": { "type": "number", "title": "Failure Rate 24H" },+ "stuck_running_24h": {+ "type": "integer",+ "title": "Stuck Running 24H"+ },+ "stuck_running_1h": {+ "type": "integer",+ "title": "Stuck Running 1H"+ },+ "oldest_running_hours": {+ "anyOf": [{ "type": "number" }, { "type": "null" }],+ "title": "Oldest Running Hours"+ },+ "stuck_queued_1h": { "type": "integer", "title": "Stuck Queued 1H" },+ "queued_never_started": {+ "type": "integer",+ "title": "Queued Never Started"+ },+ "invalid_queued_with_start": {+ "type": "integer",+ "title": "Invalid Queued With Start"+ },+ "invalid_running_without_start": {+ "type": "integer",+ "title": "Invalid Running Without Start"+ },+ "completed_1h": { "type": "integer", "title": "Completed 1H" },+ "completed_24h": { "type": "integer", "title": "Completed 24H" },+ "throughput_per_hour": {+ "type": "number",+ "title": "Throughput Per Hour"+ },+ "timestamp": { "type": "string", "title": "Timestamp" }+ },+ "type": "object",+ "required": [+ "running_executions",+ "queued_executions_db",+ "queued_executions_rabbitmq",+ "cancel_queue_depth",+ "orphaned_running",+ "orphaned_queued",+ "failed_count_1h",+ "failed_count_24h",+ "failure_rate_24h",+ "stuck_running_24h",+ "stuck_running_1h",+ "oldest_running_hours",+ "stuck_queued_1h",+ "queued_never_started",+ "invalid_queued_with_start",+ "invalid_running_without_start",+ "completed_1h",+ "completed_24h",+ "throughput_per_hour",+ "timestamp"+ ],+ "title": "ExecutionDiagnosticsResponse",+ "description": "Response model for execution diagnostics"+ },"ExecutionOptions": {"properties": {"manual": { "type": "boolean", "title": "Manual", "default": true },@@ -10004,6 +10867,73 @@"title": "ExecutionStartedResponse","description": "Response for run/schedule actions."},+ "FailedExecutionDetail": {+ "properties": {+ "execution_id": { "type": "string", "title": "Execution Id" },+ "graph_id": { "type": "string", "title": "Graph Id" },+ "graph_name": { "type": "string", "title": "Graph Name" },+ "graph_version": { "type": "integer", "title": "Graph Version" },+ "user_id": { "type": "string", "title": "User Id" },+ "user_email": {+ "anyOf": [{ "type": "string" }, { "type": "null" }],+ "title": "User Email"+ },+ "status": { "type": "string", "title": "Status" },+ "created_at": {+ "type": "string",+ "format": "date-time",+ "title": "Created At"+ },+ "started_at": {+ "anyOf": [+ { "type": "string", "format": "date-time" },+ { "type": "null" }+ ],+ "title": "Started At"+ },+ "failed_at": {+ "anyOf": [+ { "type": "string", "format": "date-time" },+ { "type": "null" }+ ],+ "title": "Failed At"+ },+ "error_message": {+ "anyOf": [{ "type": "string" }, { "type": "null" }],+ "title": "Error Message"+ }+ },+ "type": "object",+ "required": [+ "execution_id",+ "graph_id",+ "graph_name",+ "graph_version",+ "user_id",+ "user_email",+ "status",+ "created_at",+ "started_at",+ "failed_at",+ "error_message"+ ],+ "title": "FailedExecutionDetail",+ "description": "Details about a failed execution for admin view"+ },+ "FailedExecutionsListResponse": {+ "properties": {+ "executions": {+ "items": { "$ref": "#/components/schemas/FailedExecutionDetail" },+ "type": "array",+ "title": "Executions"+ },+ "total": { "type": "integer", "title": "Total" }+ },+ "type": "object",+ "required": ["executions", "total"],+ "title": "FailedExecutionsListResponse",+ "description": "Response model for list of failed executions"+ },"FolderCreateRequest": {"properties": {"name": {@@ -12226,6 +13156,48 @@],"title": "OnboardingStep"},+ "OrphanedScheduleDetail": {+ "properties": {+ "schedule_id": { "type": "string", "title": "Schedule Id" },+ "schedule_name": { "type": "string", "title": "Schedule Name" },+ "graph_id": { "type": "string", "title": "Graph Id" },+ "graph_version": { "type": "integer", "title": "Graph Version" },+ "user_id": { "type": "string", "title": "User Id" },+ "orphan_reason": { "type": "string", "title": "Orphan Reason" },+ "error_detail": {+ "anyOf": [{ "type": "string" }, { "type": "null" }],+ "title": "Error Detail"+ },+ "next_run_time": { "type": "string", "title": "Next Run Time" }+ },+ "type": "object",+ "required": [+ "schedule_id",+ "schedule_name",+ "graph_id",+ "graph_version",+ "user_id",+ "orphan_reason",+ "error_detail",+ "next_run_time"+ ],+ "title": "OrphanedScheduleDetail",+ "description": "Details about an orphaned schedule"+ },+ "OrphanedSchedulesListResponse": {+ "properties": {+ "schedules": {+ "items": { "$ref": "#/components/schemas/OrphanedScheduleDetail" },+ "type": "array",+ "title": "Schedules"+ },+ "total": { "type": "integer", "title": "Total" }+ },+ "type": "object",+ "required": ["schedules", "total"],+ "title": "OrphanedSchedulesListResponse",+ "description": "Response model for list of orphaned schedules"+ },"Pagination": {"properties": {"total_items": {@@ -13083,6 +14055,21 @@"required": ["credit_amount"],"title": "RequestTopUp"},+ "RequeueExecutionResponse": {+ "properties": {+ "success": { "type": "boolean", "title": "Success" },+ "requeued_count": {+ "type": "integer",+ "title": "Requeued Count",+ "default": 0+ },+ "message": { "type": "string", "title": "Message" }+ },+ "type": "object",+ "required": ["success", "message"],+ "title": "RequeueExecutionResponse",+ "description": "Response model for requeue execution operations"+ },"ResponseType": {"type": "string","enum": [@@ -13247,6 +14234,92 @@"required": ["store_listing_version_id", "is_approved", "comments"],"title": "ReviewSubmissionRequest"},+ "RunningExecutionDetail": {+ "properties": {+ "execution_id": { "type": "string", "title": "Execution Id" },+ "graph_id": { "type": "string", "title": "Graph Id" },+ "graph_name": { "type": "string", "title": "Graph Name" },+ "graph_version": { "type": "integer", "title": "Graph Version" },+ "user_id": { "type": "string", "title": "User Id" },+ "user_email": {+ "anyOf": [{ "type": "string" }, { "type": "null" }],+ "title": "User Email"+ },+ "status": { "type": "string", "title": "Status" },+ "created_at": {+ "type": "string",+ "format": "date-time",+ "title": "Created At"+ },+ "started_at": {+ "anyOf": [+ { "type": "string", "format": "date-time" },+ { "type": "null" }+ ],+ "title": "Started At"+ },+ "queue_status": {+ "anyOf": [{ "type": "string" }, { "type": "null" }],+ "title": "Queue Status"+ }+ },+ "type": "object",+ "required": [+ "execution_id",+ "graph_id",+ "graph_name",+ "graph_version",+ "user_id",+ "user_email",+ "status",+ "created_at",+ "started_at"+ ],+ "title": "RunningExecutionDetail",+ "description": "Details about a running execution for admin view"+ },+ "RunningExecutionsListResponse": {+ "properties": {+ "executions": {+ "items": { "$ref": "#/components/schemas/RunningExecutionDetail" },+ "type": "array",+ "title": "Executions"+ },+ "total": { "type": "integer", "title": "Total" }+ },+ "type": "object",+ "required": ["executions", "total"],+ "title": "RunningExecutionsListResponse",+ "description": "Response model for list of running executions"+ },+ "ScheduleCleanupRequest": {+ "properties": {+ "schedule_ids": {+ "items": { "type": "string" },+ "type": "array",+ "title": "Schedule Ids"+ }+ },+ "type": "object",+ "required": ["schedule_ids"],+ "title": "ScheduleCleanupRequest",+ "description": "Request model for cleaning up schedules"+ },+ "ScheduleCleanupResponse": {+ "properties": {+ "success": { "type": "boolean", "title": "Success" },+ "deleted_count": {+ "type": "integer",+ "title": "Deleted Count",+ "default": 0+ },+ "message": { "type": "string", "title": "Message" }+ },+ "type": "object",+ "required": ["success", "message"],+ "title": "ScheduleCleanupResponse",+ "description": "Response model for schedule cleanup operations"+ },"ScheduleCreationRequest": {"properties": {"graph_version": {@@ -13277,6 +14350,121 @@"required": ["name", "cron", "inputs"],"title": "ScheduleCreationRequest"},+ "ScheduleDetail": {+ "properties": {+ "schedule_id": { "type": "string", "title": "Schedule Id" },+ "schedule_name": { "type": "string", "title": "Schedule Name" },+ "graph_id": { "type": "string", "title": "Graph Id" },+ "graph_name": { "type": "string", "title": "Graph Name" },+ "graph_version": { "type": "integer", "title": "Graph Version" },+ "user_id": { "type": "string", "title": "User Id" },+ "user_email": {+ "anyOf": [{ "type": "string" }, { "type": "null" }],+ "title": "User Email"+ },+ "cron": { "type": "string", "title": "Cron" },+ "timezone": { "type": "string", "title": "Timezone" },+ "next_run_time": { "type": "string", "title": "Next Run Time" },+ "created_at": {+ "anyOf": [+ { "type": "string", "format": "date-time" },+ { "type": "null" }+ ],+ "title": "Created At"+ }+ },+ "type": "object",+ "required": [+ "schedule_id",+ "schedule_name",+ "graph_id",+ "graph_name",+ "graph_version",+ "user_id",+ "user_email",+ "cron",+ "timezone",+ "next_run_time"+ ],+ "title": "ScheduleDetail",+ "description": "Details about a schedule for admin view"+ },+ "ScheduleHealthMetrics": {+ "properties": {+ "total_schedules": { "type": "integer", "title": "Total Schedules" },+ "user_schedules": { "type": "integer", "title": "User Schedules" },+ "system_schedules": {+ "type": "integer",+ "title": "System Schedules"+ },+ "orphaned_deleted_graph": {+ "type": "integer",+ "title": "Orphaned Deleted Graph"+ },+ "orphaned_no_library_access": {+ "type": "integer",+ "title": "Orphaned No Library Access"+ },+ "orphaned_invalid_credentials": {+ "type": "integer",+ "title": "Orphaned Invalid Credentials"+ },+ "orphaned_validation_failed": {+ "type": "integer",+ "title": "Orphaned Validation Failed"+ },+ "total_orphaned": { "type": "integer", "title": "Total Orphaned" },+ "schedules_next_hour": {+ "type": "integer",+ "title": "Schedules Next Hour"+ },+ "schedules_next_24h": {+ "type": "integer",+ "title": "Schedules Next 24H"+ },+ "total_runs_next_hour": {+ "type": "integer",+ "title": "Total Runs Next Hour"+ },+ "total_runs_next_24h": {+ "type": "integer",+ "title": "Total Runs Next 24H"+ },+ "timestamp": { "type": "string", "title": "Timestamp" }+ },+ "type": "object",+ "required": [+ "total_schedules",+ "user_schedules",+ "system_schedules",+ "orphaned_deleted_graph",+ "orphaned_no_library_access",+ "orphaned_invalid_credentials",+ "orphaned_validation_failed",+ "total_orphaned",+ "schedules_next_hour",+ "schedules_next_24h",+ "total_runs_next_hour",+ "total_runs_next_24h",+ "timestamp"+ ],+ "title": "ScheduleHealthMetrics",+ "description": "Summary of schedule health diagnostics"+ },+ "SchedulesListResponse": {+ "properties": {+ "schedules": {+ "items": { "$ref": "#/components/schemas/ScheduleDetail" },+ "type": "array",+ "title": "Schedules"+ },+ "total": { "type": "integer", "title": "Total" }+ },+ "type": "object",+ "required": ["schedules", "total"],+ "title": "SchedulesListResponse",+ "description": "Response model for list of schedules"+ },"SearchEntry": {"properties": {"search_query": {@@ -13588,6 +14776,43 @@"type": "object","title": "Stats"},+ "StopExecutionRequest": {+ "properties": {+ "execution_id": { "type": "string", "title": "Execution Id" }+ },+ "type": "object",+ "required": ["execution_id"],+ "title": "StopExecutionRequest",+ "description": "Request model for stopping a single execution"+ },+ "StopExecutionResponse": {+ "properties": {+ "success": { "type": "boolean", "title": "Success" },+ "stopped_count": {+ "type": "integer",+ "title": "Stopped Count",+ "default": 0+ },+ "message": { "type": "string", "title": "Message" }+ },+ "type": "object",+ "required": ["success", "message"],+ "title": "StopExecutionResponse",+ "description": "Response model for stop execution operations"+ },+ "StopExecutionsRequest": {+ "properties": {+ "execution_ids": {+ "items": { "type": "string" },+ "type": "array",+ "title": "Execution Ids"+ }+ },+ "type": "object",+ "required": ["execution_ids"],+ "title": "StopExecutionsRequest",+ "description": "Request model for stopping multiple executions"+ },"StorageUsageResponse": {"properties": {"used_bytes": { "type": "integer", "title": "Used Bytes" },