AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): wire the application metrics that were defined but never recorded (#14319)
Bently125eb210fee2
diff --git a/autogpt_platform/backend/backend/api/features/credits_rate_limit.py b/autogpt_platform/backend/backend/api/features/credits_rate_limit.pyindex 63e649d72f..686ef6b8d2 100644--- a/autogpt_platform/backend/backend/api/features/credits_rate_limit.py+++ b/autogpt_platform/backend/backend/api/features/credits_rate_limit.py@@ -28,6 +28,7 @@ from autogpt_libs.auth import get_user_idfrom redis.exceptions import RedisClusterException, RedisErrorfrom backend.data.redis_client import get_redis_async+from backend.monitoring.instrumentation import record_rate_limit_hitlogger = logging.getLogger(__name__)@@ -142,6 +143,7 @@ async def enforce_subscription_status_rate_limit(user_id,count,)+ record_rate_limit_hit("/api/credits/subscription", user_id)raise fastapi.HTTPException(status_code=429,detail=(diff --git a/autogpt_platform/backend/backend/api/features/credits_rate_limit_test.py b/autogpt_platform/backend/backend/api/features/credits_rate_limit_test.pyindex 62bd6d8087..acb9814a4a 100644--- a/autogpt_platform/backend/backend/api/features/credits_rate_limit_test.py+++ b/autogpt_platform/backend/backend/api/features/credits_rate_limit_test.py@@ -199,3 +199,37 @@ async def test_per_user_keys_are_distinct(fake_redis):assert len(keys) == 2assert any("alice" in k for k in keys)assert any("bob" in k for k in keys)+++def _counter(name: str, **labels) -> float:+ from prometheus_client import REGISTRY++ return REGISTRY.get_sample_value(name, labels) or 0.0+++async def test_over_limit_records_a_rate_limit_hit(mock_redis):+ """The 429 must be visible to Prometheus; the counter had no callers."""+ before = _counter(+ "autogpt_rate_limit_hits_total", endpoint="/api/credits/subscription"+ )+ mock_redis.eval.return_value = rate_limit.SUBSCRIPTION_STATUS_MAX_REQUESTS + 1+ with pytest.raises(fastapi.HTTPException):+ await rate_limit.enforce_subscription_status_rate_limit("u1")+ assert (+ _counter("autogpt_rate_limit_hits_total", endpoint="/api/credits/subscription")+ == before + 1+ )+++async def test_under_limit_does_not_record(mock_redis):+ before = _counter(+ "autogpt_rate_limit_hits_total", endpoint="/api/credits/subscription"+ )+ mock_redis.eval.return_value = 1+ await rate_limit.enforce_subscription_status_rate_limit("u1")+ assert (+ _counter("autogpt_rate_limit_hits_total", endpoint="/api/credits/subscription")+ == before+ )diff --git a/autogpt_platform/backend/backend/api/features/search/rate_limit.py b/autogpt_platform/backend/backend/api/features/search/rate_limit.pyindex 76f9fcecc4..5807443566 100644--- a/autogpt_platform/backend/backend/api/features/search/rate_limit.py+++ b/autogpt_platform/backend/backend/api/features/search/rate_limit.py@@ -20,6 +20,7 @@ from datetime import UTC, datetimeimport fastapifrom backend.data.redis_client import get_redis_async+from backend.monitoring.instrumentation import record_rate_limit_hitlogger = logging.getLogger(__name__)@@ -67,6 +68,7 @@ async def enforce_global_search_rate_limit(user_id: str) -> None:returnif count > GLOBAL_SEARCH_MAX_REQUESTS:+ record_rate_limit_hit("/api/search", user_id)raise fastapi.HTTPException(status_code=429,detail=(diff --git a/autogpt_platform/backend/backend/api/features/search/rate_limit_test.py b/autogpt_platform/backend/backend/api/features/search/rate_limit_test.pyindex 58c98ecdd7..6720da7f94 100644--- a/autogpt_platform/backend/backend/api/features/search/rate_limit_test.py+++ b/autogpt_platform/backend/backend/api/features/search/rate_limit_test.py@@ -99,3 +99,20 @@ async def test_per_user_keys_are_distinct(fake_redis):assert key_a != key_bassert "alice" in key_aassert "bob" in key_b+++def _counter(name: str, **labels) -> float:+ from prometheus_client import REGISTRY++ return REGISTRY.get_sample_value(name, labels) or 0.0+++async def test_over_limit_records_a_rate_limit_hit(fake_redis):+ before = _counter("autogpt_rate_limit_hits_total", endpoint="/api/search")+ fake_redis.incr.return_value = rate_limit.GLOBAL_SEARCH_MAX_REQUESTS + 1+ with pytest.raises(fastapi.HTTPException):+ await rate_limit.enforce_global_search_rate_limit("u1")+ assert (+ _counter("autogpt_rate_limit_hits_total", endpoint="/api/search") == before + 1+ )diff --git a/autogpt_platform/backend/backend/api/features/v1.py b/autogpt_platform/backend/backend/api/features/v1.pyindex 8c4e93e5c5..c4b57333cf 100644--- a/autogpt_platform/backend/backend/api/features/v1.py+++ b/autogpt_platform/backend/backend/api/features/v1.py@@ -153,7 +153,6 @@ from backend.integrations.webhooks.graph_lifecycle_hooks import ()from backend.monitoring.instrumentation import (record_block_execution,- record_graph_execution,record_graph_operation,)from backend.notifications import lifecycle@@ -2120,8 +2119,6 @@ async def execute_graph(organization_id=ctx.org_id,team_id=ctx.team_id,)- # Record successful graph execution- record_graph_execution(graph_id=graph_id, status="success", user_id=user_id)record_graph_operation(operation="execute", status="success")if source == "library":await complete_onboarding_step(user_id, OnboardingStep.LIBRARY_RUN_AGENT)@@ -2129,10 +2126,6 @@ async def execute_graph(await complete_onboarding_step(user_id, OnboardingStep.BUILDER_RUN_AGENT)return resultexcept GraphValidationError as e:- # Record failed graph execution- record_graph_execution(- graph_id=graph_id, status="validation_error", user_id=user_id- )record_graph_operation(operation="execute", status="validation_error")# Return structured validation errors that the frontend can parseraise HTTPException(@@ -2145,8 +2138,6 @@ async def execute_graph(},)except Exception:- # Record any other failures- record_graph_execution(graph_id=graph_id, status="error", user_id=user_id)record_graph_operation(operation="execute", status="error")raisediff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex dd4efb46b0..83f50020fc 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -50,6 +50,7 @@ from backend.monitoring import (report_late_executions,send_due_briefings,)+from backend.monitoring.instrumentation import SCHEDULER_JOBSfrom backend.util.clients import (get_database_manager_async_client,get_database_manager_client,@@ -2247,6 +2248,13 @@ class Scheduler(AppService):if self._jobs_cache_version == version_at_start:self._jobs_cache = jobsself._jobs_cache_expires_at = time.monotonic() + self._JOBS_CACHE_TTL_S+ # The one scheduler metric with an alert on it was never set.+ # Only an accepted read may publish it: a read that was+ # invalidated mid-query is stale by definition and must not+ # overwrite a newer count another reader has already set.+ SCHEDULER_JOBS.labels(job_type="execution", status="scheduled").set(+ len(jobs)+ )return jobsdef _invalidate_jobs_cache(self) -> None:diff --git a/autogpt_platform/backend/backend/executor/scheduler_test.py b/autogpt_platform/backend/backend/executor/scheduler_test.pyindex 9b7e7d7a00..5c7c1e0b0f 100644--- a/autogpt_platform/backend/backend/executor/scheduler_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_test.py@@ -847,3 +847,69 @@ class TestLaunchDarklyLifecycle:):sched._shutdown_launchdarkly_for_scheduler()shutdown.assert_not_called()+++def _counter(name: str, **labels) -> float:+ from prometheus_client import REGISTRY++ return REGISTRY.get_sample_value(name, labels) or 0.0+++def test_get_jobs_cached_sets_the_scheduler_jobs_gauge():+ """autogpt_scheduler_jobs has an alert on it and was never set."""+ from unittest.mock import MagicMock++ from backend.executor.scheduler import Scheduler++ s = Scheduler.__new__(Scheduler)+ s._jobs_cache = None+ s._jobs_cache_expires_at = 0.0+ s._jobs_cache_version = 0+ s.scheduler = MagicMock()+ s.scheduler.get_jobs.return_value = [object(), object(), object()]++ assert len(s._get_jobs_cached()) == 3+ assert (+ _counter("autogpt_scheduler_jobs", job_type="execution", status="scheduled")+ == 3+ )+++def test_stale_read_invalidated_mid_query_does_not_overwrite_the_gauge():+ """An invalidation that lands while get_jobs() is in flight rejects the+ cache write; the gauge must be rejected with it, or a slow stale read can+ overwrite a newer count that another reader already published."""+ from unittest.mock import MagicMock++ from backend.executor.scheduler import Scheduler++ s = Scheduler.__new__(Scheduler)+ s._jobs_cache = None+ s._jobs_cache_expires_at = 0.0+ s._jobs_cache_version = 0+ s.scheduler = MagicMock()++ # A fresh, accepted read publishes 5.+ s.scheduler.get_jobs.return_value = [object()] * 5+ s._get_jobs_cached()++ def gauge() -> float:+ return _counter(+ "autogpt_scheduler_jobs", job_type="execution", status="scheduled"+ )++ assert gauge() == 5++ # Now a read whose query is interrupted by an invalidation: it returns a+ # different count, but the version moved, so the cache write is skipped.+ s._jobs_cache = None+ s._jobs_cache_expires_at = 0.0++ def _slow_query_then_invalidated(**_):+ s._invalidate_jobs_cache()+ return [object()] * 2++ s.scheduler.get_jobs.side_effect = _slow_query_then_invalidated+ assert len(s._get_jobs_cached()) == 2 # caller still gets the list+ assert s._jobs_cache is None # write-back was rejected+ assert gauge() == 5 # and so was the gauge updatediff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.pyindex 19866a8556..31995a6fdd 100644--- a/autogpt_platform/backend/backend/executor/utils.py+++ b/autogpt_platform/backend/backend/executor/utils.py@@ -45,6 +45,7 @@ from backend.data.model import ()from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfigfrom backend.integrations.credentials_store import is_system_credential+from backend.monitoring.instrumentation import record_graph_executionfrom backend.util.clients import (get_async_execution_event_bus,get_async_execution_queue,@@ -1245,6 +1246,61 @@ async def add_graph_execution(*,expert_id: Optional[str] = None,bypass_paywall: bool = False,+) -> GraphExecutionWithNodes:+ """Add a graph execution to the queue, recording the outcome.++ Thin wrapper over :func:`_add_graph_execution` so that every caller of+ this shared path, not only the legacy v1 route, feeds+ ``autogpt_graph_executions_total``. A paywall rejection is a policy gate,+ not an execute outcome, and is not counted.+ """+ try:+ result = await _add_graph_execution(+ graph_id=graph_id,+ user_id=user_id,+ inputs=inputs,+ preset_id=preset_id,+ graph_version=graph_version,+ graph_credentials_inputs=graph_credentials_inputs,+ nodes_input_masks=nodes_input_masks,+ execution_context=execution_context,+ graph_exec_id=graph_exec_id,+ dry_run=dry_run,+ organization_id=organization_id,+ team_id=team_id,+ expert_id=expert_id,+ bypass_paywall=bypass_paywall,+ )+ except GraphValidationError:+ record_graph_execution(+ graph_id=graph_id, status="validation_error", user_id=user_id+ )+ raise+ except UserPaywalledError:+ raise+ except Exception:+ record_graph_execution(graph_id=graph_id, status="error", user_id=user_id)+ raise+ record_graph_execution(graph_id=graph_id, status="success", user_id=user_id)+ return result+++async def _add_graph_execution(+ graph_id: str,+ user_id: str,+ inputs: Optional[GraphInput] = None,+ preset_id: Optional[str] = None,+ graph_version: Optional[int] = None,+ graph_credentials_inputs: Optional[Mapping[str, CredentialsMetaInput]] = None,+ nodes_input_masks: Optional[NodesInputMasks] = None,+ execution_context: Optional[ExecutionContext] = None,+ graph_exec_id: Optional[str] = None,+ dry_run: bool = False,+ organization_id: Optional[str] = None,+ team_id: Optional[str] = None,+ *,+ expert_id: Optional[str] = None,+ bypass_paywall: bool = False,) -> GraphExecutionWithNodes:"""Adds a graph execution to the queue and returns the execution entry.diff --git a/autogpt_platform/backend/backend/executor/utils_test.py b/autogpt_platform/backend/backend/executor/utils_test.pyindex 9c6eb904f4..4fd3eee14c 100644--- a/autogpt_platform/backend/backend/executor/utils_test.py+++ b/autogpt_platform/backend/backend/executor/utils_test.py@@ -2706,3 +2706,56 @@ async def test_add_graph_execution_subgraph_untenanted_parent_triggers_fallback(assert create_kwargs["organization_id"] == "org-sub"assert create_kwargs["team_id"] == "team-sub"assert create_kwargs["parent_graph_exec_id"] == "parent-123"+++def _counter(name: str, **labels) -> float:+ from prometheus_client import REGISTRY++ return REGISTRY.get_sample_value(name, labels) or 0.0+++async def test_add_graph_execution_records_outcome(mocker):+ """Every caller of the shared execute path must feed+ autogpt_graph_executions_total; before this only the legacy v1 route did."""+ from unittest.mock import AsyncMock++ from backend.executor import utils+ from backend.util.exceptions import GraphValidationError, UserPaywalledError++ def n(status):+ return _counter("autogpt_graph_executions_total", status=status)++ ok, verr, err = n("success"), n("validation_error"), n("error")++ mocker.patch.object(utils, "_add_graph_execution", AsyncMock(return_value="row"))+ assert await utils.add_graph_execution(graph_id="g", user_id="u") == "row"+ assert n("success") == ok + 1++ mocker.patch.object(+ utils,+ "_add_graph_execution",+ AsyncMock(side_effect=GraphValidationError("bad", {})),+ )+ with pytest.raises(GraphValidationError):+ await utils.add_graph_execution(graph_id="g", user_id="u")+ assert n("validation_error") == verr + 1++ mocker.patch.object(+ utils, "_add_graph_execution", AsyncMock(side_effect=RuntimeError("boom"))+ )+ with pytest.raises(RuntimeError):+ await utils.add_graph_execution(graph_id="g", user_id="u")+ assert n("error") == err + 1++ # A paywall is a policy gate, not an execute outcome: nothing is counted.+ mocker.patch.object(+ utils, "_add_graph_execution", AsyncMock(side_effect=UserPaywalledError("pay"))+ )+ with pytest.raises(UserPaywalledError):+ await utils.add_graph_execution(graph_id="g", user_id="u")+ assert (n("success"), n("validation_error"), n("error")) == (+ ok + 1,+ verr + 1,+ err + 1,+ )