Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): wire the application metrics that were defined but never recorded (#14319)

Bently125eb210fee2
diff --git a/autogpt_platform/backend/backend/api/features/credits_rate_limit.py b/autogpt_platform/backend/backend/api/features/credits_rate_limit.py
index 63e649d72f..686ef6b8d2 100644
--- a/autogpt_platform/backend/backend/api/features/credits_rate_limit.py
+++ b/autogpt_platform/backend/backend/api/features/credits_rate_limit.py
@@ -28,6 +28,7 @@ from autogpt_libs.auth import get_user_id
from redis.exceptions import RedisClusterException, RedisError
from backend.data.redis_client import get_redis_async
+from backend.monitoring.instrumentation import record_rate_limit_hit
logger = logging.getLogger(__name__)
@@ -142,6 +143,7 @@ async def enforce_subscription_status_rate_limit(
user_id,
count,
)
+ record_rate_limit_hit("/api/credits/subscription", user_id)
raise fastapi.HTTPException(
status_code=429,
detail=(
diff --git a/autogpt_platform/backend/backend/api/features/credits_rate_limit_test.py b/autogpt_platform/backend/backend/api/features/credits_rate_limit_test.py
index 62bd6d8087..acb9814a4a 100644
--- a/autogpt_platform/backend/backend/api/features/credits_rate_limit_test.py
+++ b/autogpt_platform/backend/backend/api/features/credits_rate_limit_test.py
@@ -199,3 +199,37 @@ async def test_per_user_keys_are_distinct(fake_redis):
assert len(keys) == 2
assert any("alice" in k for k in keys)
assert any("bob" in k for k in keys)
+
+
+def _counter(name: str, **labels) -> float:
+ from prometheus_client import REGISTRY
+
+ return REGISTRY.get_sample_value(name, labels) or 0.0
+
+
+async def test_over_limit_records_a_rate_limit_hit(mock_redis):
+ """The 429 must be visible to Prometheus; the counter had no callers."""
+ before = _counter(
+ "autogpt_rate_limit_hits_total", endpoint="/api/credits/subscription"
+ )
+ mock_redis.eval.return_value = rate_limit.SUBSCRIPTION_STATUS_MAX_REQUESTS + 1
+ with pytest.raises(fastapi.HTTPException):
+ await rate_limit.enforce_subscription_status_rate_limit("u1")
+ assert (
+ _counter("autogpt_rate_limit_hits_total", endpoint="/api/credits/subscription")
+ == before + 1
+ )
+
+
+async def test_under_limit_does_not_record(mock_redis):
+ before = _counter(
+ "autogpt_rate_limit_hits_total", endpoint="/api/credits/subscription"
+ )
+ mock_redis.eval.return_value = 1
+ await rate_limit.enforce_subscription_status_rate_limit("u1")
+ assert (
+ _counter("autogpt_rate_limit_hits_total", endpoint="/api/credits/subscription")
+ == before
+ )
diff --git a/autogpt_platform/backend/backend/api/features/search/rate_limit.py b/autogpt_platform/backend/backend/api/features/search/rate_limit.py
index 76f9fcecc4..5807443566 100644
--- a/autogpt_platform/backend/backend/api/features/search/rate_limit.py
+++ b/autogpt_platform/backend/backend/api/features/search/rate_limit.py
@@ -20,6 +20,7 @@ from datetime import UTC, datetime
import fastapi
from backend.data.redis_client import get_redis_async
+from backend.monitoring.instrumentation import record_rate_limit_hit
logger = logging.getLogger(__name__)
@@ -67,6 +68,7 @@ async def enforce_global_search_rate_limit(user_id: str) -> None:
return
if count > GLOBAL_SEARCH_MAX_REQUESTS:
+ record_rate_limit_hit("/api/search", user_id)
raise fastapi.HTTPException(
status_code=429,
detail=(
diff --git a/autogpt_platform/backend/backend/api/features/search/rate_limit_test.py b/autogpt_platform/backend/backend/api/features/search/rate_limit_test.py
index 58c98ecdd7..6720da7f94 100644
--- a/autogpt_platform/backend/backend/api/features/search/rate_limit_test.py
+++ b/autogpt_platform/backend/backend/api/features/search/rate_limit_test.py
@@ -99,3 +99,20 @@ async def test_per_user_keys_are_distinct(fake_redis):
assert key_a != key_b
assert "alice" in key_a
assert "bob" in key_b
+
+
+def _counter(name: str, **labels) -> float:
+ from prometheus_client import REGISTRY
+
+ return REGISTRY.get_sample_value(name, labels) or 0.0
+
+
+async def test_over_limit_records_a_rate_limit_hit(fake_redis):
+ before = _counter("autogpt_rate_limit_hits_total", endpoint="/api/search")
+ fake_redis.incr.return_value = rate_limit.GLOBAL_SEARCH_MAX_REQUESTS + 1
+ with pytest.raises(fastapi.HTTPException):
+ await rate_limit.enforce_global_search_rate_limit("u1")
+ assert (
+ _counter("autogpt_rate_limit_hits_total", endpoint="/api/search") == before + 1
+ )
diff --git a/autogpt_platform/backend/backend/api/features/v1.py b/autogpt_platform/backend/backend/api/features/v1.py
index 8c4e93e5c5..c4b57333cf 100644
--- a/autogpt_platform/backend/backend/api/features/v1.py
+++ b/autogpt_platform/backend/backend/api/features/v1.py
@@ -153,7 +153,6 @@ from backend.integrations.webhooks.graph_lifecycle_hooks import (
)
from backend.monitoring.instrumentation import (
record_block_execution,
- record_graph_execution,
record_graph_operation,
)
from backend.notifications import lifecycle
@@ -2120,8 +2119,6 @@ async def execute_graph(
organization_id=ctx.org_id,
team_id=ctx.team_id,
)
- # Record successful graph execution
- record_graph_execution(graph_id=graph_id, status="success", user_id=user_id)
record_graph_operation(operation="execute", status="success")
if source == "library":
await complete_onboarding_step(user_id, OnboardingStep.LIBRARY_RUN_AGENT)
@@ -2129,10 +2126,6 @@ async def execute_graph(
await complete_onboarding_step(user_id, OnboardingStep.BUILDER_RUN_AGENT)
return result
except GraphValidationError as e:
- # Record failed graph execution
- record_graph_execution(
- graph_id=graph_id, status="validation_error", user_id=user_id
- )
record_graph_operation(operation="execute", status="validation_error")
# Return structured validation errors that the frontend can parse
raise HTTPException(
@@ -2145,8 +2138,6 @@ async def execute_graph(
},
)
except Exception:
- # Record any other failures
- record_graph_execution(graph_id=graph_id, status="error", user_id=user_id)
record_graph_operation(operation="execute", status="error")
raise
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.py
index dd4efb46b0..83f50020fc 100644
--- a/autogpt_platform/backend/backend/executor/scheduler.py
+++ b/autogpt_platform/backend/backend/executor/scheduler.py
@@ -50,6 +50,7 @@ from backend.monitoring import (
report_late_executions,
send_due_briefings,
)
+from backend.monitoring.instrumentation import SCHEDULER_JOBS
from backend.util.clients import (
get_database_manager_async_client,
get_database_manager_client,
@@ -2247,6 +2248,13 @@ class Scheduler(AppService):
if self._jobs_cache_version == version_at_start:
self._jobs_cache = jobs
self._jobs_cache_expires_at = time.monotonic() + self._JOBS_CACHE_TTL_S
+ # The one scheduler metric with an alert on it was never set.
+ # Only an accepted read may publish it: a read that was
+ # invalidated mid-query is stale by definition and must not
+ # overwrite a newer count another reader has already set.
+ SCHEDULER_JOBS.labels(job_type="execution", status="scheduled").set(
+ len(jobs)
+ )
return jobs
def _invalidate_jobs_cache(self) -> None:
diff --git a/autogpt_platform/backend/backend/executor/scheduler_test.py b/autogpt_platform/backend/backend/executor/scheduler_test.py
index 9b7e7d7a00..5c7c1e0b0f 100644
--- a/autogpt_platform/backend/backend/executor/scheduler_test.py
+++ b/autogpt_platform/backend/backend/executor/scheduler_test.py
@@ -847,3 +847,69 @@ class TestLaunchDarklyLifecycle:
):
sched._shutdown_launchdarkly_for_scheduler()
shutdown.assert_not_called()
+
+
+def _counter(name: str, **labels) -> float:
+ from prometheus_client import REGISTRY
+
+ return REGISTRY.get_sample_value(name, labels) or 0.0
+
+
+def test_get_jobs_cached_sets_the_scheduler_jobs_gauge():
+ """autogpt_scheduler_jobs has an alert on it and was never set."""
+ from unittest.mock import MagicMock
+
+ from backend.executor.scheduler import Scheduler
+
+ s = Scheduler.__new__(Scheduler)
+ s._jobs_cache = None
+ s._jobs_cache_expires_at = 0.0
+ s._jobs_cache_version = 0
+ s.scheduler = MagicMock()
+ s.scheduler.get_jobs.return_value = [object(), object(), object()]
+
+ assert len(s._get_jobs_cached()) == 3
+ assert (
+ _counter("autogpt_scheduler_jobs", job_type="execution", status="scheduled")
+ == 3
+ )
+
+
+def test_stale_read_invalidated_mid_query_does_not_overwrite_the_gauge():
+ """An invalidation that lands while get_jobs() is in flight rejects the
+ cache write; the gauge must be rejected with it, or a slow stale read can
+ overwrite a newer count that another reader already published."""
+ from unittest.mock import MagicMock
+
+ from backend.executor.scheduler import Scheduler
+
+ s = Scheduler.__new__(Scheduler)
+ s._jobs_cache = None
+ s._jobs_cache_expires_at = 0.0
+ s._jobs_cache_version = 0
+ s.scheduler = MagicMock()
+
+ # A fresh, accepted read publishes 5.
+ s.scheduler.get_jobs.return_value = [object()] * 5
+ s._get_jobs_cached()
+
+ def gauge() -> float:
+ return _counter(
+ "autogpt_scheduler_jobs", job_type="execution", status="scheduled"
+ )
+
+ assert gauge() == 5
+
+ # Now a read whose query is interrupted by an invalidation: it returns a
+ # different count, but the version moved, so the cache write is skipped.
+ s._jobs_cache = None
+ s._jobs_cache_expires_at = 0.0
+
+ def _slow_query_then_invalidated(**_):
+ s._invalidate_jobs_cache()
+ return [object()] * 2
+
+ s.scheduler.get_jobs.side_effect = _slow_query_then_invalidated
+ assert len(s._get_jobs_cached()) == 2 # caller still gets the list
+ assert s._jobs_cache is None # write-back was rejected
+ assert gauge() == 5 # and so was the gauge update
diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.py
index 19866a8556..31995a6fdd 100644
--- a/autogpt_platform/backend/backend/executor/utils.py
+++ b/autogpt_platform/backend/backend/executor/utils.py
@@ -45,6 +45,7 @@ from backend.data.model import (
)
from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfig
from backend.integrations.credentials_store import is_system_credential
+from backend.monitoring.instrumentation import record_graph_execution
from backend.util.clients import (
get_async_execution_event_bus,
get_async_execution_queue,
@@ -1245,6 +1246,61 @@ async def add_graph_execution(
*,
expert_id: Optional[str] = None,
bypass_paywall: bool = False,
+) -> GraphExecutionWithNodes:
+ """Add a graph execution to the queue, recording the outcome.
+
+ Thin wrapper over :func:`_add_graph_execution` so that every caller of
+ this shared path, not only the legacy v1 route, feeds
+ ``autogpt_graph_executions_total``. A paywall rejection is a policy gate,
+ not an execute outcome, and is not counted.
+ """
+ try:
+ result = await _add_graph_execution(
+ graph_id=graph_id,
+ user_id=user_id,
+ inputs=inputs,
+ preset_id=preset_id,
+ graph_version=graph_version,
+ graph_credentials_inputs=graph_credentials_inputs,
+ nodes_input_masks=nodes_input_masks,
+ execution_context=execution_context,
+ graph_exec_id=graph_exec_id,
+ dry_run=dry_run,
+ organization_id=organization_id,
+ team_id=team_id,
+ expert_id=expert_id,
+ bypass_paywall=bypass_paywall,
+ )
+ except GraphValidationError:
+ record_graph_execution(
+ graph_id=graph_id, status="validation_error", user_id=user_id
+ )
+ raise
+ except UserPaywalledError:
+ raise
+ except Exception:
+ record_graph_execution(graph_id=graph_id, status="error", user_id=user_id)
+ raise
+ record_graph_execution(graph_id=graph_id, status="success", user_id=user_id)
+ return result
+
+
+async def _add_graph_execution(
+ graph_id: str,
+ user_id: str,
+ inputs: Optional[GraphInput] = None,
+ preset_id: Optional[str] = None,
+ graph_version: Optional[int] = None,
+ graph_credentials_inputs: Optional[Mapping[str, CredentialsMetaInput]] = None,
+ nodes_input_masks: Optional[NodesInputMasks] = None,
+ execution_context: Optional[ExecutionContext] = None,
+ graph_exec_id: Optional[str] = None,
+ dry_run: bool = False,
+ organization_id: Optional[str] = None,
+ team_id: Optional[str] = None,
+ *,
+ expert_id: Optional[str] = None,
+ bypass_paywall: bool = False,
) -> GraphExecutionWithNodes:
"""
Adds a graph execution to the queue and returns the execution entry.
diff --git a/autogpt_platform/backend/backend/executor/utils_test.py b/autogpt_platform/backend/backend/executor/utils_test.py
index 9c6eb904f4..4fd3eee14c 100644
--- a/autogpt_platform/backend/backend/executor/utils_test.py
+++ b/autogpt_platform/backend/backend/executor/utils_test.py
@@ -2706,3 +2706,56 @@ async def test_add_graph_execution_subgraph_untenanted_parent_triggers_fallback(
assert create_kwargs["organization_id"] == "org-sub"
assert create_kwargs["team_id"] == "team-sub"
assert create_kwargs["parent_graph_exec_id"] == "parent-123"
+
+
+def _counter(name: str, **labels) -> float:
+ from prometheus_client import REGISTRY
+
+ return REGISTRY.get_sample_value(name, labels) or 0.0
+
+
+async def test_add_graph_execution_records_outcome(mocker):
+ """Every caller of the shared execute path must feed
+ autogpt_graph_executions_total; before this only the legacy v1 route did."""
+ from unittest.mock import AsyncMock
+
+ from backend.executor import utils
+ from backend.util.exceptions import GraphValidationError, UserPaywalledError
+
+ def n(status):
+ return _counter("autogpt_graph_executions_total", status=status)
+
+ ok, verr, err = n("success"), n("validation_error"), n("error")
+
+ mocker.patch.object(utils, "_add_graph_execution", AsyncMock(return_value="row"))
+ assert await utils.add_graph_execution(graph_id="g", user_id="u") == "row"
+ assert n("success") == ok + 1
+
+ mocker.patch.object(
+ utils,
+ "_add_graph_execution",
+ AsyncMock(side_effect=GraphValidationError("bad", {})),
+ )
+ with pytest.raises(GraphValidationError):
+ await utils.add_graph_execution(graph_id="g", user_id="u")
+ assert n("validation_error") == verr + 1
+
+ mocker.patch.object(
+ utils, "_add_graph_execution", AsyncMock(side_effect=RuntimeError("boom"))
+ )
+ with pytest.raises(RuntimeError):
+ await utils.add_graph_execution(graph_id="g", user_id="u")
+ assert n("error") == err + 1
+
+ # A paywall is a policy gate, not an execute outcome: nothing is counted.
+ mocker.patch.object(
+ utils, "_add_graph_execution", AsyncMock(side_effect=UserPaywalledError("pay"))
+ )
+ with pytest.raises(UserPaywalledError):
+ await utils.add_graph_execution(graph_id="g", user_id="u")
+ assert (n("success"), n("validation_error"), n("error")) == (
+ ok + 1,
+ verr + 1,
+ err + 1,
+ )