AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): serve the API servers' Prometheus metrics on internal ports (#15966)
Nicholas Tindle9812096b5ae0
diff --git a/autogpt_platform/backend/backend/api/external/fastapi_app.py b/autogpt_platform/backend/backend/api/external/fastapi_app.pyindex 914a36cbd4..435f43464b 100644--- a/autogpt_platform/backend/backend/api/external/fastapi_app.py+++ b/autogpt_platform/backend/backend/api/external/fastapi_app.py@@ -35,11 +35,6 @@ async def _graph_activation_error_handler(_request: Request, exc: GraphActivatioreturn JSONResponse(status_code=400, content={"detail": str(exc)})-# Add Prometheus instrumentation-instrument_fastapi(- external_api,- service_name="external-api",- expose_endpoint=True,- endpoint="/metrics",- include_in_schema=True,-)+# Add Prometheus instrumentation. The REST API process this app is mounted in+# serves the metrics on their own port, never on this public app.+instrument_fastapi(external_api, service_name="external-api", expose_endpoint=False)diff --git a/autogpt_platform/backend/backend/api/public_metrics_test.py b/autogpt_platform/backend/backend/api/public_metrics_test.pynew file mode 100644index 0000000000..0de8ed2943--- /dev/null+++ b/autogpt_platform/backend/backend/api/public_metrics_test.py@@ -0,0 +1,45 @@+"""No public app routes /metrics: the servers serve them on ports of their own.++The route would be added when a module is imported, so the probe imports the+apps in a fresh process with metrics on.+"""++import ast+import os+import subprocess+import sys+from pathlib import Path++_PUBLIC_ROUTES_PROBE = """+from starlette.routing import Mount++from backend.api import rest_api, ws_api+++def paths(routes, prefix=""):+ for route in routes:+ yield prefix + route.path+ if isinstance(route, Mount):+ yield from paths(route.routes, prefix + route.path)+++print(sorted(set(paths(rest_api.app.routes)) | set(paths(ws_api.app.routes))))+"""+++def test_no_public_app_routes_metrics() -> None:+ probe = subprocess.run(+ [sys.executable, "-c", _PUBLIC_ROUTES_PROBE],+ cwd=Path(__file__).parents[2],+ env={**os.environ, "ENABLE_METRICS": "true"},+ capture_output=True,+ text=True,+ timeout=300,+ )++ assert probe.returncode == 0, probe.stderr+ paths = ast.literal_eval(probe.stdout.strip().splitlines()[-1])+ # The probe saw the real apps, the mounted external API's routes included.+ assert {"/health", "/ws"} <= set(paths), paths+ assert any(path.startswith("/external-api/") for path in paths), paths+ assert [path for path in paths if path.endswith("/metrics")] == []diff --git a/autogpt_platform/backend/backend/api/rest_api.py b/autogpt_platform/backend/backend/api/rest_api.pyindex d4574e3d6d..8026183e94 100644--- a/autogpt_platform/backend/backend/api/rest_api.py+++ b/autogpt_platform/backend/backend/api/rest_api.py@@ -103,7 +103,7 @@ from backend.copilot.rate_limit import UserPaywalledErrorfrom backend.data.model import Credentialsfrom backend.integrations.providers import ProviderNamefrom backend.integrations.webhooks.graph_lifecycle_hooks import GraphActivationError-from backend.monitoring.instrumentation import instrument_fastapi+from backend.monitoring.instrumentation import instrument_fastapi, start_metrics_serverfrom backend.util import jsonfrom backend.util.cloud_storage import shutdown_cloud_storage_handlerfrom backend.util.exceptions import (@@ -298,15 +298,9 @@ add_auth_responses_to_openapi(app)# Sort OpenAPI schema to eliminate diff on refactorssort_openapi(app)-# Add Prometheus instrumentation-instrument_fastapi(- app,- service_name="rest-api",- expose_endpoint=True,- endpoint="/metrics",- include_in_schema=settings.config.app_env- == backend.util.settings.AppEnvironment.LOCAL,-)+# Add Prometheus instrumentation. The metrics are served by AgentServer.run on+# their own port, never on this public app.+instrument_fastapi(app, service_name="rest-api", expose_endpoint=False)def handle_internal_http_error(status_code: int = 500, log_error: bool = True):@@ -722,6 +716,9 @@ class AgentServer(backend.util.service.AppProcess):# Enable asyncio debug mode via environment variableos.environ["PYTHONASYNCIODEBUG"] = "1"+ # This process's metrics, the mounted external API's included.+ start_metrics_server(settings.config.agent_api_metrics_port)+# Configure uvicorn with performance optimizations from Kludex FastAPI tipsuvicorn.run(app=server_app,diff --git a/autogpt_platform/backend/backend/api/rest_api_test.py b/autogpt_platform/backend/backend/api/rest_api_test.pynew file mode 100644index 0000000000..8aa1a9542b--- /dev/null+++ b/autogpt_platform/backend/backend/api/rest_api_test.py@@ -0,0 +1,26 @@+import pytest++from backend.api import rest_api++[email protected]("enabled", [True, False])+def test_metrics_get_a_listener_of_their_own_only_when_enabled(+ enabled: bool, mocker, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setenv("ENABLE_METRICS", "true" if enabled else "false")+ # run() turns asyncio debug on under a local app_env; restored afterwards.+ monkeypatch.setenv("PYTHONASYNCIODEBUG", "")+ serve_api = mocker.patch("backend.api.rest_api.uvicorn.run")+ serve_metrics = mocker.patch("backend.monitoring.instrumentation.start_http_server")++ rest_api.AgentServer().run()++ config = rest_api.settings.config+ assert serve_api.call_args.kwargs["port"] == config.agent_api_port+ if not enabled:+ serve_metrics.assert_not_called()+ return+ serve_metrics.assert_called_once_with(+ config.agent_api_metrics_port, addr=config.pyro_host+ )+ assert config.agent_api_metrics_port != config.agent_api_portdiff --git a/autogpt_platform/backend/backend/api/ws_api.py b/autogpt_platform/backend/backend/api/ws_api.pyindex 458e20b9c8..77646218ee 100644--- a/autogpt_platform/backend/backend/api/ws_api.py+++ b/autogpt_platform/backend/backend/api/ws_api.py@@ -20,6 +20,7 @@ from backend.data import db, redis_clientfrom backend.data.user import DEFAULT_USER_ID, is_user_bannedfrom backend.monitoring.instrumentation import (instrument_fastapi,+ start_metrics_server,update_websocket_connections,)from backend.util.service import AppProcess@@ -57,14 +58,9 @@ docs_url = "/docs" if settings.config.app_env == AppEnvironment.LOCAL else Noneapp = FastAPI(lifespan=lifespan, docs_url=docs_url)_connection_manager = None-# Add Prometheus instrumentation-instrument_fastapi(- app,- service_name="websocket-server",- expose_endpoint=True,- endpoint="/metrics",- include_in_schema=settings.config.app_env == AppEnvironment.LOCAL,-)+# Add Prometheus instrumentation. The metrics are served by WebsocketServer.run+# on their own port, never on this public app.+instrument_fastapi(app, service_name="websocket-server", expose_endpoint=False)def get_connection_manager():@@ -382,6 +378,7 @@ class WebsocketServer(AppProcess):allow_headers=["*"],)+ start_metrics_server(Config().websocket_server_metrics_port)uvicorn.run(server_app,host=Config().websocket_server_host,diff --git a/autogpt_platform/backend/backend/api/ws_api_test.py b/autogpt_platform/backend/backend/api/ws_api_test.pyindex c8f7f8966f..9e68fdfe07 100644--- a/autogpt_platform/backend/backend/api/ws_api_test.py+++ b/autogpt_platform/backend/backend/api/ws_api_test.py@@ -78,6 +78,27 @@ def test_websocket_server_blocks_localhost_in_production(mocker) -> None:WebsocketServer().run()[email protected]("enabled", [True, False])+def test_websocket_server_metrics_get_a_listener_of_their_own_only_when_enabled(+ enabled: bool, mocker, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setenv("ENABLE_METRICS", "true" if enabled else "false")+ serve_websockets = mocker.patch("backend.api.ws_api.uvicorn.run")+ serve_metrics = mocker.patch("backend.monitoring.instrumentation.start_http_server")++ WebsocketServer().run()++ config = settings.config+ assert serve_websockets.call_args.kwargs["port"] == config.websocket_server_port+ if not enabled:+ serve_metrics.assert_not_called()+ return+ serve_metrics.assert_called_once_with(+ config.websocket_server_metrics_port, addr=config.pyro_host+ )+ assert config.websocket_server_metrics_port != config.websocket_server_port++@pytest.mark.asyncioasync def test_websocket_router_subscribe(mock_websocket: AsyncMock, mock_manager: AsyncMock, snapshot: Snapshot, mockerdiff --git a/autogpt_platform/backend/backend/monitoring/instrumentation.py b/autogpt_platform/backend/backend/monitoring/instrumentation.pyindex 45dc4a7f6e..a35db63e5f 100644--- a/autogpt_platform/backend/backend/monitoring/instrumentation.py+++ b/autogpt_platform/backend/backend/monitoring/instrumentation.py@@ -6,12 +6,15 @@ for all FastAPI services in the AutoGPT platform."""import logging+import osfrom typing import Optionalfrom fastapi import FastAPI-from prometheus_client import Counter, Gauge, Histogram, Info+from prometheus_client import Counter, Gauge, Histogram, Info, start_http_serverfrom prometheus_fastapi_instrumentator import Instrumentator, metrics+from backend.util.settings import Config+logger = logging.getLogger(__name__)# Custom business metrics with controlled cardinality@@ -137,6 +140,24 @@ SERVICE_INFO = Info("Service information",)+METRICS_ENV_VAR = "ENABLE_METRICS"+++def metrics_enabled() -> bool:+ """The switch the HTTP instrumentation obeys, read the way it reads it."""+ return os.getenv(METRICS_ENV_VAR, "False").lower() in ("true", "1")+++def start_metrics_server(port: int) -> None:+ """Serve this process's metrics on a listener of their own, when they are on.++ The public servers call this instead of routing /metrics on their app, so+ no client of theirs can read the metrics. Bound like the executors'+ metrics: pyro_host is every service's internal listener address.+ """+ if metrics_enabled():+ start_http_server(port, addr=Config().pyro_host)+def instrument_fastapi(app: FastAPI,@@ -186,7 +207,7 @@ def instrument_fastapi(should_respect_env_var=True,should_instrument_requests_inprogress=True,excluded_handlers=excluded_handlers or ["/health", "/readiness"],- env_var_name="ENABLE_METRICS",+ env_var_name=METRICS_ENV_VAR,inprogress_name=f"autogpt_{service_subsystem}_http_requests_inprogress",inprogress_labels=True,)diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.pyindex 37980fd254..62ae4d52df 100644--- a/autogpt_platform/backend/backend/util/settings.py+++ b/autogpt_platform/backend/backend/util/settings.py@@ -441,6 +441,13 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):description="The port for the websocket server to run on",)+ websocket_server_metrics_port: int = Field(+ default=8014,+ description="The port the websocket server serves its Prometheus metrics "+ "on when ENABLE_METRICS is set: never the public websocket port, so a "+ "client of the server can't read them.",+ )+execution_manager_port: int = Field(default=8002,description="The port for execution manager daemon to run on",@@ -484,6 +491,13 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):description="The port for agent server API to run on",)+ agent_api_metrics_port: int = Field(+ default=8013,+ description="The port the agent server API serves its Prometheus metrics "+ "on when ENABLE_METRICS is set: never the public API port, so a client of "+ "the API can't read them.",+ )+notification_service_port: int = Field(default=8007,description="The port for notification service daemon to run on",