Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): serve the API servers' Prometheus metrics on internal ports (#15966)

Nicholas Tindle9812096b5ae0
diff --git a/autogpt_platform/backend/backend/api/external/fastapi_app.py b/autogpt_platform/backend/backend/api/external/fastapi_app.py
index 914a36cbd4..435f43464b 100644
--- a/autogpt_platform/backend/backend/api/external/fastapi_app.py
+++ b/autogpt_platform/backend/backend/api/external/fastapi_app.py
@@ -35,11 +35,6 @@ async def _graph_activation_error_handler(_request: Request, exc: GraphActivatio
return JSONResponse(status_code=400, content={"detail": str(exc)})
-# Add Prometheus instrumentation
-instrument_fastapi(
- external_api,
- service_name="external-api",
- expose_endpoint=True,
- endpoint="/metrics",
- include_in_schema=True,
-)
+# Add Prometheus instrumentation. The REST API process this app is mounted in
+# serves the metrics on their own port, never on this public app.
+instrument_fastapi(external_api, service_name="external-api", expose_endpoint=False)
diff --git a/autogpt_platform/backend/backend/api/public_metrics_test.py b/autogpt_platform/backend/backend/api/public_metrics_test.py
new file mode 100644
index 0000000000..0de8ed2943
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/public_metrics_test.py
@@ -0,0 +1,45 @@
+"""No public app routes /metrics: the servers serve them on ports of their own.
+
+The route would be added when a module is imported, so the probe imports the
+apps in a fresh process with metrics on.
+"""
+
+import ast
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+_PUBLIC_ROUTES_PROBE = """
+from starlette.routing import Mount
+
+from backend.api import rest_api, ws_api
+
+
+def paths(routes, prefix=""):
+ for route in routes:
+ yield prefix + route.path
+ if isinstance(route, Mount):
+ yield from paths(route.routes, prefix + route.path)
+
+
+print(sorted(set(paths(rest_api.app.routes)) | set(paths(ws_api.app.routes))))
+"""
+
+
+def test_no_public_app_routes_metrics() -> None:
+ probe = subprocess.run(
+ [sys.executable, "-c", _PUBLIC_ROUTES_PROBE],
+ cwd=Path(__file__).parents[2],
+ env={**os.environ, "ENABLE_METRICS": "true"},
+ capture_output=True,
+ text=True,
+ timeout=300,
+ )
+
+ assert probe.returncode == 0, probe.stderr
+ paths = ast.literal_eval(probe.stdout.strip().splitlines()[-1])
+ # The probe saw the real apps, the mounted external API's routes included.
+ assert {"/health", "/ws"} <= set(paths), paths
+ assert any(path.startswith("/external-api/") for path in paths), paths
+ assert [path for path in paths if path.endswith("/metrics")] == []
diff --git a/autogpt_platform/backend/backend/api/rest_api.py b/autogpt_platform/backend/backend/api/rest_api.py
index d4574e3d6d..8026183e94 100644
--- a/autogpt_platform/backend/backend/api/rest_api.py
+++ b/autogpt_platform/backend/backend/api/rest_api.py
@@ -103,7 +103,7 @@ from backend.copilot.rate_limit import UserPaywalledError
from backend.data.model import Credentials
from backend.integrations.providers import ProviderName
from backend.integrations.webhooks.graph_lifecycle_hooks import GraphActivationError
-from backend.monitoring.instrumentation import instrument_fastapi
+from backend.monitoring.instrumentation import instrument_fastapi, start_metrics_server
from backend.util import json
from backend.util.cloud_storage import shutdown_cloud_storage_handler
from backend.util.exceptions import (
@@ -298,15 +298,9 @@ add_auth_responses_to_openapi(app)
# Sort OpenAPI schema to eliminate diff on refactors
sort_openapi(app)
-# Add Prometheus instrumentation
-instrument_fastapi(
- app,
- service_name="rest-api",
- expose_endpoint=True,
- endpoint="/metrics",
- include_in_schema=settings.config.app_env
- == backend.util.settings.AppEnvironment.LOCAL,
-)
+# Add Prometheus instrumentation. The metrics are served by AgentServer.run on
+# their own port, never on this public app.
+instrument_fastapi(app, service_name="rest-api", expose_endpoint=False)
def handle_internal_http_error(status_code: int = 500, log_error: bool = True):
@@ -722,6 +716,9 @@ class AgentServer(backend.util.service.AppProcess):
# Enable asyncio debug mode via environment variable
os.environ["PYTHONASYNCIODEBUG"] = "1"
+ # This process's metrics, the mounted external API's included.
+ start_metrics_server(settings.config.agent_api_metrics_port)
+
# Configure uvicorn with performance optimizations from Kludex FastAPI tips
uvicorn.run(
app=server_app,
diff --git a/autogpt_platform/backend/backend/api/rest_api_test.py b/autogpt_platform/backend/backend/api/rest_api_test.py
new file mode 100644
index 0000000000..8aa1a9542b
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/rest_api_test.py
@@ -0,0 +1,26 @@
+import pytest
+
+from backend.api import rest_api
+
+
[email protected]("enabled", [True, False])
+def test_metrics_get_a_listener_of_their_own_only_when_enabled(
+ enabled: bool, mocker, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setenv("ENABLE_METRICS", "true" if enabled else "false")
+ # run() turns asyncio debug on under a local app_env; restored afterwards.
+ monkeypatch.setenv("PYTHONASYNCIODEBUG", "")
+ serve_api = mocker.patch("backend.api.rest_api.uvicorn.run")
+ serve_metrics = mocker.patch("backend.monitoring.instrumentation.start_http_server")
+
+ rest_api.AgentServer().run()
+
+ config = rest_api.settings.config
+ assert serve_api.call_args.kwargs["port"] == config.agent_api_port
+ if not enabled:
+ serve_metrics.assert_not_called()
+ return
+ serve_metrics.assert_called_once_with(
+ config.agent_api_metrics_port, addr=config.pyro_host
+ )
+ assert config.agent_api_metrics_port != config.agent_api_port
diff --git a/autogpt_platform/backend/backend/api/ws_api.py b/autogpt_platform/backend/backend/api/ws_api.py
index 458e20b9c8..77646218ee 100644
--- a/autogpt_platform/backend/backend/api/ws_api.py
+++ b/autogpt_platform/backend/backend/api/ws_api.py
@@ -20,6 +20,7 @@ from backend.data import db, redis_client
from backend.data.user import DEFAULT_USER_ID, is_user_banned
from backend.monitoring.instrumentation import (
instrument_fastapi,
+ start_metrics_server,
update_websocket_connections,
)
from backend.util.service import AppProcess
@@ -57,14 +58,9 @@ docs_url = "/docs" if settings.config.app_env == AppEnvironment.LOCAL else None
app = FastAPI(lifespan=lifespan, docs_url=docs_url)
_connection_manager = None
-# Add Prometheus instrumentation
-instrument_fastapi(
- app,
- service_name="websocket-server",
- expose_endpoint=True,
- endpoint="/metrics",
- include_in_schema=settings.config.app_env == AppEnvironment.LOCAL,
-)
+# Add Prometheus instrumentation. The metrics are served by WebsocketServer.run
+# on their own port, never on this public app.
+instrument_fastapi(app, service_name="websocket-server", expose_endpoint=False)
def get_connection_manager():
@@ -382,6 +378,7 @@ class WebsocketServer(AppProcess):
allow_headers=["*"],
)
+ start_metrics_server(Config().websocket_server_metrics_port)
uvicorn.run(
server_app,
host=Config().websocket_server_host,
diff --git a/autogpt_platform/backend/backend/api/ws_api_test.py b/autogpt_platform/backend/backend/api/ws_api_test.py
index c8f7f8966f..9e68fdfe07 100644
--- a/autogpt_platform/backend/backend/api/ws_api_test.py
+++ b/autogpt_platform/backend/backend/api/ws_api_test.py
@@ -78,6 +78,27 @@ def test_websocket_server_blocks_localhost_in_production(mocker) -> None:
WebsocketServer().run()
[email protected]("enabled", [True, False])
+def test_websocket_server_metrics_get_a_listener_of_their_own_only_when_enabled(
+ enabled: bool, mocker, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setenv("ENABLE_METRICS", "true" if enabled else "false")
+ serve_websockets = mocker.patch("backend.api.ws_api.uvicorn.run")
+ serve_metrics = mocker.patch("backend.monitoring.instrumentation.start_http_server")
+
+ WebsocketServer().run()
+
+ config = settings.config
+ assert serve_websockets.call_args.kwargs["port"] == config.websocket_server_port
+ if not enabled:
+ serve_metrics.assert_not_called()
+ return
+ serve_metrics.assert_called_once_with(
+ config.websocket_server_metrics_port, addr=config.pyro_host
+ )
+ assert config.websocket_server_metrics_port != config.websocket_server_port
+
+
@pytest.mark.asyncio
async def test_websocket_router_subscribe(
mock_websocket: AsyncMock, mock_manager: AsyncMock, snapshot: Snapshot, mocker
diff --git a/autogpt_platform/backend/backend/monitoring/instrumentation.py b/autogpt_platform/backend/backend/monitoring/instrumentation.py
index 45dc4a7f6e..a35db63e5f 100644
--- a/autogpt_platform/backend/backend/monitoring/instrumentation.py
+++ b/autogpt_platform/backend/backend/monitoring/instrumentation.py
@@ -6,12 +6,15 @@ for all FastAPI services in the AutoGPT platform.
"""
import logging
+import os
from typing import Optional
from fastapi import FastAPI
-from prometheus_client import Counter, Gauge, Histogram, Info
+from prometheus_client import Counter, Gauge, Histogram, Info, start_http_server
from prometheus_fastapi_instrumentator import Instrumentator, metrics
+from backend.util.settings import Config
+
logger = logging.getLogger(__name__)
# Custom business metrics with controlled cardinality
@@ -137,6 +140,24 @@ SERVICE_INFO = Info(
"Service information",
)
+METRICS_ENV_VAR = "ENABLE_METRICS"
+
+
+def metrics_enabled() -> bool:
+ """The switch the HTTP instrumentation obeys, read the way it reads it."""
+ return os.getenv(METRICS_ENV_VAR, "False").lower() in ("true", "1")
+
+
+def start_metrics_server(port: int) -> None:
+ """Serve this process's metrics on a listener of their own, when they are on.
+
+ The public servers call this instead of routing /metrics on their app, so
+ no client of theirs can read the metrics. Bound like the executors'
+ metrics: pyro_host is every service's internal listener address.
+ """
+ if metrics_enabled():
+ start_http_server(port, addr=Config().pyro_host)
+
def instrument_fastapi(
app: FastAPI,
@@ -186,7 +207,7 @@ def instrument_fastapi(
should_respect_env_var=True,
should_instrument_requests_inprogress=True,
excluded_handlers=excluded_handlers or ["/health", "/readiness"],
- env_var_name="ENABLE_METRICS",
+ env_var_name=METRICS_ENV_VAR,
inprogress_name=f"autogpt_{service_subsystem}_http_requests_inprogress",
inprogress_labels=True,
)
diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.py
index 37980fd254..62ae4d52df 100644
--- a/autogpt_platform/backend/backend/util/settings.py
+++ b/autogpt_platform/backend/backend/util/settings.py
@@ -441,6 +441,13 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):
description="The port for the websocket server to run on",
)
+ websocket_server_metrics_port: int = Field(
+ default=8014,
+ description="The port the websocket server serves its Prometheus metrics "
+ "on when ENABLE_METRICS is set: never the public websocket port, so a "
+ "client of the server can't read them.",
+ )
+
execution_manager_port: int = Field(
default=8002,
description="The port for execution manager daemon to run on",
@@ -484,6 +491,13 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):
description="The port for agent server API to run on",
)
+ agent_api_metrics_port: int = Field(
+ default=8013,
+ description="The port the agent server API serves its Prometheus metrics "
+ "on when ENABLE_METRICS is set: never the public API port, so a client of "
+ "the API can't read them.",
+ )
+
notification_service_port: int = Field(
default=8007,
description="The port for notification service daemon to run on",