Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): implement azure api management source (#118173)

Daniel RC0888827c2ce1
diff --git a/frontend/public/services/azure_api_management.png b/frontend/public/services/azure_api_management.png
index c5736e43b50d..2130e12512af 100644
Binary files a/frontend/public/services/azure_api_management.png and b/frontend/public/services/azure_api_management.png differ
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
index 13a501fc371d..32f47b1bfa7e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
@@ -136,6 +136,7 @@ the row lists both.
| aws_systems_manager | HTTP | requests | ✅ |
| aws_waf | HTTP | requests | ✅ |
| axiom | HTTP | requests | ✅ |
+| azure_api_management | HTTP | requests | ✅ |
| azure_application_insights | HTTP | requests | ✅ |
| azure_blob | HTTP (vendor SDK) | azure-storage-blob (listing), adlfs (reads), deltalake | ⚠️ |
| azure_cost_management | HTTP | requests | ✅ |
@@ -1066,7 +1067,6 @@ doesn't conflict with concurrent PRs.
- aws_xray
- azure_activity_log
- azure_advisor
-- azure_api_management
- azure_data_explorer
- azure_data_factory
- azure_monitor_alerts
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/canonical_descriptions.py
new file mode 100644
index 000000000000..a888a2878de9
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/canonical_descriptions.py
@@ -0,0 +1,71 @@
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
+ CanonicalDescriptions,
+)
+
+_SHARED_COLUMNS = {
+ "TimeGenerated": "Date and time when request processing started.",
+ "_ItemId": "Unique identifier of the log record.",
+ "_ResourceId": "Azure resource ID of the API Management instance that wrote the record.",
+ "_SubscriptionId": "Azure subscription that the API Management instance belongs to.",
+ "TenantId": "ID of the Log Analytics workspace that stores the record.",
+ "Region": "Azure region of the API Management gateway that handled the request.",
+ "CorrelationId": "ID that groups the records written for one request.",
+}
+
+CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
+ "ApiManagementGatewayLogs": {
+ "description": "One row per request that passed through the API Management gateway, with timings, response codes, and the API, operation, product, and subscription it matched.",
+ "docs_url": "https://learn.microsoft.com/en-us/azure/azure-monitor/reference/tables/apimanagementgatewaylogs",
+ "columns": {
+ **_SHARED_COLUMNS,
+ "ApiId": "ID of the API the request matched.",
+ "OperationId": "ID of the API operation the request matched.",
+ "ProductId": "ID of the product the calling subscription belongs to.",
+ "ApimSubscriptionId": "ID of the API Management subscription key the caller used.",
+ "UserId": "ID of the API Management user that owns the subscription.",
+ "Method": "HTTP method of the client request.",
+ "Url": "URL of the client request.",
+ "ResponseCode": "HTTP status code the gateway returned to the client.",
+ "BackendResponseCode": "HTTP status code the backend returned to the gateway.",
+ "TotalTime": "Time in milliseconds from the first byte the gateway received to the last byte it sent.",
+ "BackendTime": "Time in milliseconds the gateway spent waiting for the backend.",
+ "CallerIpAddress": "IP address of the client that called the gateway.",
+ "IsRequestSuccess": "Whether the request completed with a success status code.",
+ "LastErrorReason": "Reason of the last error that occurred while the gateway processed the request.",
+ "LastErrorMessage": "Message of the last error that occurred while the gateway processed the request.",
+ "RequestBody": "Client request body, when body logging is turned on.",
+ "ResponseBody": "Gateway response body, when body logging is turned on.",
+ },
+ },
+ "ApiManagementGatewayLlmLog": {
+ "description": "One row per language model request that passed through the API Management AI gateway, with token counts and the model used.",
+ "docs_url": "https://learn.microsoft.com/en-us/azure/azure-monitor/reference/tables/apimanagementgatewayllmlog",
+ "columns": {
+ **_SHARED_COLUMNS,
+ "CorrelationId": "ID that matches the CorrelationId of the same request in ApiManagementGatewayLogs.",
+ "ModelName": "Model name used by the request.",
+ "DeploymentName": "Deployment name used by the client.",
+ "ApiVersion": "API version used by the client.",
+ "PromptTokens": "Number of prompt tokens used by the request.",
+ "CompletionTokens": "Number of completion tokens used by the request.",
+ "TotalTokens": "Total number of tokens used by the request.",
+ "IsStreamCompletion": "Whether the request asked for a streamed response.",
+ "RequestId": "Request ID that the language model returned.",
+ "RequestMessages": "Contents of the request messages.",
+ "ResponseMessages": "Contents of the response messages.",
+ "SequenceNumber": "Index of this entry in the message exchange.",
+ },
+ },
+ "ApiManagementWebSocketConnectionLogs": {
+ "description": "One row per WebSocket connection event that passed through the API Management gateway.",
+ "docs_url": "https://learn.microsoft.com/en-us/azure/azure-monitor/reference/tables/apimanagementwebsocketconnectionlogs",
+ "columns": {
+ **_SHARED_COLUMNS,
+ "CorrelationId": "ID that groups the events of one WebSocket request.",
+ "EventName": "Name of the event that describes the operation.",
+ "Source": "Source of the request or message for the WebSocket connection.",
+ "Destination": "Destination of the request or message for the WebSocket connection.",
+ "Error": "Error details for the WebSocket connection, if any.",
+ },
+ },
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/source.py
index 807bbfa022fd..1619e3dfdae7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/source.py
@@ -1,16 +1,60 @@
from typing import cast
-from products.warehouse_sources.backend.facade.source_config import DataWarehouseSourceCategory, SourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource
+from products.warehouse_sources.backend.facade.codebase_detection import NOT_DETECTABLE
+from products.warehouse_sources.backend.facade.source_config import (
+ DataWarehouseSourceCategory,
+ ReleaseStatus,
+ SourceConfig,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.azure_log_analytics import (
+ AzureLogAnalyticsResumeConfig,
+ WorkspaceCredentials,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.source import (
+ WORKSPACE_FIELDS,
+ LogAnalyticsWorkspaceSource,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
+ CanonicalDescriptions,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.azureapimanagement import (
AzureApiManagementSourceConfig,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
+# The resource-specific tables API Management writes when its diagnostic setting sends logs to a
+# Log Analytics workspace.
+API_MANAGEMENT_TABLES = [
+ "ApiManagementGatewayLogs",
+ "ApiManagementGatewayLlmLog",
+ "ApiManagementWebSocketConnectionLogs",
+]
+
@SourceRegistry.register
-class AzureApiManagementSource(SimpleSource[AzureApiManagementSourceConfig]):
+class AzureApiManagementSource(LogAnalyticsWorkspaceSource[AzureApiManagementSourceConfig]):
+ # Applications call an API Management gateway over plain HTTP, so no package or key names it.
+ codebase_detection = NOT_DETECTABLE
+
+ lists_tables_without_credentials = True
+ api_docs_url = "https://learn.microsoft.com/en-us/azure/api-management/monitor-api-management"
+
+ # A table is missing when its log category is not turned on, not because of a typo.
+ missing_table_reason = (
+ "No logs for this table yet. Turn on its log category in the API Management diagnostic setting."
+ )
+ missing_table_message = (
+ "The workspace has no {table} logs yet. Turn on its log category in the API Management diagnostic setting."
+ )
+ no_tables_message = (
+ "This workspace has no API Management logs. Check that the diagnostic setting sends logs to this "
+ "workspace with Resource specific tables."
+ )
+
@property
def source_type(self) -> ExternalDataSourceType:
return ExternalDataSourceType.AZUREAPIMANAGEMENT
@@ -21,7 +65,45 @@ class AzureApiManagementSource(SimpleSource[AzureApiManagementSourceConfig]):
name=ExternalDataSourceType.AZUREAPIMANAGEMENT,
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="Microsoft Azure API Management",
+ keywords=["apim", "api gateway", "ai gateway"],
+ releaseStatus=ReleaseStatus.ALPHA,
+ caption="""Sync Azure API Management gateway logs into the PostHog Data warehouse: requests, LLM token usage, and WebSocket connections.
+
+API Management must send its logs to a Log Analytics workspace. In the API Management instance, add a diagnostic setting that sends the `GatewayLogs`, `GatewayLlmLogs` and `WebSocketConnectionLogs` categories to the workspace, with **Resource specific** as the destination table.
+
+Register an app in Microsoft Entra ID, create a client secret for it, and give its service principal the **Log Analytics Reader** role on the workspace. Then enter the directory (tenant) ID, the application (client) ID, the secret value, and the workspace ID from the workspace's **Overview** page.""",
iconPath="/static/services/azure_api_management.png",
- fields=cast(list[FieldType], []),
- unreleasedSource=True,
+ docsUrl="https://posthog.com/docs/cdp/sources/azure-api-management",
+ fields=cast(list[FieldType], WORKSPACE_FIELDS),
+ )
+
+ def get_canonical_descriptions(self) -> CanonicalDescriptions:
+ from products.warehouse_sources.backend.temporal.data_imports.sources.azure_api_management.canonical_descriptions import (
+ CANONICAL_DESCRIPTIONS,
)
+
+ return CANONICAL_DESCRIPTIONS
+
+ def credentials(self, config: AzureApiManagementSourceConfig) -> WorkspaceCredentials:
+ return WorkspaceCredentials(
+ tenant_id=config.tenant_id,
+ client_id=config.client_id,
+ client_secret=config.client_secret,
+ workspace_id=config.workspace_id,
+ )
+
+ def table_names(self, config: AzureApiManagementSourceConfig) -> list[str]:
+ return API_MANAGEMENT_TABLES
+
+ def tables_checked_at_creation(self, config: AzureApiManagementSourceConfig) -> list[str]:
+ return API_MANAGEMENT_TABLES
+
+ def source_for_pipeline(
+ self,
+ config: AzureApiManagementSourceConfig,
+ resumable_source_manager: ResumableSourceManager[AzureLogAnalyticsResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
+ if inputs.schema_name not in API_MANAGEMENT_TABLES:
+ raise ValueError(f"Unknown Azure API Management schema '{inputs.schema_name}'")
+ return super().source_for_pipeline(config, resumable_source_manager, inputs)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/tests/test_azure_api_management_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/tests/test_azure_api_management_source.py
new file mode 100644
index 000000000000..4b74c4768444
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/tests/test_azure_api_management_source.py
@@ -0,0 +1,119 @@
+import pytest
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_api_management.source import (
+ AzureApiManagementSource,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ ScriptedResponse,
+ route,
+ scripted_network,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.azureapimanagement import (
+ AzureApiManagementSourceConfig,
+)
+
+TENANT = "11111111-1111-1111-1111-111111111111"
+WORKSPACE = "22222222-2222-2222-2222-222222222222"
+TOKEN_PATH = f"/{TENANT}/oauth2/v2.0/token"
+QUERY_PATH = f"/v1/workspaces/{WORKSPACE}/query"
+CONFIG = AzureApiManagementSourceConfig(
+ tenant_id=TENANT, client_id="client", client_secret="secret", workspace_id=WORKSPACE
+)
+
+
+def _token() -> ScriptedResponse:
+ return ScriptedResponse(json={"token_type": "Bearer", "expires_in": 3599, "access_token": "token-1"})
+
+
+def _unresolved(table: str) -> ScriptedResponse:
+ return ScriptedResponse(
+ status=400,
+ json={
+ "error": {
+ "code": "BadArgumentError",
+ "message": "The request had some invalid properties",
+ "innererror": {
+ "code": "SemanticError",
+ "message": f"'union' operator: Failed to resolve table or column expression named '{table}'",
+ },
+ }
+ },
+ )
+
+
+def _empty() -> ScriptedResponse:
+ return ScriptedResponse(json={"tables": [{"name": "PrimaryResult", "columns": [], "rows": []}]})
+
+
+def _basic_plan() -> ScriptedResponse:
+ return ScriptedResponse(
+ status=400,
+ json={"error": {"code": "BadArgumentError", "message": "The query can't run on a table with the Basic plan."}},
+ )
+
+
+ ("gateway", "llm", "websocket", "expected_ok", "expected_probes"),
+ [
+ (
+ _unresolved("ApiManagementGatewayLogs"),
+ _unresolved("ApiManagementGatewayLlmLog"),
+ _unresolved("ApiManagementWebSocketConnectionLogs"),
+ False,
+ 3,
+ ),
+ (_basic_plan(), _empty(), _unresolved("ApiManagementWebSocketConnectionLogs"), True, 2),
+ ],
+)
+def test_creation_fails_only_when_the_workspace_has_none_of_the_tables(
+ gateway: ScriptedResponse,
+ llm: ScriptedResponse,
+ websocket: ScriptedResponse,
+ expected_ok: bool,
+ expected_probes: int,
+) -> None:
+ with scripted_network(
+ route({TOKEN_PATH: [_token(), _token()], QUERY_PATH: [_empty(), gateway, llm, websocket]})
+ ) as network:
+ ok, message = AzureApiManagementSource().validate_credentials(CONFIG, team_id=1)
+
+ assert ok is expected_ok
+ if expected_ok:
+ assert message is None
+ else:
+ assert message is not None and "This workspace has no API Management logs" in message
+ queries = [request.json()["query"] for request in network.requests_log if request.path == QUERY_PATH]
+ assert queries[0] == "print ok = 1"
+ assert [query.split(" ")[0] for query in queries[1:]] == [
+ "ApiManagementGatewayLogs",
+ "ApiManagementGatewayLlmLog",
+ "ApiManagementWebSocketConnectionLogs",
+ ][:expected_probes]
+
+
+def test_a_missing_named_table_points_at_its_log_category() -> None:
+ with scripted_network(route({TOKEN_PATH: [_token()], QUERY_PATH: [_unresolved("ApiManagementGatewayLlmLog")]})):
+ ok, message = AzureApiManagementSource().validate_credentials(
+ CONFIG, team_id=1, schema_name="ApiManagementGatewayLlmLog"
+ )
+
+ assert ok is False
+ assert message is not None and "no ApiManagementGatewayLlmLog logs yet. Turn on its log category" in message
+
+
+def test_the_schema_picker_names_the_log_category_for_a_missing_table() -> None:
+ with scripted_network(
+ route(
+ {
+ TOKEN_PATH: [_token()],
+ QUERY_PATH: [_empty(), _unresolved("ApiManagementGatewayLlmLog")],
+ }
+ )
+ ):
+ permissions = AzureApiManagementSource().get_endpoint_permissions(
+ CONFIG, team_id=1, endpoints=["ApiManagementGatewayLogs", "ApiManagementGatewayLlmLog"]
+ )
+
+ assert permissions["ApiManagementGatewayLogs"] is None
+ assert permissions["ApiManagementGatewayLlmLog"] is not None
+ assert "log category" in permissions["ApiManagementGatewayLlmLog"]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureapimanagement.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureapimanagement.py
index e1e040f4cd1b..fb6b22289b70 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureapimanagement.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureapimanagement.py
@@ -6,4 +6,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common imp
@config.config
class AzureApiManagementSourceConfig(config.Config):
- pass
+ tenant_id: str
+ client_id: str
+ client_secret: str
+ workspace_id: str