:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): implement azure api management source (#118173)
Daniel RC0888827c2ce1
diff --git a/frontend/public/services/azure_api_management.png b/frontend/public/services/azure_api_management.pngindex c5736e43b50d..2130e12512af 100644Binary files a/frontend/public/services/azure_api_management.png and b/frontend/public/services/azure_api_management.png differdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.mdindex 13a501fc371d..32f47b1bfa7e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md@@ -136,6 +136,7 @@ the row lists both.| aws_systems_manager | HTTP | requests | ✅ || aws_waf | HTTP | requests | ✅ || axiom | HTTP | requests | ✅ |+| azure_api_management | HTTP | requests | ✅ || azure_application_insights | HTTP | requests | ✅ || azure_blob | HTTP (vendor SDK) | azure-storage-blob (listing), adlfs (reads), deltalake | ⚠️ || azure_cost_management | HTTP | requests | ✅ |@@ -1066,7 +1067,6 @@ doesn't conflict with concurrent PRs.- aws_xray- azure_activity_log- azure_advisor-- azure_api_management- azure_data_explorer- azure_data_factory- azure_monitor_alertsdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/canonical_descriptions.pynew file mode 100644index 000000000000..a888a2878de9--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/canonical_descriptions.py@@ -0,0 +1,71 @@+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (+ CanonicalDescriptions,+)++_SHARED_COLUMNS = {+ "TimeGenerated": "Date and time when request processing started.",+ "_ItemId": "Unique identifier of the log record.",+ "_ResourceId": "Azure resource ID of the API Management instance that wrote the record.",+ "_SubscriptionId": "Azure subscription that the API Management instance belongs to.",+ "TenantId": "ID of the Log Analytics workspace that stores the record.",+ "Region": "Azure region of the API Management gateway that handled the request.",+ "CorrelationId": "ID that groups the records written for one request.",+}++CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {+ "ApiManagementGatewayLogs": {+ "description": "One row per request that passed through the API Management gateway, with timings, response codes, and the API, operation, product, and subscription it matched.",+ "docs_url": "https://learn.microsoft.com/en-us/azure/azure-monitor/reference/tables/apimanagementgatewaylogs",+ "columns": {+ **_SHARED_COLUMNS,+ "ApiId": "ID of the API the request matched.",+ "OperationId": "ID of the API operation the request matched.",+ "ProductId": "ID of the product the calling subscription belongs to.",+ "ApimSubscriptionId": "ID of the API Management subscription key the caller used.",+ "UserId": "ID of the API Management user that owns the subscription.",+ "Method": "HTTP method of the client request.",+ "Url": "URL of the client request.",+ "ResponseCode": "HTTP status code the gateway returned to the client.",+ "BackendResponseCode": "HTTP status code the backend returned to the gateway.",+ "TotalTime": "Time in milliseconds from the first byte the gateway received to the last byte it sent.",+ "BackendTime": "Time in milliseconds the gateway spent waiting for the backend.",+ "CallerIpAddress": "IP address of the client that called the gateway.",+ "IsRequestSuccess": "Whether the request completed with a success status code.",+ "LastErrorReason": "Reason of the last error that occurred while the gateway processed the request.",+ "LastErrorMessage": "Message of the last error that occurred while the gateway processed the request.",+ "RequestBody": "Client request body, when body logging is turned on.",+ "ResponseBody": "Gateway response body, when body logging is turned on.",+ },+ },+ "ApiManagementGatewayLlmLog": {+ "description": "One row per language model request that passed through the API Management AI gateway, with token counts and the model used.",+ "docs_url": "https://learn.microsoft.com/en-us/azure/azure-monitor/reference/tables/apimanagementgatewayllmlog",+ "columns": {+ **_SHARED_COLUMNS,+ "CorrelationId": "ID that matches the CorrelationId of the same request in ApiManagementGatewayLogs.",+ "ModelName": "Model name used by the request.",+ "DeploymentName": "Deployment name used by the client.",+ "ApiVersion": "API version used by the client.",+ "PromptTokens": "Number of prompt tokens used by the request.",+ "CompletionTokens": "Number of completion tokens used by the request.",+ "TotalTokens": "Total number of tokens used by the request.",+ "IsStreamCompletion": "Whether the request asked for a streamed response.",+ "RequestId": "Request ID that the language model returned.",+ "RequestMessages": "Contents of the request messages.",+ "ResponseMessages": "Contents of the response messages.",+ "SequenceNumber": "Index of this entry in the message exchange.",+ },+ },+ "ApiManagementWebSocketConnectionLogs": {+ "description": "One row per WebSocket connection event that passed through the API Management gateway.",+ "docs_url": "https://learn.microsoft.com/en-us/azure/azure-monitor/reference/tables/apimanagementwebsocketconnectionlogs",+ "columns": {+ **_SHARED_COLUMNS,+ "CorrelationId": "ID that groups the events of one WebSocket request.",+ "EventName": "Name of the event that describes the operation.",+ "Source": "Source of the request or message for the WebSocket connection.",+ "Destination": "Destination of the request or message for the WebSocket connection.",+ "Error": "Error details for the WebSocket connection, if any.",+ },+ },+}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/source.pyindex 807bbfa022fd..1619e3dfdae7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/source.py@@ -1,16 +1,60 @@from typing import cast-from products.warehouse_sources.backend.facade.source_config import DataWarehouseSourceCategory, SourceConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource+from products.warehouse_sources.backend.facade.codebase_detection import NOT_DETECTABLE+from products.warehouse_sources.backend.facade.source_config import (+ DataWarehouseSourceCategory,+ ReleaseStatus,+ SourceConfig,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.azure_log_analytics import (+ AzureLogAnalyticsResumeConfig,+ WorkspaceCredentials,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.source import (+ WORKSPACE_FIELDS,+ LogAnalyticsWorkspaceSource,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (+ CanonicalDescriptions,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.azureapimanagement import (AzureApiManagementSourceConfig,)from products.warehouse_sources.backend.types import ExternalDataSourceType+# The resource-specific tables API Management writes when its diagnostic setting sends logs to a+# Log Analytics workspace.+API_MANAGEMENT_TABLES = [+ "ApiManagementGatewayLogs",+ "ApiManagementGatewayLlmLog",+ "ApiManagementWebSocketConnectionLogs",+]+@SourceRegistry.register-class AzureApiManagementSource(SimpleSource[AzureApiManagementSourceConfig]):+class AzureApiManagementSource(LogAnalyticsWorkspaceSource[AzureApiManagementSourceConfig]):+ # Applications call an API Management gateway over plain HTTP, so no package or key names it.+ codebase_detection = NOT_DETECTABLE++ lists_tables_without_credentials = True+ api_docs_url = "https://learn.microsoft.com/en-us/azure/api-management/monitor-api-management"++ # A table is missing when its log category is not turned on, not because of a typo.+ missing_table_reason = (+ "No logs for this table yet. Turn on its log category in the API Management diagnostic setting."+ )+ missing_table_message = (+ "The workspace has no {table} logs yet. Turn on its log category in the API Management diagnostic setting."+ )+ no_tables_message = (+ "This workspace has no API Management logs. Check that the diagnostic setting sends logs to this "+ "workspace with Resource specific tables."+ )+@propertydef source_type(self) -> ExternalDataSourceType:return ExternalDataSourceType.AZUREAPIMANAGEMENT@@ -21,7 +65,45 @@ class AzureApiManagementSource(SimpleSource[AzureApiManagementSourceConfig]):name=ExternalDataSourceType.AZUREAPIMANAGEMENT,category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,label="Microsoft Azure API Management",+ keywords=["apim", "api gateway", "ai gateway"],+ releaseStatus=ReleaseStatus.ALPHA,+ caption="""Sync Azure API Management gateway logs into the PostHog Data warehouse: requests, LLM token usage, and WebSocket connections.++API Management must send its logs to a Log Analytics workspace. In the API Management instance, add a diagnostic setting that sends the `GatewayLogs`, `GatewayLlmLogs` and `WebSocketConnectionLogs` categories to the workspace, with **Resource specific** as the destination table.++Register an app in Microsoft Entra ID, create a client secret for it, and give its service principal the **Log Analytics Reader** role on the workspace. Then enter the directory (tenant) ID, the application (client) ID, the secret value, and the workspace ID from the workspace's **Overview** page.""",iconPath="/static/services/azure_api_management.png",- fields=cast(list[FieldType], []),- unreleasedSource=True,+ docsUrl="https://posthog.com/docs/cdp/sources/azure-api-management",+ fields=cast(list[FieldType], WORKSPACE_FIELDS),+ )++ def get_canonical_descriptions(self) -> CanonicalDescriptions:+ from products.warehouse_sources.backend.temporal.data_imports.sources.azure_api_management.canonical_descriptions import (+ CANONICAL_DESCRIPTIONS,)++ return CANONICAL_DESCRIPTIONS++ def credentials(self, config: AzureApiManagementSourceConfig) -> WorkspaceCredentials:+ return WorkspaceCredentials(+ tenant_id=config.tenant_id,+ client_id=config.client_id,+ client_secret=config.client_secret,+ workspace_id=config.workspace_id,+ )++ def table_names(self, config: AzureApiManagementSourceConfig) -> list[str]:+ return API_MANAGEMENT_TABLES++ def tables_checked_at_creation(self, config: AzureApiManagementSourceConfig) -> list[str]:+ return API_MANAGEMENT_TABLES++ def source_for_pipeline(+ self,+ config: AzureApiManagementSourceConfig,+ resumable_source_manager: ResumableSourceManager[AzureLogAnalyticsResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:+ if inputs.schema_name not in API_MANAGEMENT_TABLES:+ raise ValueError(f"Unknown Azure API Management schema '{inputs.schema_name}'")+ return super().source_for_pipeline(config, resumable_source_manager, inputs)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/tests/test_azure_api_management_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/tests/test_azure_api_management_source.pynew file mode 100644index 000000000000..4b74c4768444--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_api_management/tests/test_azure_api_management_source.py@@ -0,0 +1,119 @@+import pytest++from products.warehouse_sources.backend.temporal.data_imports.sources.azure_api_management.source import (+ AzureApiManagementSource,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ ScriptedResponse,+ route,+ scripted_network,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.azureapimanagement import (+ AzureApiManagementSourceConfig,+)++TENANT = "11111111-1111-1111-1111-111111111111"+WORKSPACE = "22222222-2222-2222-2222-222222222222"+TOKEN_PATH = f"/{TENANT}/oauth2/v2.0/token"+QUERY_PATH = f"/v1/workspaces/{WORKSPACE}/query"+CONFIG = AzureApiManagementSourceConfig(+ tenant_id=TENANT, client_id="client", client_secret="secret", workspace_id=WORKSPACE+)+++def _token() -> ScriptedResponse:+ return ScriptedResponse(json={"token_type": "Bearer", "expires_in": 3599, "access_token": "token-1"})+++def _unresolved(table: str) -> ScriptedResponse:+ return ScriptedResponse(+ status=400,+ json={+ "error": {+ "code": "BadArgumentError",+ "message": "The request had some invalid properties",+ "innererror": {+ "code": "SemanticError",+ "message": f"'union' operator: Failed to resolve table or column expression named '{table}'",+ },+ }+ },+ )+++def _empty() -> ScriptedResponse:+ return ScriptedResponse(json={"tables": [{"name": "PrimaryResult", "columns": [], "rows": []}]})+++def _basic_plan() -> ScriptedResponse:+ return ScriptedResponse(+ status=400,+ json={"error": {"code": "BadArgumentError", "message": "The query can't run on a table with the Basic plan."}},+ )+++ ("gateway", "llm", "websocket", "expected_ok", "expected_probes"),+ [+ (+ _unresolved("ApiManagementGatewayLogs"),+ _unresolved("ApiManagementGatewayLlmLog"),+ _unresolved("ApiManagementWebSocketConnectionLogs"),+ False,+ 3,+ ),+ (_basic_plan(), _empty(), _unresolved("ApiManagementWebSocketConnectionLogs"), True, 2),+ ],+)+def test_creation_fails_only_when_the_workspace_has_none_of_the_tables(+ gateway: ScriptedResponse,+ llm: ScriptedResponse,+ websocket: ScriptedResponse,+ expected_ok: bool,+ expected_probes: int,+) -> None:+ with scripted_network(+ route({TOKEN_PATH: [_token(), _token()], QUERY_PATH: [_empty(), gateway, llm, websocket]})+ ) as network:+ ok, message = AzureApiManagementSource().validate_credentials(CONFIG, team_id=1)++ assert ok is expected_ok+ if expected_ok:+ assert message is None+ else:+ assert message is not None and "This workspace has no API Management logs" in message+ queries = [request.json()["query"] for request in network.requests_log if request.path == QUERY_PATH]+ assert queries[0] == "print ok = 1"+ assert [query.split(" ")[0] for query in queries[1:]] == [+ "ApiManagementGatewayLogs",+ "ApiManagementGatewayLlmLog",+ "ApiManagementWebSocketConnectionLogs",+ ][:expected_probes]+++def test_a_missing_named_table_points_at_its_log_category() -> None:+ with scripted_network(route({TOKEN_PATH: [_token()], QUERY_PATH: [_unresolved("ApiManagementGatewayLlmLog")]})):+ ok, message = AzureApiManagementSource().validate_credentials(+ CONFIG, team_id=1, schema_name="ApiManagementGatewayLlmLog"+ )++ assert ok is False+ assert message is not None and "no ApiManagementGatewayLlmLog logs yet. Turn on its log category" in message+++def test_the_schema_picker_names_the_log_category_for_a_missing_table() -> None:+ with scripted_network(+ route(+ {+ TOKEN_PATH: [_token()],+ QUERY_PATH: [_empty(), _unresolved("ApiManagementGatewayLlmLog")],+ }+ )+ ):+ permissions = AzureApiManagementSource().get_endpoint_permissions(+ CONFIG, team_id=1, endpoints=["ApiManagementGatewayLogs", "ApiManagementGatewayLlmLog"]+ )++ assert permissions["ApiManagementGatewayLogs"] is None+ assert permissions["ApiManagementGatewayLlmLog"] is not None+ assert "log category" in permissions["ApiManagementGatewayLlmLog"]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureapimanagement.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureapimanagement.pyindex e1e040f4cd1b..fb6b22289b70 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureapimanagement.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureapimanagement.py@@ -6,4 +6,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common imp@config.configclass AzureApiManagementSourceConfig(config.Config):- pass+ tenant_id: str+ client_id: str+ client_secret: str+ workspace_id: str