:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): implement azure log analytics source (#118158)
Daniel RC3d774c584d24
diff --git a/frontend/public/services/azure_log_analytics.png b/frontend/public/services/azure_log_analytics.pngindex c5736e43b50d..9d1f80398d88 100644Binary files a/frontend/public/services/azure_log_analytics.png and b/frontend/public/services/azure_log_analytics.png differdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.mdindex 1e2d572ccc7a..13a501fc371d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md@@ -140,6 +140,7 @@ the row lists both.| azure_blob | HTTP (vendor SDK) | azure-storage-blob (listing), adlfs (reads), deltalake | ⚠️ || azure_cost_management | HTTP | requests | ✅ || azure_devops | HTTP | requests | ✅ |+| azure_log_analytics | HTTP | requests | ✅ || babelforce | HTTP | requests | ✅ || back_market | HTTP | requests + `rest_source.RESTClient` | ✅ || bamboohr | HTTP | requests | ✅ |@@ -1068,7 +1069,6 @@ doesn't conflict with concurrent PRs.- azure_api_management- azure_data_explorer- azure_data_factory-- azure_log_analytics- azure_monitor_alerts- azure_monitor_metrics- azure_openai_usagediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/azure_log_analytics.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/azure_log_analytics.pynew file mode 100644index 000000000000..49d855de9bf9--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/azure_log_analytics.py@@ -0,0 +1,554 @@+import re+import json+import time+from collections.abc import Callable, Iterator+from dataclasses import field+from datetime import UTC, date, datetime, timedelta+from typing import Any, Optional+from urllib.parse import quote++import requests+import structlog+from structlog.types import FilteringBoundLogger+from urllib3.util.retry import Retry++from posthog.dataclasses import frozen++from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.settings import (+ API_HOST,+ DISCOVERY_LOOKBACK_DAYS,+ INGESTION_TIME_FIELD,+ ITEM_ID_FIELD,+ LOGIN_HOST,+ PAGE_SIZE,+ PRIMARY_KEYS,+ TABLE_NAME_RE,+ TIME_GENERATED_PRUNE,+ TIMESTAMP_FIELD,+ TOKEN_SCOPE,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (+ interruptible_wait,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse++# A query runs for up to three minutes server-side before Log Analytics cancels it.+REQUEST_TIMEOUT_SECONDS = (30, 200)+MAX_RETRY_ATTEMPTS = 5+MAX_BACKOFF_SECONDS = 60.0+# One request with its retries and waits holds the worker for five minutes at most (the source+# contract), so a retry only starts when a full-length attempt still fits in the budget.+RETRY_BUDGET_SECONDS = 300.0+# A page that hits a result-size or execution limit is read again with half as many rows.+MIN_PAGE_SIZE = 100+# Permission probes run inside the source setup request, so they are short, never retried, and+# stop once the total passes this budget.+PROBE_REQUEST_TIMEOUT_SECONDS = (10, 30)+PROBE_BUDGET_SECONDS = 60.0+# Creation and the schema picker each probe inside the same 120 s setup request.+CREATION_PROBE_BUDGET_SECONDS = 30.0++_GUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$")+# The cursor values are embedded in KQL, so they must keep the exact shape Log Analytics returns.+_DATETIME_LITERAL_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,7})?Z$")+_ITEM_ID_LITERAL_RE = re.compile(r"^[A-Za-z0-9-]{0,128}$")+# The ingestion time as KQL renders it (100 ns precision), so the cursor matches rows exactly.+CURSOR_COLUMN = "__posthog_cursor"+++class AzureLogAnalyticsRetryableError(Exception):+ """Throttled (429) or a gateway failure (5xx) that outlived the retry budget."""++ pass+++class AzureLogAnalyticsPartialResultError(ValueError):+ """The query hit a service limit and returned only part of its rows."""++ pass+++@frozen+class AzureLogAnalyticsResumeConfig:+ # Where the next page starts: after this ingestion time (exactly as KQL renders it, 100 ns+ # precision, which a Python datetime cannot hold) and this _ItemId. An empty item_id means+ # "at or after" the time, which starts a window or a watermark.+ ingested_at: str+ item_id: str = ""+++@frozen+class WorkspaceCredentials:+ tenant_id: str+ client_id: str+ client_secret: str = field(repr=False)+ workspace_id: str++ def client(+ self,+ logger: FilteringBoundLogger,+ safe_point: Callable[[], None] | None = None,+ probe: bool = False,+ ) -> "AzureLogAnalyticsClient":+ return AzureLogAnalyticsClient(+ self.tenant_id,+ self.client_id,+ self.client_secret,+ self.workspace_id,+ logger,+ safe_point=safe_point,+ request_timeout=PROBE_REQUEST_TIMEOUT_SECONDS if probe else REQUEST_TIMEOUT_SECONDS,+ max_attempts=1 if probe else MAX_RETRY_ATTEMPTS,+ )+++def validate_workspace_id(workspace_id: str) -> str:+ value = (workspace_id or "").strip()+ if not _GUID_RE.match(value):+ raise ValueError("Workspace ID must be the workspace GUID from the workspace's Overview page.")+ return value+++def parse_table_names(raw: str | None) -> list[str]:+ """One table per line (commas also accepted), deduplicated in order."""+ names: list[str] = []+ for part in re.split(r"[\n,]", raw or ""):+ name = part.strip()+ if not name:+ continue+ if not TABLE_NAME_RE.match(name):+ raise ValueError(f"{name!r} is not a valid Log Analytics table name.")+ if name not in names:+ names.append(name)+ return names+++def _error_description(response: requests.Response) -> str:+ # Log Analytics errors look like {"error": {"code": ..., "message": ..., "innererror": {...}}};+ # the inner error carries the KQL message that names the actual problem.+ try:+ body = response.json()+ except ValueError:+ return ""+ if not isinstance(body, dict):+ return ""+ error = body.get("error")+ if isinstance(error, dict):+ messages = [str(error["message"])] if error.get("message") else []+ inner = error.get("innererror")+ while isinstance(inner, dict):+ if inner.get("message"):+ messages.append(str(inner["message"]))+ inner = inner.get("innererror")+ if messages:+ return " — " + " ".join(messages)+ if isinstance(body.get("error_description"), str):+ return f" — {body['error_description']}"+ return ""+++def _backoff_seconds(response: requests.Response | None, attempt: int) -> float:+ raw = response.headers.get("Retry-After") if response is not None else None+ if raw is not None:+ try:+ return min(max(float(raw), 0.0), MAX_BACKOFF_SECONDS)+ except ValueError:+ pass+ return min(2.0**attempt, MAX_BACKOFF_SECONDS)+++class AzureLogAnalyticsClient:+ """Client-credentials client for the Log Analytics query API.++ Transport retries are off (`Retry(total=0)`): every query is a POST, which the default retry+ never covers, so throttling and gateway errors are retried here, within `RETRY_BUDGET_SECONDS`.+ """++ def __init__(+ self,+ tenant_id: str,+ client_id: str,+ client_secret: str,+ workspace_id: str,+ logger: FilteringBoundLogger,+ safe_point: Callable[[], None] | None = None,+ request_timeout: tuple[float, float] = REQUEST_TIMEOUT_SECONDS,+ max_attempts: int = MAX_RETRY_ATTEMPTS,+ ) -> None:+ self._request_timeout = request_timeout+ self._max_attempts = max_attempts+ self._tenant_id = tenant_id.strip()+ self._client_id = client_id.strip()+ self._client_secret = client_secret+ self._workspace_id = validate_workspace_id(workspace_id)+ self._logger = logger+ self._safe_point = safe_point+ self._token: Optional[str] = None+ redact = tuple(value for value in (client_secret,) if value)+ # The token exchange carries the secret and returns a bearer token; keep it out of samples.+ self._auth_session = make_tracked_session(retry=Retry(total=0), redact_values=redact, capture=False)+ self._api_session = make_tracked_session(retry=Retry(total=0), redact_values=redact)++ def mint_token(self) -> str:+ url = f"{LOGIN_HOST}/{quote(self._tenant_id, safe='')}/oauth2/v2.0/token"+ response = self._auth_session.post(+ url,+ data={+ "grant_type": "client_credentials",+ "client_id": self._client_id,+ "client_secret": self._client_secret,+ "scope": TOKEN_SCOPE,+ },+ timeout=self._request_timeout,+ )+ if not response.ok:+ raise requests.HTTPError(+ f"{response.status_code} Client Error: {response.reason} for url: {LOGIN_HOST}{_error_description(response)}",+ response=response,+ )+ token = response.json().get("access_token")+ if not token:+ raise requests.HTTPError(f"Azure AD returned no access token for url: {LOGIN_HOST}", response=response)+ self._token = token+ return token++ def query(self, kql: str) -> list[dict[str, Any]]:+ url = f"{API_HOST}/v1/workspaces/{self._workspace_id}/query"+ attempt = 0+ reminted = False+ started = time.monotonic()++ while True:+ token = self._token or self.mint_token()+ response: requests.Response | None+ try:+ response = self._api_session.post(+ url,+ json={"query": kql},+ headers={"Authorization": f"Bearer {token}"},+ timeout=self._request_timeout,+ )+ status = str(response.status_code)+ except (requests.ConnectionError, requests.Timeout) as error:+ # Queries only read, so a dropped connection or a timeout is safe to send again.+ response = None+ status = type(error).__name__++ # Tokens last about an hour; a long backfill can outlive one.+ if response is not None and response.status_code == 401 and not reminted:+ reminted = True+ self._token = None+ continue++ if response is None or response.status_code == 429 or response.status_code >= 500:+ attempt += 1+ delay = _backoff_seconds(response, attempt)+ elapsed = time.monotonic() - started+ if attempt >= self._max_attempts or elapsed + delay + sum(self._request_timeout) > RETRY_BUDGET_SECONDS:+ raise AzureLogAnalyticsRetryableError(f"Azure Log Analytics error (retryable): status={status}")+ self._logger.debug(f"Azure Log Analytics retry: status={status}, sleeping {delay}s (attempt {attempt})")+ interruptible_wait(delay, safe_point=self._safe_point)+ continue++ if not response.ok:+ raise requests.HTTPError(+ f"{response.status_code} Client Error: {response.reason} for url: {API_HOST}{_error_description(response)}",+ response=response,+ )++ return rows_from_response(response.json())+++def rows_from_response(payload: Any) -> list[dict[str, Any]]:+ """Zip the primary result table's `columns` + `rows` into typed row dicts."""+ if not isinstance(payload, dict):+ return []+ # A query that hit a service limit returns 200 with the rows it managed and an `error`;+ # syncing those as if complete would lose the rest silently.+ if isinstance(payload.get("error"), dict):+ error = payload["error"]+ raise AzureLogAnalyticsPartialResultError(+ f"Azure Log Analytics returned a partial result: {error.get('code')} {error.get('message')}"+ )+ tables = payload.get("tables") or []+ if not tables:+ return []+ table = tables[0]+ columns = [(str(column.get("name")), str(column.get("type", ""))) for column in table.get("columns") or []]+ return [+ {name: _convert(value, column_type) for (name, column_type), value in zip(columns, values)}+ for values in table.get("rows") or []+ ]+++def _convert(value: Any, column_type: str) -> Any:+ if value is None:+ return None+ if column_type == "datetime" and isinstance(value, str):+ try:+ return datetime.fromisoformat(value.replace("Z", "+00:00"))+ except ValueError:+ return value+ if column_type == "dynamic" and isinstance(value, str):+ try:+ return json.loads(value)+ except ValueError:+ return value+ if column_type == "bool" and isinstance(value, int | str):+ return str(value).lower() in ("1", "true")+ return value+++def _format_datetime_literal(value: Any) -> str:+ if isinstance(value, str):+ text = value.strip()+ if _DATETIME_LITERAL_RE.match(text):+ return text+ value = datetime.fromisoformat(text.replace("Z", "+00:00"))+ if isinstance(value, datetime):+ moment = value if value.tzinfo else value.replace(tzinfo=UTC)+ return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")+ if isinstance(value, date):+ return f"{value.isoformat()}T00:00:00Z"+ raise ValueError(f"Azure Log Analytics incremental value is not a datetime: {value!r}")+++def build_query(+ table: str,+ after: AzureLogAnalyticsResumeConfig,+ before: str,+ page_size: int,+) -> str:+ if not TABLE_NAME_RE.match(table):+ raise ValueError(f"{table!r} is not a valid Log Analytics table name.")+ for literal in (after.ingested_at, before):+ if not _DATETIME_LITERAL_RE.match(literal):+ raise ValueError("Azure Log Analytics cursor has an unexpected shape")+ if not _ITEM_ID_LITERAL_RE.match(after.item_id):+ raise ValueError("Azure Log Analytics cursor has an unexpected shape")++ moment = f"datetime({after.ingested_at})"+ lines = [table, f"| where {TIMESTAMP_FIELD} >= {moment} - {TIME_GENERATED_PRUNE}"]+ if after.item_id:+ lines.append(+ f"| where ingestion_time() > {moment} or "+ f"(ingestion_time() == {moment} and strcmp({ITEM_ID_FIELD}, '{after.item_id}') > 0)"+ )+ else:+ lines.append(f"| where ingestion_time() >= {moment}")+ lines.append(f"| where ingestion_time() < datetime({before})")+ # ingestion_time() only resolves against the table, so it is captured before the sort.+ lines.append(f"| extend {INGESTION_TIME_FIELD} = ingestion_time()")+ # _ItemId breaks ties, so a page that ends inside one ingestion time resumes without gaps.+ lines.append(f"| sort by {INGESTION_TIME_FIELD} asc, {ITEM_ID_FIELD} asc")+ lines.append(f"| take {page_size}")+ # Re-assigning _ItemId makes it a visible column in the result.+ lines.append(f"| extend {ITEM_ID_FIELD} = {ITEM_ID_FIELD}, {CURSOR_COLUMN} = tostring({INGESTION_TIME_FIELD})")+ return "\n".join(lines)+++def window_end(after: AzureLogAnalyticsResumeConfig, run_start: datetime, hourly: bool) -> str:+ """The end of the day or hour the cursor sits in, capped at the run start.++ Each page sorts only the rows left in its window, so a backfill never re-scans the whole table+ for every page. A busy table walks hours instead of days. The cap leaves rows ingested during+ the run to the next run.+ """+ moment = datetime.fromisoformat(after.ingested_at[:19]).replace(tzinfo=UTC)+ if hourly:+ end = moment.replace(minute=0, second=0) + timedelta(hours=1)+ else:+ end = moment.replace(hour=0, minute=0, second=0) + timedelta(days=1)+ if end >= run_start:+ return _format_datetime_literal(run_start)+ return end.strftime("%Y-%m-%dT%H:%M:%SZ")+++def _first_cursor(client: AzureLogAnalyticsClient, table: str, watermark: Any) -> AzureLogAnalyticsResumeConfig | None:+ if watermark is not None:+ return AzureLogAnalyticsResumeConfig(ingested_at=_format_datetime_literal(watermark))+ rows = client.query(f"{table} | summarize start = min(ingestion_time()) | project start = tostring(start)")+ start = str(rows[0].get("start") or "") if rows else ""+ return AzureLogAnalyticsResumeConfig(ingested_at=start) if start else None+++def get_rows(+ client: AzureLogAnalyticsClient,+ table: str,+ watermark: Any,+ resumable_source_manager: ResumableSourceManager[AzureLogAnalyticsResumeConfig],+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ if resume is not None:+ logger.debug(f"Azure Log Analytics: resuming {table} after {resume.ingested_at}")+ after = resume or _first_cursor(client, table, watermark)+ if after is None:+ return++ run_start = datetime.now(UTC)+ run_end = _format_datetime_literal(run_start)+ page_size = PAGE_SIZE+ hourly = False+ while True:+ before = window_end(after, run_start, hourly)+ try:+ rows = client.query(build_query(table, after, before, page_size))+ except AzureLogAnalyticsPartialResultError:+ if page_size <= MIN_PAGE_SIZE:+ raise+ hourly = True+ page_size = max(MIN_PAGE_SIZE, page_size // 2)+ logger.debug(f"Azure Log Analytics: {table} page hit a service limit, retrying with {page_size} rows")+ # The staged cursor covers every yielded row, so the run can hand over between retries.+ resumable_source_manager.safe_point()+ continue++ window_done = len(rows) < page_size+ if not window_done:+ hourly = True+ last = rows[-1]+ after = AzureLogAnalyticsResumeConfig(+ ingested_at=str(last.get(CURSOR_COLUMN)), item_id=str(last.get(ITEM_ID_FIELD))+ )+ else:+ page_size = PAGE_SIZE+ hourly = hourly and not before.endswith("T00:00:00Z")+ after = AzureLogAnalyticsResumeConfig(ingested_at=before)+ for row in rows:+ row.pop(CURSOR_COLUMN, None)++ # Staged before the yield it covers, the final page included: the pipeline commits it once+ # the page is written.+ resumable_source_manager.save_state(after)+ if rows:+ yield rows+ else:+ # A window with no rows yields nothing; the pipeline only sees a shutdown at an item or here.+ resumable_source_manager.safe_point()+ if window_done and before == run_end:+ return+++def discover_tables(client: AzureLogAnalyticsClient) -> list[str]:+ rows = client.query(+ f"Usage | where TimeGenerated > ago({DISCOVERY_LOOKBACK_DAYS}d) | distinct DataType | sort by DataType asc"+ )+ return [str(row["DataType"]) for row in rows if TABLE_NAME_RE.match(str(row.get("DataType") or ""))]+++def _is_missing_table(error: requests.HTTPError) -> bool:+ return getattr(error.response, "status_code", None) == 400 and "Failed to resolve table" in str(error)+++def probe_tables(+ credentials: WorkspaceCredentials,+ tables: list[str],+ missing_table_reason: str,+ stop_at_first_readable: bool = False,+ budget_seconds: float = PROBE_BUDGET_SECONDS,+) -> dict[str, str | None]:+ """Why the query API cannot read each table, or None when it can.++ Basic and Auxiliary plan tables are only readable through the search API, so the query API+ rejects them with a 400, and table-level access control answers 403. A throttle or an outage+ is not a reason: the table counts as readable, and probing stops so setup stays fast.+ """+ reasons: dict[str, str | None] = dict.fromkeys(tables)+ client = credentials.client(structlog.get_logger(__name__), probe=True)+ try:+ client.mint_token()+ except (requests.RequestException, ValueError):+ return reasons+ started = time.monotonic()+ for table in tables:+ if time.monotonic() - started > budget_seconds:+ break+ try:+ client.query(f"{table} | take 0")+ if stop_at_first_readable:+ break+ except requests.HTTPError as error:+ status = getattr(error.response, "status_code", None)+ if _is_missing_table(error):+ reasons[table] = missing_table_reason+ elif status == 400:+ reasons[table] = (+ "Log Analytics can't query this table. Basic and Auxiliary plan tables aren't supported."+ f"{_error_description(error.response)}"+ )+ elif status == 403:+ reasons[table] = "The app can't read this table. Give it access to the table, or leave the table out."+ else:+ break+ except (AzureLogAnalyticsRetryableError, requests.RequestException):+ break+ return reasons+++def _probe_failure_message(error: Exception, missing_table_message: str) -> str:+ status = getattr(getattr(error, "response", None), "status_code", None)+ if status == 403:+ return (+ "The app registration cannot query this workspace. Give its service principal the Log Analytics "+ "Reader role on the workspace, or read access to the table, then try again."+ )+ if status == 404:+ return "No Log Analytics workspace has this workspace ID. Copy the workspace ID from the workspace's Overview page."+ if isinstance(error, requests.HTTPError) and _is_missing_table(error):+ return missing_table_message+ return f"Could not query the Log Analytics workspace: {error}"+++def validate_workspace(+ credentials: WorkspaceCredentials, probe_query: str, missing_table_message: str+) -> tuple[bool, Optional[str]]:+ """Mint a token, then run `probe_query`, which proves the app can query the workspace.++ This runs inside the source setup request, so it uses the short probe deadlines.+ """+ try:+ client = credentials.client(structlog.get_logger(__name__), probe=True)+ except ValueError as error:+ return False, str(error)++ try:+ client.mint_token()+ except Exception:+ return (+ False,+ "Could not authenticate with Microsoft Entra ID. Check the tenant ID, client ID and client secret.",+ )++ try:+ client.query(probe_query)+ except Exception as error:+ return False, _probe_failure_message(error, missing_table_message)++ return True, None+++def azure_log_analytics_source(+ credentials: WorkspaceCredentials,+ table: str,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[AzureLogAnalyticsResumeConfig],+ db_incremental_field_last_value: Any = None,+) -> SourceResponse:+ client = credentials.client(logger, safe_point=resumable_source_manager.safe_point)++ return SourceResponse(+ name=table,+ items=lambda: get_rows(client, table, db_incremental_field_last_value, resumable_source_manager, logger),+ primary_keys=PRIMARY_KEYS,+ partition_count=1,+ partition_size=1,+ partition_keys=[TIMESTAMP_FIELD],+ partition_mode="datetime",+ partition_format="day",+ # Every page is one query sorted by ingestion time, then _ItemId.+ sort_mode="asc",+ on_complete=resumable_source_manager.clear_state,+ )diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/settings.pynew file mode 100644index 000000000000..cd2ca8f6fbea--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/settings.py@@ -0,0 +1,37 @@+import re++from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType++API_HOST = "https://api.loganalytics.azure.com"+# api.loganalytics.azure.com still issues tokens for the original resource URI.+TOKEN_SCOPE = "https://api.loganalytics.io/.default"+LOGIN_HOST = "https://login.microsoftonline.com"++# A query returns at most 500,000 rows and ~100 MB; log rows can be wide, so pages stay small.+PAGE_SIZE = 5000+# Every Log Analytics table carries these two standard columns.+TIMESTAMP_FIELD = "TimeGenerated"+ITEM_ID_FIELD = "_ItemId"+PRIMARY_KEYS = [ITEM_ID_FIELD]+# Reads page on ingestion time: an agent that was offline delivers records hours after their TimeGenerated.+INGESTION_TIME_FIELD = "_IngestionTime"+# ingestion_time() is approximate and concurrent ingestions land out of order.+INGESTION_LOOKBACK_SECONDS = 15 * 60+# Azure replaces a TimeGenerated more than two days older than its arrival, so this bound loses no row.+TIME_GENERATED_PRUNE = "3d"+# Discovery reads table names from the Usage table, which every workspace has.+DISCOVERY_LOOKBACK_DAYS = 30++# Table names are interpolated into KQL, so only plain identifiers (including `_CL` custom tables)+# are accepted.+TABLE_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,254}$")++INCREMENTAL_FIELDS: list[IncrementalField] = [+ {+ "label": INGESTION_TIME_FIELD,+ "type": IncrementalFieldType.DateTime,+ "field": INGESTION_TIME_FIELD,+ "field_type": IncrementalFieldType.DateTime,+ "is_indexed": True,+ }+]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/source.pyindex 5354653fc7af..7da152d0d1df 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/source.py@@ -1,16 +1,228 @@-from typing import cast+from abc import abstractmethod+from typing import Generic, Optional, cast-from products.warehouse_sources.backend.facade.source_config import DataWarehouseSourceCategory, SourceConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource+import structlog++from products.warehouse_sources.backend.facade.codebase_detection import detect+from products.warehouse_sources.backend.facade.source_config import (+ DataWarehouseSourceCategory,+ ReleaseStatus,+ SourceConfig,+ SourceFieldInputConfig,+ SourceFieldInputConfigType,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.azure_log_analytics import (+ CREATION_PROBE_BUDGET_SECONDS,+ AzureLogAnalyticsResumeConfig,+ WorkspaceCredentials,+ azure_log_analytics_source,+ discover_tables,+ parse_table_names,+ probe_tables,+ validate_workspace,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.settings import (+ INCREMENTAL_FIELDS,+ INGESTION_LOOKBACK_SECONDS,+ TABLE_NAME_RE,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (+ ConfigType,+ FieldType,+ ResumableSource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.azureloganalytics import (AzureLogAnalyticsSourceConfig,)from products.warehouse_sources.backend.types import ExternalDataSourceType+WORKSPACE_FIELDS: list[FieldType] = [+ SourceFieldInputConfig(+ name="tenant_id",+ label="Directory (tenant) ID",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="00000000-0000-0000-0000-000000000000",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="client_id",+ label="Application (client) ID",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="00000000-0000-0000-0000-000000000000",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="client_secret",+ label="Client secret",+ type=SourceFieldInputConfigType.PASSWORD,+ required=True,+ placeholder="",+ secret=True,+ ),+ SourceFieldInputConfig(+ name="workspace_id",+ label="Workspace ID",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="00000000-0000-0000-0000-000000000000",+ secret=False,+ ),+]+++class LogAnalyticsWorkspaceSource(ResumableSource[ConfigType, AzureLogAnalyticsResumeConfig], Generic[ConfigType]):+ """A source that syncs tables of one Log Analytics workspace. Subclasses pick the tables."""++ missing_table_reason = (+ "This table doesn't exist in the workspace. Check the table name, or remove it from the list."+ )+ missing_table_message = (+ "The workspace has no table named {table}. Check the table name, or remove it from the list."+ )+ no_tables_message = "None of the listed tables exist in the workspace. Check the table names."++ @abstractmethod+ def credentials(self, config: ConfigType) -> WorkspaceCredentials: ...++ @abstractmethod+ def table_names(self, config: ConfigType) -> list[str]: ...++ def tables_checked_at_creation(self, config: ConfigType) -> list[str]:+ """Tables probed at source creation; creation fails when every one of them is missing."""+ return []++ @property+ def connection_host_fields(self) -> list[str]:+ # `tenant_id` picks the directory the stored secret is exchanged in, and `workspace_id` picks+ # the workspace the token is spent on. Retargeting either must re-require the secret.+ return ["tenant_id", "workspace_id"]++ def get_non_retryable_errors(self) -> dict[str, str | None]:+ return {+ "400 Client Error: Bad Request for url: https://login.microsoftonline.com": "Microsoft Entra ID rejected the app credentials. Check the tenant ID, client ID and client secret, then reconnect.",+ "401 Client Error: Unauthorized for url: https://login.microsoftonline.com": "Microsoft Entra ID rejected the app credentials. Check the client ID and client secret, then reconnect.",+ "403 Client Error: Forbidden for url: https://login.microsoftonline.com": "Microsoft Entra ID refused to issue a token for the app. Check that the app registration can sign in to this tenant, then reconnect.",+ "401 Client Error: Unauthorized for url: https://api.loganalytics.azure.com": "Log Analytics rejected the access token. Check that the app registration is still enabled, then reconnect.",+ "403 Client Error: Forbidden for url: https://api.loganalytics.azure.com": "The app cannot query this workspace. Give its service principal the Log Analytics Reader role on the workspace, or read access to the table, then reconnect.",+ "404 Client Error: Not Found for url: https://api.loganalytics.azure.com": "No Log Analytics workspace has this workspace ID. Check the workspace ID, then reconnect.",+ "Failed to resolve table": self.missing_table_reason,+ }++ def get_retryable_errors(self) -> set[str]:+ return {"Azure Log Analytics error (retryable)"}++ def get_schemas(+ self,+ config: ConfigType,+ team_id: int,+ with_counts: bool = False,+ names: list[str] | None = None,+ force_refresh: bool = False,+ api_version: str | None = None,+ ) -> list[SourceSchema]:+ schemas = [+ SourceSchema(+ name=table_name,+ supports_incremental=True,+ # The watermark re-reads a short overlap, which only merge dedupes.+ supports_append=False,+ incremental_fields=INCREMENTAL_FIELDS,+ default_incremental_lookback_seconds=INGESTION_LOOKBACK_SECONDS,+ )+ for table_name in self.table_names(config)+ ]+ if names is not None:+ names_set = set(names)+ schemas = [s for s in schemas if s.name in names_set]+ return schemas++ def get_endpoint_permissions(+ self,+ config: ConfigType,+ team_id: int,+ endpoints: list[str],+ api_version: str | None = None,+ ) -> dict[str, str | None]:+ return probe_tables(self.credentials(config), endpoints, self.missing_table_reason)++ def validate_credentials(+ self,+ config: ConfigType,+ team_id: int,+ schema_name: Optional[str] = None,+ api_version: str | None = None,+ ) -> tuple[bool, str | None]:+ if schema_name is not None:+ if not TABLE_NAME_RE.match(schema_name):+ return False, f"{schema_name!r} is not a valid Log Analytics table name."+ return validate_workspace(+ self.credentials(config),+ f"{schema_name} | take 0",+ self.missing_table_message.format(table=schema_name),+ )+ try:+ tables = self.tables_checked_at_creation(config)+ except ValueError as error:+ return False, str(error)+ valid, message = validate_workspace(self.credentials(config), "print ok = 1", self.no_tables_message)+ if not valid or not tables:+ return valid, message+ # One query per table, stopping at the first readable one: a Basic plan table fails any+ # query that names it, and must not block the rest.+ reasons = probe_tables(+ self.credentials(config),+ tables,+ self.missing_table_reason,+ stop_at_first_readable=True,+ budget_seconds=CREATION_PROBE_BUDGET_SECONDS,+ )+ if any(reason is None for reason in reasons.values()):+ return True, None+ if all(reason == self.missing_table_reason for reason in reasons.values()):+ return False, self.no_tables_message+ first_reason = next(reason for reason in reasons.values() if reason != self.missing_table_reason)+ return False, f"None of the listed tables can be read. {first_reason}"++ def get_resumable_source_manager(+ self, inputs: SourceInputs+ ) -> ResumableSourceManager[AzureLogAnalyticsResumeConfig]:+ return ResumableSourceManager[AzureLogAnalyticsResumeConfig](inputs, AzureLogAnalyticsResumeConfig)++ def source_for_pipeline(+ self,+ config: ConfigType,+ resumable_source_manager: ResumableSourceManager[AzureLogAnalyticsResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:+ return azure_log_analytics_source(+ credentials=self.credentials(config),+ table=inputs.schema_name,+ logger=inputs.logger,+ resumable_source_manager=resumable_source_manager,+ db_incremental_field_last_value=inputs.db_incremental_field_last_value+ if inputs.should_use_incremental_field+ else None,+ )+@SourceRegistry.register-class AzureLogAnalyticsSource(SimpleSource[AzureLogAnalyticsSourceConfig]):+class AzureLogAnalyticsSource(LogAnalyticsWorkspaceSource[AzureLogAnalyticsSourceConfig]):+ codebase_detection = detect(+ env=("^(AZURE_LOG_ANALYTICS|LOG_ANALYTICS)_",),+ npm=("@azure/monitor-query", "@azure/monitor-query-logs"),+ pypi=("azure-monitor-query",),+ go=("github.com/Azure/azure-sdk-for-go/sdk/monitor/query/azlogs",),+ maven=("com.azure:azure-monitor-query",),+ )++ api_docs_url = "https://learn.microsoft.com/en-us/azure/azure-monitor/logs/api/overview"+@propertydef source_type(self) -> ExternalDataSourceType:return ExternalDataSourceType.AZURELOGANALYTICS@@ -21,7 +233,44 @@ class AzureLogAnalyticsSource(SimpleSource[AzureLogAnalyticsSourceConfig]):name=ExternalDataSourceType.AZURELOGANALYTICS,category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,label="Microsoft Azure (Azure Monitor / Log Analytics)",+ keywords=["azure monitor", "log analytics", "kql"],+ releaseStatus=ReleaseStatus.ALPHA,+ caption="""Sync tables from an Azure Monitor Log Analytics workspace into the PostHog Data warehouse. Each table becomes a warehouse table, synced incrementally on the time Azure ingested each record, so records that arrive late still sync.++Register an app in Microsoft Entra ID, create a client secret for it, and give its service principal the **Log Analytics Reader** role on the workspace. Then enter the directory (tenant) ID, the application (client) ID, the secret value, and the workspace ID from the workspace's **Overview** page.++List the tables to sync, one per line, for example `AppRequests` or `MyEvents_CL`. Leave the list empty to offer every table that received data in the last 30 days.""",iconPath="/static/services/azure_log_analytics.png",- fields=cast(list[FieldType], []),- unreleasedSource=True,+ docsUrl="https://posthog.com/docs/cdp/sources/azure-log-analytics",+ fields=cast(+ list[FieldType],+ [+ *WORKSPACE_FIELDS,+ SourceFieldInputConfig(+ name="tables",+ label="Tables",+ type=SourceFieldInputConfigType.TEXTAREA,+ required=False,+ placeholder="AppRequests\nAppExceptions\nMyEvents_CL",+ secret=False,+ ),+ ],+ ),+ )++ def credentials(self, config: AzureLogAnalyticsSourceConfig) -> WorkspaceCredentials:+ return WorkspaceCredentials(+ tenant_id=config.tenant_id,+ client_id=config.client_id,+ client_secret=config.client_secret,+ workspace_id=config.workspace_id,+ )++ def table_names(self, config: AzureLogAnalyticsSourceConfig) -> list[str]:+ # Discovery runs inside the source setup request, so it uses the short probe deadlines.+ return parse_table_names(config.tables) or discover_tables(+ self.credentials(config).client(structlog.get_logger(__name__), probe=True))++ def tables_checked_at_creation(self, config: AzureLogAnalyticsSourceConfig) -> list[str]:+ return parse_table_names(config.tables)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/tests/test_azure_log_analytics.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/tests/test_azure_log_analytics.pynew file mode 100644index 000000000000..56e85ae4c988--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/tests/test_azure_log_analytics.py@@ -0,0 +1,632 @@+from collections.abc import Iterator+from datetime import UTC, datetime+from typing import Any++import pytest+import time_machine++from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics import azure_log_analytics+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.azure_log_analytics import (+ AzureLogAnalyticsResumeConfig,+ AzureLogAnalyticsRetryableError,+ build_query,+ parse_table_names,+ rows_from_response,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.source import (+ AzureLogAnalyticsSource,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ RecordedRequest,+ ScriptedResponse,+ SourceDriver,+ route,+ scripted_network,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.azureloganalytics import (+ AzureLogAnalyticsSourceConfig,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.tests.contract_harness import check_extraction++TENANT = "11111111-1111-1111-1111-111111111111"+WORKSPACE = "22222222-2222-2222-2222-222222222222"+TOKEN_PATH = f"/{TENANT}/oauth2/v2.0/token"+QUERY_PATH = f"/v1/workspaces/{WORKSPACE}/query"+COLUMNS = [+ {"name": "TimeGenerated", "type": "datetime"},+ {"name": "_IngestionTime", "type": "datetime"},+ {"name": "Message", "type": "string"},+ {"name": "Properties", "type": "dynamic"},+ {"name": "_ItemId", "type": "string"},+ {"name": "__posthog_cursor", "type": "string"},+]+++def _config(tables: str | None = "AppTraces") -> AzureLogAnalyticsSourceConfig:+ return AzureLogAnalyticsSourceConfig(+ tenant_id=TENANT, client_id="client", client_secret="secret", workspace_id=WORKSPACE, tables=tables+ )+++def _token() -> ScriptedResponse:+ return ScriptedResponse(json={"token_type": "Bearer", "expires_in": 3599, "access_token": "token-1"})+++def _row(ingested_at: str, item_id: str) -> list[Any]:+ return [ingested_at, ingested_at, f"message {item_id}", '{"level": "info"}', item_id, ingested_at]+++def _missing(table: str) -> ScriptedResponse:+ return ScriptedResponse(+ status=400,+ json={+ "error": {+ "code": "BadArgumentError",+ "message": "The request had some invalid properties",+ "innererror": {+ "code": "SemanticError",+ "message": f"'take' operator: Failed to resolve table or column expression named '{table}'",+ },+ }+ },+ )+++def _page(rows: list[list[Any]]) -> ScriptedResponse:+ return ScriptedResponse(json={"tables": [{"name": "PrimaryResult", "columns": COLUMNS, "rows": rows}]})+++def _queries(requests: list[RecordedRequest]) -> list[str]:+ return [request.json()["query"] for request in requests if request.path == QUERY_PATH]+++NOW = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)+RUN_END = "2026-10-02T12:00:00.000000Z"+WATERMARK_TODAY = {"incremental_field": "_IngestionTime", "db_incremental_field_last_value": "2026-10-02T09:30:00Z"}+MIN_QUERY = "AppTraces | summarize start = min(ingestion_time()) | project start = tostring(start)"+++def _min_start(start: str) -> ScriptedResponse:+ return ScriptedResponse(+ json={+ "tables": [{"name": "PrimaryResult", "columns": [{"name": "start", "type": "string"}], "rows": [[start]]}]+ }+ )+++def _where_lines(query: str) -> list[str]:+ return [line for line in query.splitlines() if line.startswith("| where")]+++def _window_end(query: str) -> str:+ return next(line for line in query.splitlines() if line.startswith("| where ingestion_time() < "))[+ len("| where ingestion_time() < datetime(") : -1+ ]+++def _take(query: str) -> str:+ return next(line for line in query.splitlines() if line.startswith("| take "))+++class TestQueryWalk:+ @pytest.fixture(autouse=True)+ def _frozen_clock(self) -> Iterator[None]:+ with time_machine.travel(NOW, tick=False):+ yield++ def test_full_refresh_walks_an_exact_cursor_and_stages_every_page(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(azure_log_analytics, "PAGE_SIZE", 2)+ # Both rows of the first page share one ingestion time, so only _ItemId separates them.+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(+ "AppTraces",+ route(+ {+ TOKEN_PATH: [_token()],+ QUERY_PATH: [+ _min_start("2026-10-01T23:00:00.1234567Z"),+ _page([_row("2026-10-01T23:00:00.1234567Z", "a"), _row("2026-10-01T23:00:00.1234567Z", "b")]),+ _page([_row("2026-10-01T23:59:59.9999999Z", "c")]),+ _page([_row("2026-10-02T08:00:00Z", "d")]),+ ],+ }+ ),+ )++ assert result.raised is None+ queries = _queries(result.requests)+ assert queries[0] == MIN_QUERY+ assert [_where_lines(query) for query in queries[1:]] == [+ [+ "| where TimeGenerated >= datetime(2026-10-01T23:00:00.1234567Z) - 3d",+ "| where ingestion_time() >= datetime(2026-10-01T23:00:00.1234567Z)",+ "| where ingestion_time() < datetime(2026-10-02T00:00:00Z)",+ ],+ [+ "| where TimeGenerated >= datetime(2026-10-01T23:00:00.1234567Z) - 3d",+ "| where ingestion_time() > datetime(2026-10-01T23:00:00.1234567Z) or "+ "(ingestion_time() == datetime(2026-10-01T23:00:00.1234567Z) and strcmp(_ItemId, 'b') > 0)",+ "| where ingestion_time() < datetime(2026-10-02T00:00:00Z)",+ ],+ [+ "| where TimeGenerated >= datetime(2026-10-02T00:00:00Z) - 3d",+ "| where ingestion_time() >= datetime(2026-10-02T00:00:00Z)",+ f"| where ingestion_time() < datetime({RUN_END})",+ ],+ ]+ assert all(+ "| sort by _IngestionTime asc, _ItemId asc\n| take 2\n| extend _ItemId = _ItemId" in query+ for query in queries[1:]+ )+ assert [row["_ItemId"] for row in result.rows] == ["a", "b", "c", "d"]+ assert all("__posthog_cursor" not in row for row in result.rows)+ assert result.saved_states == [+ AzureLogAnalyticsResumeConfig(ingested_at="2026-10-01T23:00:00.1234567Z", item_id="b"),+ AzureLogAnalyticsResumeConfig(ingested_at="2026-10-02T00:00:00Z"),+ AzureLogAnalyticsResumeConfig(ingested_at=RUN_END),+ ]+ assert result.committed_states == result.saved_states+ assert result.completed_state is None++ def test_a_full_page_narrows_the_rest_of_that_day_to_hour_windows(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(azure_log_analytics, "PAGE_SIZE", 2)+ answers = iter([_page([_row("2026-10-01T22:40:00Z", "a"), _row("2026-10-01T22:41:00Z", "b")])])++ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == TOKEN_PATH:+ return _token()+ return next(answers, _page([]))++ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(+ "AppTraces",+ respond,+ incremental_field="_IngestionTime",+ db_incremental_field_last_value=datetime(2026, 10, 1, 22, 30, tzinfo=UTC),+ )++ assert result.raised is None+ assert [_window_end(query) for query in _queries(result.requests)] == [+ "2026-10-02T00:00:00Z",+ "2026-10-01T23:00:00Z",+ "2026-10-02T00:00:00Z",+ RUN_END,+ ]++ def test_empty_days_advance_the_window_without_rows(self) -> None:+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(+ "AppTraces",+ route(+ {+ TOKEN_PATH: [_token()],+ QUERY_PATH: [_page([]), _page([]), _page([_row("2026-10-02T01:00:00Z", "z")])],+ }+ ),+ incremental_field="_IngestionTime",+ db_incremental_field_last_value=datetime(2026, 9, 30, 23, 0, tzinfo=UTC),+ )++ assert result.raised is None+ assert [_window_end(query) for query in _queries(result.requests)] == [+ "2026-10-01T00:00:00Z",+ "2026-10-02T00:00:00Z",+ RUN_END,+ ]+ assert _where_lines(_queries(result.requests)[0])[:2] == [+ "| where TimeGenerated >= datetime(2026-09-30T23:00:00.000000Z) - 3d",+ "| where ingestion_time() >= datetime(2026-09-30T23:00:00.000000Z)",+ ]+ assert [row["_ItemId"] for row in result.rows] == ["z"]++ @pytest.mark.parametrize(+ "inputs",+ [+ {},+ {+ "incremental_field": "_IngestionTime",+ "should_use_incremental_field": False,+ "db_incremental_field_last_value": datetime(2026, 10, 2, tzinfo=UTC),+ },+ ],+ )+ def test_full_refresh_starts_at_the_oldest_row_and_stops_on_an_empty_table(self, inputs: dict[str, Any]) -> None:+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(+ "AppTraces", route({TOKEN_PATH: [_token()], QUERY_PATH: [_min_start("")]}), **inputs+ )++ assert result.raised is None+ assert _queries(result.requests) == [MIN_QUERY]+ assert result.rows == []++ def test_resume_starts_after_the_saved_row_and_ignores_the_watermark(self) -> None:+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(+ "AppTraces",+ route({TOKEN_PATH: [_token()], QUERY_PATH: [_page([_row("2026-10-02T11:00:00Z", "z")])]}),+ resume_state=AzureLogAnalyticsResumeConfig(ingested_at="2026-10-02T10:59:59.9999999Z", item_id="y"),+ incremental_field="_IngestionTime",+ db_incremental_field_last_value=datetime(2026, 10, 1, tzinfo=UTC),+ )++ assert result.raised is None+ (query,) = _queries(result.requests)+ assert "strcmp(_ItemId, 'y') > 0" in query+ assert "ingestion_time() >= datetime(" not in query++ def test_a_page_over_the_size_limit_is_read_again_smaller_then_the_size_resets(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setattr(azure_log_analytics, "PAGE_SIZE", 400)+ too_large = ScriptedResponse(+ json={+ "tables": [{"name": "PrimaryResult", "columns": COLUMNS, "rows": [_row("2026-10-01T22:10:00Z", "a")]}],+ "error": {+ "code": "PartialError",+ "message": "Query result set has exceeded the internal data size limit",+ },+ }+ )+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(+ "AppTraces",+ route(+ {+ TOKEN_PATH: [_token()],+ QUERY_PATH: [too_large, _page([_row("2026-10-01T22:10:00Z", "a")]), _page([]), _page([])],+ }+ ),+ incremental_field="_IngestionTime",+ db_incremental_field_last_value="2026-10-01T22:00:00Z",+ )++ assert result.raised is None+ queries = _queries(result.requests)+ assert [_take(query) for query in queries] == ["| take 400", "| take 200", "| take 400", "| take 400"]+ assert [_window_end(query) for query in queries] == [+ "2026-10-02T00:00:00Z",+ "2026-10-01T23:00:00Z",+ "2026-10-02T00:00:00Z",+ RUN_END,+ ]+ assert [row["_ItemId"] for row in result.rows] == ["a"]++ def test_a_partial_result_at_the_smallest_page_fails_the_run(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(azure_log_analytics, "PAGE_SIZE", 100)+ partial = ScriptedResponse(+ json={+ "tables": [{"name": "PrimaryResult", "columns": COLUMNS, "rows": [_row("2026-10-02T10:00:00Z", "a")]}],+ "error": {"code": "PartialError", "message": "There were some errors when processing your query."},+ }+ )+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(+ "AppTraces", route({TOKEN_PATH: [_token()], QUERY_PATH: [partial]}), **WATERMARK_TODAY+ )++ assert result.rows == []+ assert result.raised is not None and "PartialError" in str(result.raised)++ def test_expired_token_is_reminted_once(self) -> None:+ expired = ScriptedResponse(+ status=401, json={"error": {"code": "TokenExpired", "message": "The provided authentication has expired"}}+ )+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(+ "AppTraces",+ route(+ {TOKEN_PATH: [_token(), _token()], QUERY_PATH: [expired, _page([_row("2026-10-02T10:00:00Z", "a")])]}+ ),+ **WATERMARK_TODAY,+ )++ assert result.raised is None+ assert result.paths == [TOKEN_PATH, QUERY_PATH, TOKEN_PATH, QUERY_PATH]+ assert len(result.rows) == 1++ def test_a_dropped_connection_is_retried_within_the_budget(self) -> None:+ answers: Iterator[ScriptedResponse | BaseException] = iter(+ [ConnectionResetError(), _page([_row("2026-10-02T10:00:00Z", "a")])]+ )++ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == TOKEN_PATH:+ return _token()+ answer = next(answers)+ if isinstance(answer, BaseException):+ raise answer+ return answer++ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run("AppTraces", respond, **WATERMARK_TODAY)++ assert result.raised is None+ assert len(_queries(result.requests)) == 2+ assert [row["_ItemId"] for row in result.rows] == ["a"]++ def test_retries_stop_before_a_full_attempt_would_overrun_five_minutes(self) -> None:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == TOKEN_PATH:+ return _token()+ return ScriptedResponse(+ status=504, json={"error": {"code": "GatewayTimeout"}}, headers={"Retry-After": "30"}+ )++ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run("AppTraces", respond, **WATERMARK_TODAY)++ assert isinstance(result.raised, AzureLogAnalyticsRetryableError)+ # Waits at 0 s and 30 s leave room for a full 230 s attempt; a third retry at 60 s does not.+ assert len(_queries(result.requests)) == 3++ def test_meets_the_source_contract_on_an_incremental_table(self) -> None:+ def start(inputs: SourceInputs) -> tuple[SourceResponse, ResumableSourceManager[Any]]:+ inputs.incremental_field = "_IngestionTime"+ inputs.should_use_incremental_field = True+ inputs.db_incremental_field_last_value = datetime(2026, 9, 20, tzinfo=UTC)+ source = AzureLogAnalyticsSource()+ manager = source.get_resumable_source_manager(inputs)+ return source.source_for_pipeline(_config(), manager, inputs), manager++ verdict = check_extraction(start, ["AppTraces"])++ assert verdict.failed == frozenset()+ assert verdict.safe_point_exercised+++class TestRowShaping:+ @pytest.mark.parametrize(+ ("column_type", "value", "expected"),+ [+ ("datetime", "2026-10-01T10:00:00.1234567Z", datetime(2026, 10, 1, 10, 0, 0, 123456, tzinfo=UTC)),+ ("dynamic", '{"a": [1, 2]}', {"a": [1, 2]}),+ ("dynamic", "not json", "not json"),+ ("bool", "True", True),+ ("bool", 0, False),+ ("long", 7, 7),+ ("string", None, None),+ ],+ )+ def test_converts_values_by_column_type(self, column_type: str, value: Any, expected: Any) -> None:+ payload = {"tables": [{"columns": [{"name": "c", "type": column_type}], "rows": [[value]]}]}++ assert rows_from_response(payload) == [{"c": expected}]+++class TestInputs:+ @pytest.mark.parametrize(+ ("raw", "expected"),+ [+ ("AppTraces\n\nMyEvents_CL\nAppTraces", ["AppTraces", "MyEvents_CL"]),+ ("AppTraces, AppRequests", ["AppTraces", "AppRequests"]),+ (None, []),+ ],+ )+ def test_parses_one_table_per_line(self, raw: str | None, expected: list[str]) -> None:+ assert parse_table_names(raw) == expected++ @pytest.mark.parametrize("raw", ["AppTraces | union SecurityEvent", "App-Traces", "1Table"])+ def test_rejects_table_names_that_are_not_plain_identifiers(self, raw: str) -> None:+ with pytest.raises(ValueError):+ parse_table_names(raw)++ @pytest.mark.parametrize(+ "resume",+ [+ AzureLogAnalyticsResumeConfig(ingested_at="2026-10-01) | union X", item_id="a"),+ AzureLogAnalyticsResumeConfig(ingested_at="2026-10-01T10:00:00Z", item_id="a') or true or ('"),+ ],+ )+ def test_rejects_a_resume_cursor_that_would_change_the_query(self, resume: AzureLogAnalyticsResumeConfig) -> None:+ with pytest.raises(ValueError):+ build_query("AppTraces", resume, "2026-10-02T00:00:00Z", 10)+++class TestValidateCredentials:+ @pytest.mark.parametrize(+ ("query_response", "table", "expected_ok", "expected_fragment"),+ [+ (_page([]), None, True, None),+ (+ ScriptedResponse(+ status=403,+ json={+ "error": {+ "code": "InsufficientAccessError",+ "message": "The provided credentials have insufficient access to perform the requested operation",+ }+ },+ ),+ None,+ False,+ "Log Analytics Reader",+ ),+ (+ ScriptedResponse(+ status=404,+ json={"error": {"code": "WorkspaceNotFoundError", "message": "Workspace could not be found"}},+ ),+ None,+ False,+ "No Log Analytics workspace",+ ),+ (+ ScriptedResponse(+ status=400,+ json={+ "error": {+ "code": "BadArgumentError",+ "message": "The request had some invalid properties",+ "innererror": {+ "code": "SemanticError",+ "message": "A semantic error occurred.",+ "innererror": {+ "code": "SEM0100",+ "message": "'take' operator: Failed to resolve table or column expression named 'NoSuchTable'",+ },+ },+ }+ },+ ),+ "NoSuchTable",+ False,+ "no table named NoSuchTable",+ ),+ ],+ )+ def test_maps_the_workspace_probe_to_a_message(+ self, query_response: ScriptedResponse, table: str | None, expected_ok: bool, expected_fragment: str | None+ ) -> None:+ with scripted_network(route({TOKEN_PATH: [_token()], QUERY_PATH: [query_response]})) as network:+ ok, message = AzureLogAnalyticsSource().validate_credentials(+ _config(tables=None), team_id=1, schema_name=table+ )++ assert ok is expected_ok+ if expected_fragment is None:+ assert message is None+ else:+ assert message is not None and expected_fragment in message+ probe = network.requests_log[-1].json()["query"]+ assert probe == (f"{table} | take 0" if table else "print ok = 1")++ def test_rejected_secret_stops_before_the_workspace_probe(self) -> None:+ rejected = ScriptedResponse(+ status=401,+ json={"error": "invalid_client", "error_description": "AADSTS7000215: Invalid client secret provided."},+ )+ with scripted_network(route({TOKEN_PATH: [rejected]})) as network:+ ok, message = AzureLogAnalyticsSource().validate_credentials(_config(), team_id=1)++ assert ok is False+ assert message is not None and "Microsoft Entra ID" in message+ assert [request.path for request in network.requests_log] == [TOKEN_PATH]++ @pytest.mark.parametrize(+ ("answers", "expected_ok", "expected_fragment", "expected_probes"),+ [+ ([_missing("AppTrace"), _missing("MyEvents_CL")], False, "None of the listed tables exist", 2),+ (+ [+ ScriptedResponse(status=400, json={"error": {"code": "BadArgumentError", "message": "Basic plan"}}),+ ScriptedResponse(status=403, json={"error": {"code": "InsufficientAccessError", "message": "No"}}),+ ],+ False,+ "None of the listed tables can be read",+ 2,+ ),+ ([_missing("AppTrace"), _page([])], True, None, 2),+ ([_page([])], True, None, 1),+ ],+ )+ def test_creation_needs_one_readable_listed_table(+ self, answers: list[ScriptedResponse], expected_ok: bool, expected_fragment: str | None, expected_probes: int+ ) -> None:+ with scripted_network(route({TOKEN_PATH: [_token(), _token()], QUERY_PATH: [_page([]), *answers]})) as network:+ ok, message = AzureLogAnalyticsSource().validate_credentials(+ _config(tables="AppTrace\nMyEvents_CL"), team_id=1+ )++ assert ok is expected_ok+ if expected_fragment is None:+ assert message is None+ else:+ assert message is not None and expected_fragment in message+ queries = [request.json()["query"] for request in network.requests_log if request.path == QUERY_PATH]+ assert queries == ["print ok = 1", "AppTrace | take 0", "MyEvents_CL | take 0"][: 1 + expected_probes]++ def test_setup_probes_use_short_deadlines_and_no_retries(self) -> None:+ unavailable = ScriptedResponse(+ status=504, json={"error": {"code": "GatewayTimeout"}}, headers={"Retry-After": "1"}+ )+ with scripted_network(route({TOKEN_PATH: [_token()], QUERY_PATH: [unavailable]})) as network:+ ok, message = AzureLogAnalyticsSource().validate_credentials(_config(), team_id=1)++ assert ok is False+ assert message is not None and "Could not query the Log Analytics workspace" in message+ queries = [request for request in network.requests_log if request.path == QUERY_PATH]+ assert len(queries) == 1+ assert queries[0].read_timeout == 30++ @pytest.mark.parametrize(+ ("config", "schema_name"),+ [(_config(tables="AppTraces | union SecurityEvent"), None), (_config(), "AppTraces | union SecurityEvent")],+ )+ def test_rejects_a_table_name_that_would_change_the_query_without_a_request(+ self, config: AzureLogAnalyticsSourceConfig, schema_name: str | None+ ) -> None:+ with scripted_network([]) as network:+ ok, message = AzureLogAnalyticsSource().validate_credentials(config, team_id=1, schema_name=schema_name)++ assert ok is False+ assert message is not None and "not a valid Log Analytics table name" in message+ assert network.requests_log == []+++class TestSchemaDiscovery:+ def test_blank_table_list_discovers_tables_from_usage(self) -> None:+ usage = ScriptedResponse(+ json={+ "tables": [+ {+ "name": "PrimaryResult",+ "columns": [{"name": "DataType", "type": "string"}],+ "rows": [["AppRequests"], ["MyEvents_CL"], ["not a table"]],+ }+ ]+ }+ )+ with scripted_network(route({TOKEN_PATH: [_token()], QUERY_PATH: [usage]})) as network:+ schemas = AzureLogAnalyticsSource().get_schemas(_config(tables=None), team_id=1)++ assert [schema.name for schema in schemas] == ["AppRequests", "MyEvents_CL"]+ assert network.requests_log[-1].json()["query"].startswith("Usage | where TimeGenerated > ago(30d)")++ def test_endpoint_permissions_flag_tables_the_query_api_cannot_read(self) -> None:+ basic_plan = ScriptedResponse(+ status=400,+ json={+ "error": {+ "code": "BadArgumentError",+ "message": "The request had some invalid properties",+ "innererror": {"code": "SemanticError", "message": "Table 'BasicLogs_CL' has the Basic plan."},+ }+ },+ )++ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == TOKEN_PATH:+ return _token()+ query = request.json()["query"]+ if query.startswith("BasicLogs_CL"):+ return basic_plan+ if query.startswith("Missing_CL"):+ return ScriptedResponse(+ status=400,+ json={+ "error": {+ "code": "BadArgumentError",+ "message": "The request had some invalid properties",+ "innererror": {+ "code": "SemanticError",+ "message": "'take' operator: Failed to resolve table or column expression named 'Missing_CL'",+ },+ }+ },+ )+ if query.startswith("Restricted"):+ return ScriptedResponse(+ status=403, json={"error": {"code": "InsufficientAccessError", "message": "Insufficient access"}}+ )+ if query.startswith("Throttled"):+ return ScriptedResponse(status=429, json={}, headers={"Retry-After": "300"})+ return _page([])++ with scripted_network(respond) as network:+ permissions = AzureLogAnalyticsSource().get_endpoint_permissions(+ _config(),+ team_id=1,+ endpoints=["AppTraces", "BasicLogs_CL", "Missing_CL", "Restricted", "Throttled", "Later_CL"],+ )++ assert permissions["AppTraces"] is None+ assert permissions["BasicLogs_CL"] is not None and "Basic and Auxiliary" in permissions["BasicLogs_CL"]+ assert permissions["Missing_CL"] is not None and "doesn't exist in the workspace" in permissions["Missing_CL"]+ assert permissions["Restricted"] is not None and "can't read this table" in permissions["Restricted"]+ # A throttle counts as readable and ends probing, without retries, so setup stays fast.+ assert permissions["Throttled"] is None and permissions["Later_CL"] is None+ probed = [r.json()["query"].split(" ")[0] for r in network.requests_log if r.path == QUERY_PATH]+ assert probed == ["AppTraces", "BasicLogs_CL", "Missing_CL", "Restricted", "Throttled"]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureloganalytics.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureloganalytics.pyindex ebeaa64d6cd0..37ea8cb439ce 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureloganalytics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureloganalytics.py@@ -6,4 +6,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common imp@config.configclass AzureLogAnalyticsSourceConfig(config.Config):- pass+ tenant_id: str+ client_id: str+ client_secret: str+ workspace_id: str+ tables: str | None = Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txtindex 74cb057e878d..c58cb0b3a0d8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt@@ -12,6 +12,7 @@ AwsAthena not-checkable:schemasAwsCostExplorer fail:safe-pointAzureBlob not-checkable:schemasAzureCostManagement not-checkable:no-request+AzureLogAnalytics not-checkable:schemasBaserow not-checkable:schemasBigMailer fail:safe-pointBigQuery not-checkable:driver:google.clouddiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.pyindex f9ab0a124c5d..32f28988add4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py@@ -81,6 +81,7 @@ NOT_CHECKABLE: dict[str, str] = {"AwsAthena": "schemas","AzureBlob": "schemas","AzureCostManagement": "no-request",+ "AzureLogAnalytics": "no-request","Baserow": "schemas","BigQuery": "driver","BingAds": "database",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.pyindex e38ff3943aba..9a0fc71c1a2d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py@@ -104,6 +104,7 @@ UNKNOWN_SCHEMA_NOT_CHECKABLE: dict[str, str] = {"Attentive": "webhook","AwsAthena": "request","AzureBlob": "request",+ "AzureLogAnalytics": "request","Baserow": "request","BigQuery": "driver","BingAds": "database",