Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): implement azure log analytics source (#118158)

Daniel RC3d774c584d24
diff --git a/frontend/public/services/azure_log_analytics.png b/frontend/public/services/azure_log_analytics.png
index c5736e43b50d..9d1f80398d88 100644
Binary files a/frontend/public/services/azure_log_analytics.png and b/frontend/public/services/azure_log_analytics.png differ
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
index 1e2d572ccc7a..13a501fc371d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
@@ -140,6 +140,7 @@ the row lists both.
| azure_blob | HTTP (vendor SDK) | azure-storage-blob (listing), adlfs (reads), deltalake | ⚠️ |
| azure_cost_management | HTTP | requests | ✅ |
| azure_devops | HTTP | requests | ✅ |
+| azure_log_analytics | HTTP | requests | ✅ |
| babelforce | HTTP | requests | ✅ |
| back_market | HTTP | requests + `rest_source.RESTClient` | ✅ |
| bamboohr | HTTP | requests | ✅ |
@@ -1068,7 +1069,6 @@ doesn't conflict with concurrent PRs.
- azure_api_management
- azure_data_explorer
- azure_data_factory
-- azure_log_analytics
- azure_monitor_alerts
- azure_monitor_metrics
- azure_openai_usage
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/azure_log_analytics.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/azure_log_analytics.py
new file mode 100644
index 000000000000..49d855de9bf9
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/azure_log_analytics.py
@@ -0,0 +1,554 @@
+import re
+import json
+import time
+from collections.abc import Callable, Iterator
+from dataclasses import field
+from datetime import UTC, date, datetime, timedelta
+from typing import Any, Optional
+from urllib.parse import quote
+
+import requests
+import structlog
+from structlog.types import FilteringBoundLogger
+from urllib3.util.retry import Retry
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.settings import (
+ API_HOST,
+ DISCOVERY_LOOKBACK_DAYS,
+ INGESTION_TIME_FIELD,
+ ITEM_ID_FIELD,
+ LOGIN_HOST,
+ PAGE_SIZE,
+ PRIMARY_KEYS,
+ TABLE_NAME_RE,
+ TIME_GENERATED_PRUNE,
+ TIMESTAMP_FIELD,
+ TOKEN_SCOPE,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (
+ interruptible_wait,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
+
+# A query runs for up to three minutes server-side before Log Analytics cancels it.
+REQUEST_TIMEOUT_SECONDS = (30, 200)
+MAX_RETRY_ATTEMPTS = 5
+MAX_BACKOFF_SECONDS = 60.0
+# One request with its retries and waits holds the worker for five minutes at most (the source
+# contract), so a retry only starts when a full-length attempt still fits in the budget.
+RETRY_BUDGET_SECONDS = 300.0
+# A page that hits a result-size or execution limit is read again with half as many rows.
+MIN_PAGE_SIZE = 100
+# Permission probes run inside the source setup request, so they are short, never retried, and
+# stop once the total passes this budget.
+PROBE_REQUEST_TIMEOUT_SECONDS = (10, 30)
+PROBE_BUDGET_SECONDS = 60.0
+# Creation and the schema picker each probe inside the same 120 s setup request.
+CREATION_PROBE_BUDGET_SECONDS = 30.0
+
+_GUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$")
+# The cursor values are embedded in KQL, so they must keep the exact shape Log Analytics returns.
+_DATETIME_LITERAL_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,7})?Z$")
+_ITEM_ID_LITERAL_RE = re.compile(r"^[A-Za-z0-9-]{0,128}$")
+# The ingestion time as KQL renders it (100 ns precision), so the cursor matches rows exactly.
+CURSOR_COLUMN = "__posthog_cursor"
+
+
+class AzureLogAnalyticsRetryableError(Exception):
+ """Throttled (429) or a gateway failure (5xx) that outlived the retry budget."""
+
+ pass
+
+
+class AzureLogAnalyticsPartialResultError(ValueError):
+ """The query hit a service limit and returned only part of its rows."""
+
+ pass
+
+
+@frozen
+class AzureLogAnalyticsResumeConfig:
+ # Where the next page starts: after this ingestion time (exactly as KQL renders it, 100 ns
+ # precision, which a Python datetime cannot hold) and this _ItemId. An empty item_id means
+ # "at or after" the time, which starts a window or a watermark.
+ ingested_at: str
+ item_id: str = ""
+
+
+@frozen
+class WorkspaceCredentials:
+ tenant_id: str
+ client_id: str
+ client_secret: str = field(repr=False)
+ workspace_id: str
+
+ def client(
+ self,
+ logger: FilteringBoundLogger,
+ safe_point: Callable[[], None] | None = None,
+ probe: bool = False,
+ ) -> "AzureLogAnalyticsClient":
+ return AzureLogAnalyticsClient(
+ self.tenant_id,
+ self.client_id,
+ self.client_secret,
+ self.workspace_id,
+ logger,
+ safe_point=safe_point,
+ request_timeout=PROBE_REQUEST_TIMEOUT_SECONDS if probe else REQUEST_TIMEOUT_SECONDS,
+ max_attempts=1 if probe else MAX_RETRY_ATTEMPTS,
+ )
+
+
+def validate_workspace_id(workspace_id: str) -> str:
+ value = (workspace_id or "").strip()
+ if not _GUID_RE.match(value):
+ raise ValueError("Workspace ID must be the workspace GUID from the workspace's Overview page.")
+ return value
+
+
+def parse_table_names(raw: str | None) -> list[str]:
+ """One table per line (commas also accepted), deduplicated in order."""
+ names: list[str] = []
+ for part in re.split(r"[\n,]", raw or ""):
+ name = part.strip()
+ if not name:
+ continue
+ if not TABLE_NAME_RE.match(name):
+ raise ValueError(f"{name!r} is not a valid Log Analytics table name.")
+ if name not in names:
+ names.append(name)
+ return names
+
+
+def _error_description(response: requests.Response) -> str:
+ # Log Analytics errors look like {"error": {"code": ..., "message": ..., "innererror": {...}}};
+ # the inner error carries the KQL message that names the actual problem.
+ try:
+ body = response.json()
+ except ValueError:
+ return ""
+ if not isinstance(body, dict):
+ return ""
+ error = body.get("error")
+ if isinstance(error, dict):
+ messages = [str(error["message"])] if error.get("message") else []
+ inner = error.get("innererror")
+ while isinstance(inner, dict):
+ if inner.get("message"):
+ messages.append(str(inner["message"]))
+ inner = inner.get("innererror")
+ if messages:
+ return " — " + " ".join(messages)
+ if isinstance(body.get("error_description"), str):
+ return f" — {body['error_description']}"
+ return ""
+
+
+def _backoff_seconds(response: requests.Response | None, attempt: int) -> float:
+ raw = response.headers.get("Retry-After") if response is not None else None
+ if raw is not None:
+ try:
+ return min(max(float(raw), 0.0), MAX_BACKOFF_SECONDS)
+ except ValueError:
+ pass
+ return min(2.0**attempt, MAX_BACKOFF_SECONDS)
+
+
+class AzureLogAnalyticsClient:
+ """Client-credentials client for the Log Analytics query API.
+
+ Transport retries are off (`Retry(total=0)`): every query is a POST, which the default retry
+ never covers, so throttling and gateway errors are retried here, within `RETRY_BUDGET_SECONDS`.
+ """
+
+ def __init__(
+ self,
+ tenant_id: str,
+ client_id: str,
+ client_secret: str,
+ workspace_id: str,
+ logger: FilteringBoundLogger,
+ safe_point: Callable[[], None] | None = None,
+ request_timeout: tuple[float, float] = REQUEST_TIMEOUT_SECONDS,
+ max_attempts: int = MAX_RETRY_ATTEMPTS,
+ ) -> None:
+ self._request_timeout = request_timeout
+ self._max_attempts = max_attempts
+ self._tenant_id = tenant_id.strip()
+ self._client_id = client_id.strip()
+ self._client_secret = client_secret
+ self._workspace_id = validate_workspace_id(workspace_id)
+ self._logger = logger
+ self._safe_point = safe_point
+ self._token: Optional[str] = None
+ redact = tuple(value for value in (client_secret,) if value)
+ # The token exchange carries the secret and returns a bearer token; keep it out of samples.
+ self._auth_session = make_tracked_session(retry=Retry(total=0), redact_values=redact, capture=False)
+ self._api_session = make_tracked_session(retry=Retry(total=0), redact_values=redact)
+
+ def mint_token(self) -> str:
+ url = f"{LOGIN_HOST}/{quote(self._tenant_id, safe='')}/oauth2/v2.0/token"
+ response = self._auth_session.post(
+ url,
+ data={
+ "grant_type": "client_credentials",
+ "client_id": self._client_id,
+ "client_secret": self._client_secret,
+ "scope": TOKEN_SCOPE,
+ },
+ timeout=self._request_timeout,
+ )
+ if not response.ok:
+ raise requests.HTTPError(
+ f"{response.status_code} Client Error: {response.reason} for url: {LOGIN_HOST}{_error_description(response)}",
+ response=response,
+ )
+ token = response.json().get("access_token")
+ if not token:
+ raise requests.HTTPError(f"Azure AD returned no access token for url: {LOGIN_HOST}", response=response)
+ self._token = token
+ return token
+
+ def query(self, kql: str) -> list[dict[str, Any]]:
+ url = f"{API_HOST}/v1/workspaces/{self._workspace_id}/query"
+ attempt = 0
+ reminted = False
+ started = time.monotonic()
+
+ while True:
+ token = self._token or self.mint_token()
+ response: requests.Response | None
+ try:
+ response = self._api_session.post(
+ url,
+ json={"query": kql},
+ headers={"Authorization": f"Bearer {token}"},
+ timeout=self._request_timeout,
+ )
+ status = str(response.status_code)
+ except (requests.ConnectionError, requests.Timeout) as error:
+ # Queries only read, so a dropped connection or a timeout is safe to send again.
+ response = None
+ status = type(error).__name__
+
+ # Tokens last about an hour; a long backfill can outlive one.
+ if response is not None and response.status_code == 401 and not reminted:
+ reminted = True
+ self._token = None
+ continue
+
+ if response is None or response.status_code == 429 or response.status_code >= 500:
+ attempt += 1
+ delay = _backoff_seconds(response, attempt)
+ elapsed = time.monotonic() - started
+ if attempt >= self._max_attempts or elapsed + delay + sum(self._request_timeout) > RETRY_BUDGET_SECONDS:
+ raise AzureLogAnalyticsRetryableError(f"Azure Log Analytics error (retryable): status={status}")
+ self._logger.debug(f"Azure Log Analytics retry: status={status}, sleeping {delay}s (attempt {attempt})")
+ interruptible_wait(delay, safe_point=self._safe_point)
+ continue
+
+ if not response.ok:
+ raise requests.HTTPError(
+ f"{response.status_code} Client Error: {response.reason} for url: {API_HOST}{_error_description(response)}",
+ response=response,
+ )
+
+ return rows_from_response(response.json())
+
+
+def rows_from_response(payload: Any) -> list[dict[str, Any]]:
+ """Zip the primary result table's `columns` + `rows` into typed row dicts."""
+ if not isinstance(payload, dict):
+ return []
+ # A query that hit a service limit returns 200 with the rows it managed and an `error`;
+ # syncing those as if complete would lose the rest silently.
+ if isinstance(payload.get("error"), dict):
+ error = payload["error"]
+ raise AzureLogAnalyticsPartialResultError(
+ f"Azure Log Analytics returned a partial result: {error.get('code')} {error.get('message')}"
+ )
+ tables = payload.get("tables") or []
+ if not tables:
+ return []
+ table = tables[0]
+ columns = [(str(column.get("name")), str(column.get("type", ""))) for column in table.get("columns") or []]
+ return [
+ {name: _convert(value, column_type) for (name, column_type), value in zip(columns, values)}
+ for values in table.get("rows") or []
+ ]
+
+
+def _convert(value: Any, column_type: str) -> Any:
+ if value is None:
+ return None
+ if column_type == "datetime" and isinstance(value, str):
+ try:
+ return datetime.fromisoformat(value.replace("Z", "+00:00"))
+ except ValueError:
+ return value
+ if column_type == "dynamic" and isinstance(value, str):
+ try:
+ return json.loads(value)
+ except ValueError:
+ return value
+ if column_type == "bool" and isinstance(value, int | str):
+ return str(value).lower() in ("1", "true")
+ return value
+
+
+def _format_datetime_literal(value: Any) -> str:
+ if isinstance(value, str):
+ text = value.strip()
+ if _DATETIME_LITERAL_RE.match(text):
+ return text
+ value = datetime.fromisoformat(text.replace("Z", "+00:00"))
+ if isinstance(value, datetime):
+ moment = value if value.tzinfo else value.replace(tzinfo=UTC)
+ return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+ if isinstance(value, date):
+ return f"{value.isoformat()}T00:00:00Z"
+ raise ValueError(f"Azure Log Analytics incremental value is not a datetime: {value!r}")
+
+
+def build_query(
+ table: str,
+ after: AzureLogAnalyticsResumeConfig,
+ before: str,
+ page_size: int,
+) -> str:
+ if not TABLE_NAME_RE.match(table):
+ raise ValueError(f"{table!r} is not a valid Log Analytics table name.")
+ for literal in (after.ingested_at, before):
+ if not _DATETIME_LITERAL_RE.match(literal):
+ raise ValueError("Azure Log Analytics cursor has an unexpected shape")
+ if not _ITEM_ID_LITERAL_RE.match(after.item_id):
+ raise ValueError("Azure Log Analytics cursor has an unexpected shape")
+
+ moment = f"datetime({after.ingested_at})"
+ lines = [table, f"| where {TIMESTAMP_FIELD} >= {moment} - {TIME_GENERATED_PRUNE}"]
+ if after.item_id:
+ lines.append(
+ f"| where ingestion_time() > {moment} or "
+ f"(ingestion_time() == {moment} and strcmp({ITEM_ID_FIELD}, '{after.item_id}') > 0)"
+ )
+ else:
+ lines.append(f"| where ingestion_time() >= {moment}")
+ lines.append(f"| where ingestion_time() < datetime({before})")
+ # ingestion_time() only resolves against the table, so it is captured before the sort.
+ lines.append(f"| extend {INGESTION_TIME_FIELD} = ingestion_time()")
+ # _ItemId breaks ties, so a page that ends inside one ingestion time resumes without gaps.
+ lines.append(f"| sort by {INGESTION_TIME_FIELD} asc, {ITEM_ID_FIELD} asc")
+ lines.append(f"| take {page_size}")
+ # Re-assigning _ItemId makes it a visible column in the result.
+ lines.append(f"| extend {ITEM_ID_FIELD} = {ITEM_ID_FIELD}, {CURSOR_COLUMN} = tostring({INGESTION_TIME_FIELD})")
+ return "\n".join(lines)
+
+
+def window_end(after: AzureLogAnalyticsResumeConfig, run_start: datetime, hourly: bool) -> str:
+ """The end of the day or hour the cursor sits in, capped at the run start.
+
+ Each page sorts only the rows left in its window, so a backfill never re-scans the whole table
+ for every page. A busy table walks hours instead of days. The cap leaves rows ingested during
+ the run to the next run.
+ """
+ moment = datetime.fromisoformat(after.ingested_at[:19]).replace(tzinfo=UTC)
+ if hourly:
+ end = moment.replace(minute=0, second=0) + timedelta(hours=1)
+ else:
+ end = moment.replace(hour=0, minute=0, second=0) + timedelta(days=1)
+ if end >= run_start:
+ return _format_datetime_literal(run_start)
+ return end.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _first_cursor(client: AzureLogAnalyticsClient, table: str, watermark: Any) -> AzureLogAnalyticsResumeConfig | None:
+ if watermark is not None:
+ return AzureLogAnalyticsResumeConfig(ingested_at=_format_datetime_literal(watermark))
+ rows = client.query(f"{table} | summarize start = min(ingestion_time()) | project start = tostring(start)")
+ start = str(rows[0].get("start") or "") if rows else ""
+ return AzureLogAnalyticsResumeConfig(ingested_at=start) if start else None
+
+
+def get_rows(
+ client: AzureLogAnalyticsClient,
+ table: str,
+ watermark: Any,
+ resumable_source_manager: ResumableSourceManager[AzureLogAnalyticsResumeConfig],
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if resume is not None:
+ logger.debug(f"Azure Log Analytics: resuming {table} after {resume.ingested_at}")
+ after = resume or _first_cursor(client, table, watermark)
+ if after is None:
+ return
+
+ run_start = datetime.now(UTC)
+ run_end = _format_datetime_literal(run_start)
+ page_size = PAGE_SIZE
+ hourly = False
+ while True:
+ before = window_end(after, run_start, hourly)
+ try:
+ rows = client.query(build_query(table, after, before, page_size))
+ except AzureLogAnalyticsPartialResultError:
+ if page_size <= MIN_PAGE_SIZE:
+ raise
+ hourly = True
+ page_size = max(MIN_PAGE_SIZE, page_size // 2)
+ logger.debug(f"Azure Log Analytics: {table} page hit a service limit, retrying with {page_size} rows")
+ # The staged cursor covers every yielded row, so the run can hand over between retries.
+ resumable_source_manager.safe_point()
+ continue
+
+ window_done = len(rows) < page_size
+ if not window_done:
+ hourly = True
+ last = rows[-1]
+ after = AzureLogAnalyticsResumeConfig(
+ ingested_at=str(last.get(CURSOR_COLUMN)), item_id=str(last.get(ITEM_ID_FIELD))
+ )
+ else:
+ page_size = PAGE_SIZE
+ hourly = hourly and not before.endswith("T00:00:00Z")
+ after = AzureLogAnalyticsResumeConfig(ingested_at=before)
+ for row in rows:
+ row.pop(CURSOR_COLUMN, None)
+
+ # Staged before the yield it covers, the final page included: the pipeline commits it once
+ # the page is written.
+ resumable_source_manager.save_state(after)
+ if rows:
+ yield rows
+ else:
+ # A window with no rows yields nothing; the pipeline only sees a shutdown at an item or here.
+ resumable_source_manager.safe_point()
+ if window_done and before == run_end:
+ return
+
+
+def discover_tables(client: AzureLogAnalyticsClient) -> list[str]:
+ rows = client.query(
+ f"Usage | where TimeGenerated > ago({DISCOVERY_LOOKBACK_DAYS}d) | distinct DataType | sort by DataType asc"
+ )
+ return [str(row["DataType"]) for row in rows if TABLE_NAME_RE.match(str(row.get("DataType") or ""))]
+
+
+def _is_missing_table(error: requests.HTTPError) -> bool:
+ return getattr(error.response, "status_code", None) == 400 and "Failed to resolve table" in str(error)
+
+
+def probe_tables(
+ credentials: WorkspaceCredentials,
+ tables: list[str],
+ missing_table_reason: str,
+ stop_at_first_readable: bool = False,
+ budget_seconds: float = PROBE_BUDGET_SECONDS,
+) -> dict[str, str | None]:
+ """Why the query API cannot read each table, or None when it can.
+
+ Basic and Auxiliary plan tables are only readable through the search API, so the query API
+ rejects them with a 400, and table-level access control answers 403. A throttle or an outage
+ is not a reason: the table counts as readable, and probing stops so setup stays fast.
+ """
+ reasons: dict[str, str | None] = dict.fromkeys(tables)
+ client = credentials.client(structlog.get_logger(__name__), probe=True)
+ try:
+ client.mint_token()
+ except (requests.RequestException, ValueError):
+ return reasons
+ started = time.monotonic()
+ for table in tables:
+ if time.monotonic() - started > budget_seconds:
+ break
+ try:
+ client.query(f"{table} | take 0")
+ if stop_at_first_readable:
+ break
+ except requests.HTTPError as error:
+ status = getattr(error.response, "status_code", None)
+ if _is_missing_table(error):
+ reasons[table] = missing_table_reason
+ elif status == 400:
+ reasons[table] = (
+ "Log Analytics can't query this table. Basic and Auxiliary plan tables aren't supported."
+ f"{_error_description(error.response)}"
+ )
+ elif status == 403:
+ reasons[table] = "The app can't read this table. Give it access to the table, or leave the table out."
+ else:
+ break
+ except (AzureLogAnalyticsRetryableError, requests.RequestException):
+ break
+ return reasons
+
+
+def _probe_failure_message(error: Exception, missing_table_message: str) -> str:
+ status = getattr(getattr(error, "response", None), "status_code", None)
+ if status == 403:
+ return (
+ "The app registration cannot query this workspace. Give its service principal the Log Analytics "
+ "Reader role on the workspace, or read access to the table, then try again."
+ )
+ if status == 404:
+ return "No Log Analytics workspace has this workspace ID. Copy the workspace ID from the workspace's Overview page."
+ if isinstance(error, requests.HTTPError) and _is_missing_table(error):
+ return missing_table_message
+ return f"Could not query the Log Analytics workspace: {error}"
+
+
+def validate_workspace(
+ credentials: WorkspaceCredentials, probe_query: str, missing_table_message: str
+) -> tuple[bool, Optional[str]]:
+ """Mint a token, then run `probe_query`, which proves the app can query the workspace.
+
+ This runs inside the source setup request, so it uses the short probe deadlines.
+ """
+ try:
+ client = credentials.client(structlog.get_logger(__name__), probe=True)
+ except ValueError as error:
+ return False, str(error)
+
+ try:
+ client.mint_token()
+ except Exception:
+ return (
+ False,
+ "Could not authenticate with Microsoft Entra ID. Check the tenant ID, client ID and client secret.",
+ )
+
+ try:
+ client.query(probe_query)
+ except Exception as error:
+ return False, _probe_failure_message(error, missing_table_message)
+
+ return True, None
+
+
+def azure_log_analytics_source(
+ credentials: WorkspaceCredentials,
+ table: str,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[AzureLogAnalyticsResumeConfig],
+ db_incremental_field_last_value: Any = None,
+) -> SourceResponse:
+ client = credentials.client(logger, safe_point=resumable_source_manager.safe_point)
+
+ return SourceResponse(
+ name=table,
+ items=lambda: get_rows(client, table, db_incremental_field_last_value, resumable_source_manager, logger),
+ primary_keys=PRIMARY_KEYS,
+ partition_count=1,
+ partition_size=1,
+ partition_keys=[TIMESTAMP_FIELD],
+ partition_mode="datetime",
+ partition_format="day",
+ # Every page is one query sorted by ingestion time, then _ItemId.
+ sort_mode="asc",
+ on_complete=resumable_source_manager.clear_state,
+ )
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/settings.py
new file mode 100644
index 000000000000..cd2ca8f6fbea
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/settings.py
@@ -0,0 +1,37 @@
+import re
+
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+
+API_HOST = "https://api.loganalytics.azure.com"
+# api.loganalytics.azure.com still issues tokens for the original resource URI.
+TOKEN_SCOPE = "https://api.loganalytics.io/.default"
+LOGIN_HOST = "https://login.microsoftonline.com"
+
+# A query returns at most 500,000 rows and ~100 MB; log rows can be wide, so pages stay small.
+PAGE_SIZE = 5000
+# Every Log Analytics table carries these two standard columns.
+TIMESTAMP_FIELD = "TimeGenerated"
+ITEM_ID_FIELD = "_ItemId"
+PRIMARY_KEYS = [ITEM_ID_FIELD]
+# Reads page on ingestion time: an agent that was offline delivers records hours after their TimeGenerated.
+INGESTION_TIME_FIELD = "_IngestionTime"
+# ingestion_time() is approximate and concurrent ingestions land out of order.
+INGESTION_LOOKBACK_SECONDS = 15 * 60
+# Azure replaces a TimeGenerated more than two days older than its arrival, so this bound loses no row.
+TIME_GENERATED_PRUNE = "3d"
+# Discovery reads table names from the Usage table, which every workspace has.
+DISCOVERY_LOOKBACK_DAYS = 30
+
+# Table names are interpolated into KQL, so only plain identifiers (including `_CL` custom tables)
+# are accepted.
+TABLE_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,254}$")
+
+INCREMENTAL_FIELDS: list[IncrementalField] = [
+ {
+ "label": INGESTION_TIME_FIELD,
+ "type": IncrementalFieldType.DateTime,
+ "field": INGESTION_TIME_FIELD,
+ "field_type": IncrementalFieldType.DateTime,
+ "is_indexed": True,
+ }
+]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/source.py
index 5354653fc7af..7da152d0d1df 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/source.py
@@ -1,16 +1,228 @@
-from typing import cast
+from abc import abstractmethod
+from typing import Generic, Optional, cast
-from products.warehouse_sources.backend.facade.source_config import DataWarehouseSourceCategory, SourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource
+import structlog
+
+from products.warehouse_sources.backend.facade.codebase_detection import detect
+from products.warehouse_sources.backend.facade.source_config import (
+ DataWarehouseSourceCategory,
+ ReleaseStatus,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.azure_log_analytics import (
+ CREATION_PROBE_BUDGET_SECONDS,
+ AzureLogAnalyticsResumeConfig,
+ WorkspaceCredentials,
+ azure_log_analytics_source,
+ discover_tables,
+ parse_table_names,
+ probe_tables,
+ validate_workspace,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.settings import (
+ INCREMENTAL_FIELDS,
+ INGESTION_LOOKBACK_SECONDS,
+ TABLE_NAME_RE,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+ ConfigType,
+ FieldType,
+ ResumableSource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.azureloganalytics import (
AzureLogAnalyticsSourceConfig,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
+WORKSPACE_FIELDS: list[FieldType] = [
+ SourceFieldInputConfig(
+ name="tenant_id",
+ label="Directory (tenant) ID",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="00000000-0000-0000-0000-000000000000",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="client_id",
+ label="Application (client) ID",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="00000000-0000-0000-0000-000000000000",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="client_secret",
+ label="Client secret",
+ type=SourceFieldInputConfigType.PASSWORD,
+ required=True,
+ placeholder="",
+ secret=True,
+ ),
+ SourceFieldInputConfig(
+ name="workspace_id",
+ label="Workspace ID",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="00000000-0000-0000-0000-000000000000",
+ secret=False,
+ ),
+]
+
+
+class LogAnalyticsWorkspaceSource(ResumableSource[ConfigType, AzureLogAnalyticsResumeConfig], Generic[ConfigType]):
+ """A source that syncs tables of one Log Analytics workspace. Subclasses pick the tables."""
+
+ missing_table_reason = (
+ "This table doesn't exist in the workspace. Check the table name, or remove it from the list."
+ )
+ missing_table_message = (
+ "The workspace has no table named {table}. Check the table name, or remove it from the list."
+ )
+ no_tables_message = "None of the listed tables exist in the workspace. Check the table names."
+
+ @abstractmethod
+ def credentials(self, config: ConfigType) -> WorkspaceCredentials: ...
+
+ @abstractmethod
+ def table_names(self, config: ConfigType) -> list[str]: ...
+
+ def tables_checked_at_creation(self, config: ConfigType) -> list[str]:
+ """Tables probed at source creation; creation fails when every one of them is missing."""
+ return []
+
+ @property
+ def connection_host_fields(self) -> list[str]:
+ # `tenant_id` picks the directory the stored secret is exchanged in, and `workspace_id` picks
+ # the workspace the token is spent on. Retargeting either must re-require the secret.
+ return ["tenant_id", "workspace_id"]
+
+ def get_non_retryable_errors(self) -> dict[str, str | None]:
+ return {
+ "400 Client Error: Bad Request for url: https://login.microsoftonline.com": "Microsoft Entra ID rejected the app credentials. Check the tenant ID, client ID and client secret, then reconnect.",
+ "401 Client Error: Unauthorized for url: https://login.microsoftonline.com": "Microsoft Entra ID rejected the app credentials. Check the client ID and client secret, then reconnect.",
+ "403 Client Error: Forbidden for url: https://login.microsoftonline.com": "Microsoft Entra ID refused to issue a token for the app. Check that the app registration can sign in to this tenant, then reconnect.",
+ "401 Client Error: Unauthorized for url: https://api.loganalytics.azure.com": "Log Analytics rejected the access token. Check that the app registration is still enabled, then reconnect.",
+ "403 Client Error: Forbidden for url: https://api.loganalytics.azure.com": "The app cannot query this workspace. Give its service principal the Log Analytics Reader role on the workspace, or read access to the table, then reconnect.",
+ "404 Client Error: Not Found for url: https://api.loganalytics.azure.com": "No Log Analytics workspace has this workspace ID. Check the workspace ID, then reconnect.",
+ "Failed to resolve table": self.missing_table_reason,
+ }
+
+ def get_retryable_errors(self) -> set[str]:
+ return {"Azure Log Analytics error (retryable)"}
+
+ def get_schemas(
+ self,
+ config: ConfigType,
+ team_id: int,
+ with_counts: bool = False,
+ names: list[str] | None = None,
+ force_refresh: bool = False,
+ api_version: str | None = None,
+ ) -> list[SourceSchema]:
+ schemas = [
+ SourceSchema(
+ name=table_name,
+ supports_incremental=True,
+ # The watermark re-reads a short overlap, which only merge dedupes.
+ supports_append=False,
+ incremental_fields=INCREMENTAL_FIELDS,
+ default_incremental_lookback_seconds=INGESTION_LOOKBACK_SECONDS,
+ )
+ for table_name in self.table_names(config)
+ ]
+ if names is not None:
+ names_set = set(names)
+ schemas = [s for s in schemas if s.name in names_set]
+ return schemas
+
+ def get_endpoint_permissions(
+ self,
+ config: ConfigType,
+ team_id: int,
+ endpoints: list[str],
+ api_version: str | None = None,
+ ) -> dict[str, str | None]:
+ return probe_tables(self.credentials(config), endpoints, self.missing_table_reason)
+
+ def validate_credentials(
+ self,
+ config: ConfigType,
+ team_id: int,
+ schema_name: Optional[str] = None,
+ api_version: str | None = None,
+ ) -> tuple[bool, str | None]:
+ if schema_name is not None:
+ if not TABLE_NAME_RE.match(schema_name):
+ return False, f"{schema_name!r} is not a valid Log Analytics table name."
+ return validate_workspace(
+ self.credentials(config),
+ f"{schema_name} | take 0",
+ self.missing_table_message.format(table=schema_name),
+ )
+ try:
+ tables = self.tables_checked_at_creation(config)
+ except ValueError as error:
+ return False, str(error)
+ valid, message = validate_workspace(self.credentials(config), "print ok = 1", self.no_tables_message)
+ if not valid or not tables:
+ return valid, message
+ # One query per table, stopping at the first readable one: a Basic plan table fails any
+ # query that names it, and must not block the rest.
+ reasons = probe_tables(
+ self.credentials(config),
+ tables,
+ self.missing_table_reason,
+ stop_at_first_readable=True,
+ budget_seconds=CREATION_PROBE_BUDGET_SECONDS,
+ )
+ if any(reason is None for reason in reasons.values()):
+ return True, None
+ if all(reason == self.missing_table_reason for reason in reasons.values()):
+ return False, self.no_tables_message
+ first_reason = next(reason for reason in reasons.values() if reason != self.missing_table_reason)
+ return False, f"None of the listed tables can be read. {first_reason}"
+
+ def get_resumable_source_manager(
+ self, inputs: SourceInputs
+ ) -> ResumableSourceManager[AzureLogAnalyticsResumeConfig]:
+ return ResumableSourceManager[AzureLogAnalyticsResumeConfig](inputs, AzureLogAnalyticsResumeConfig)
+
+ def source_for_pipeline(
+ self,
+ config: ConfigType,
+ resumable_source_manager: ResumableSourceManager[AzureLogAnalyticsResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
+ return azure_log_analytics_source(
+ credentials=self.credentials(config),
+ table=inputs.schema_name,
+ logger=inputs.logger,
+ resumable_source_manager=resumable_source_manager,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value
+ if inputs.should_use_incremental_field
+ else None,
+ )
+
@SourceRegistry.register
-class AzureLogAnalyticsSource(SimpleSource[AzureLogAnalyticsSourceConfig]):
+class AzureLogAnalyticsSource(LogAnalyticsWorkspaceSource[AzureLogAnalyticsSourceConfig]):
+ codebase_detection = detect(
+ env=("^(AZURE_LOG_ANALYTICS|LOG_ANALYTICS)_",),
+ npm=("@azure/monitor-query", "@azure/monitor-query-logs"),
+ pypi=("azure-monitor-query",),
+ go=("github.com/Azure/azure-sdk-for-go/sdk/monitor/query/azlogs",),
+ maven=("com.azure:azure-monitor-query",),
+ )
+
+ api_docs_url = "https://learn.microsoft.com/en-us/azure/azure-monitor/logs/api/overview"
+
@property
def source_type(self) -> ExternalDataSourceType:
return ExternalDataSourceType.AZURELOGANALYTICS
@@ -21,7 +233,44 @@ class AzureLogAnalyticsSource(SimpleSource[AzureLogAnalyticsSourceConfig]):
name=ExternalDataSourceType.AZURELOGANALYTICS,
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="Microsoft Azure (Azure Monitor / Log Analytics)",
+ keywords=["azure monitor", "log analytics", "kql"],
+ releaseStatus=ReleaseStatus.ALPHA,
+ caption="""Sync tables from an Azure Monitor Log Analytics workspace into the PostHog Data warehouse. Each table becomes a warehouse table, synced incrementally on the time Azure ingested each record, so records that arrive late still sync.
+
+Register an app in Microsoft Entra ID, create a client secret for it, and give its service principal the **Log Analytics Reader** role on the workspace. Then enter the directory (tenant) ID, the application (client) ID, the secret value, and the workspace ID from the workspace's **Overview** page.
+
+List the tables to sync, one per line, for example `AppRequests` or `MyEvents_CL`. Leave the list empty to offer every table that received data in the last 30 days.""",
iconPath="/static/services/azure_log_analytics.png",
- fields=cast(list[FieldType], []),
- unreleasedSource=True,
+ docsUrl="https://posthog.com/docs/cdp/sources/azure-log-analytics",
+ fields=cast(
+ list[FieldType],
+ [
+ *WORKSPACE_FIELDS,
+ SourceFieldInputConfig(
+ name="tables",
+ label="Tables",
+ type=SourceFieldInputConfigType.TEXTAREA,
+ required=False,
+ placeholder="AppRequests\nAppExceptions\nMyEvents_CL",
+ secret=False,
+ ),
+ ],
+ ),
+ )
+
+ def credentials(self, config: AzureLogAnalyticsSourceConfig) -> WorkspaceCredentials:
+ return WorkspaceCredentials(
+ tenant_id=config.tenant_id,
+ client_id=config.client_id,
+ client_secret=config.client_secret,
+ workspace_id=config.workspace_id,
+ )
+
+ def table_names(self, config: AzureLogAnalyticsSourceConfig) -> list[str]:
+ # Discovery runs inside the source setup request, so it uses the short probe deadlines.
+ return parse_table_names(config.tables) or discover_tables(
+ self.credentials(config).client(structlog.get_logger(__name__), probe=True)
)
+
+ def tables_checked_at_creation(self, config: AzureLogAnalyticsSourceConfig) -> list[str]:
+ return parse_table_names(config.tables)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/tests/test_azure_log_analytics.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/tests/test_azure_log_analytics.py
new file mode 100644
index 000000000000..56e85ae4c988
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_log_analytics/tests/test_azure_log_analytics.py
@@ -0,0 +1,632 @@
+from collections.abc import Iterator
+from datetime import UTC, datetime
+from typing import Any
+
+import pytest
+import time_machine
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics import azure_log_analytics
+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.azure_log_analytics import (
+ AzureLogAnalyticsResumeConfig,
+ AzureLogAnalyticsRetryableError,
+ build_query,
+ parse_table_names,
+ rows_from_response,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.azure_log_analytics.source import (
+ AzureLogAnalyticsSource,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ RecordedRequest,
+ ScriptedResponse,
+ SourceDriver,
+ route,
+ scripted_network,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.azureloganalytics import (
+ AzureLogAnalyticsSourceConfig,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.tests.contract_harness import check_extraction
+
+TENANT = "11111111-1111-1111-1111-111111111111"
+WORKSPACE = "22222222-2222-2222-2222-222222222222"
+TOKEN_PATH = f"/{TENANT}/oauth2/v2.0/token"
+QUERY_PATH = f"/v1/workspaces/{WORKSPACE}/query"
+COLUMNS = [
+ {"name": "TimeGenerated", "type": "datetime"},
+ {"name": "_IngestionTime", "type": "datetime"},
+ {"name": "Message", "type": "string"},
+ {"name": "Properties", "type": "dynamic"},
+ {"name": "_ItemId", "type": "string"},
+ {"name": "__posthog_cursor", "type": "string"},
+]
+
+
+def _config(tables: str | None = "AppTraces") -> AzureLogAnalyticsSourceConfig:
+ return AzureLogAnalyticsSourceConfig(
+ tenant_id=TENANT, client_id="client", client_secret="secret", workspace_id=WORKSPACE, tables=tables
+ )
+
+
+def _token() -> ScriptedResponse:
+ return ScriptedResponse(json={"token_type": "Bearer", "expires_in": 3599, "access_token": "token-1"})
+
+
+def _row(ingested_at: str, item_id: str) -> list[Any]:
+ return [ingested_at, ingested_at, f"message {item_id}", '{"level": "info"}', item_id, ingested_at]
+
+
+def _missing(table: str) -> ScriptedResponse:
+ return ScriptedResponse(
+ status=400,
+ json={
+ "error": {
+ "code": "BadArgumentError",
+ "message": "The request had some invalid properties",
+ "innererror": {
+ "code": "SemanticError",
+ "message": f"'take' operator: Failed to resolve table or column expression named '{table}'",
+ },
+ }
+ },
+ )
+
+
+def _page(rows: list[list[Any]]) -> ScriptedResponse:
+ return ScriptedResponse(json={"tables": [{"name": "PrimaryResult", "columns": COLUMNS, "rows": rows}]})
+
+
+def _queries(requests: list[RecordedRequest]) -> list[str]:
+ return [request.json()["query"] for request in requests if request.path == QUERY_PATH]
+
+
+NOW = datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
+RUN_END = "2026-10-02T12:00:00.000000Z"
+WATERMARK_TODAY = {"incremental_field": "_IngestionTime", "db_incremental_field_last_value": "2026-10-02T09:30:00Z"}
+MIN_QUERY = "AppTraces | summarize start = min(ingestion_time()) | project start = tostring(start)"
+
+
+def _min_start(start: str) -> ScriptedResponse:
+ return ScriptedResponse(
+ json={
+ "tables": [{"name": "PrimaryResult", "columns": [{"name": "start", "type": "string"}], "rows": [[start]]}]
+ }
+ )
+
+
+def _where_lines(query: str) -> list[str]:
+ return [line for line in query.splitlines() if line.startswith("| where")]
+
+
+def _window_end(query: str) -> str:
+ return next(line for line in query.splitlines() if line.startswith("| where ingestion_time() < "))[
+ len("| where ingestion_time() < datetime(") : -1
+ ]
+
+
+def _take(query: str) -> str:
+ return next(line for line in query.splitlines() if line.startswith("| take "))
+
+
+class TestQueryWalk:
+ @pytest.fixture(autouse=True)
+ def _frozen_clock(self) -> Iterator[None]:
+ with time_machine.travel(NOW, tick=False):
+ yield
+
+ def test_full_refresh_walks_an_exact_cursor_and_stages_every_page(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(azure_log_analytics, "PAGE_SIZE", 2)
+ # Both rows of the first page share one ingestion time, so only _ItemId separates them.
+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(
+ "AppTraces",
+ route(
+ {
+ TOKEN_PATH: [_token()],
+ QUERY_PATH: [
+ _min_start("2026-10-01T23:00:00.1234567Z"),
+ _page([_row("2026-10-01T23:00:00.1234567Z", "a"), _row("2026-10-01T23:00:00.1234567Z", "b")]),
+ _page([_row("2026-10-01T23:59:59.9999999Z", "c")]),
+ _page([_row("2026-10-02T08:00:00Z", "d")]),
+ ],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ queries = _queries(result.requests)
+ assert queries[0] == MIN_QUERY
+ assert [_where_lines(query) for query in queries[1:]] == [
+ [
+ "| where TimeGenerated >= datetime(2026-10-01T23:00:00.1234567Z) - 3d",
+ "| where ingestion_time() >= datetime(2026-10-01T23:00:00.1234567Z)",
+ "| where ingestion_time() < datetime(2026-10-02T00:00:00Z)",
+ ],
+ [
+ "| where TimeGenerated >= datetime(2026-10-01T23:00:00.1234567Z) - 3d",
+ "| where ingestion_time() > datetime(2026-10-01T23:00:00.1234567Z) or "
+ "(ingestion_time() == datetime(2026-10-01T23:00:00.1234567Z) and strcmp(_ItemId, 'b') > 0)",
+ "| where ingestion_time() < datetime(2026-10-02T00:00:00Z)",
+ ],
+ [
+ "| where TimeGenerated >= datetime(2026-10-02T00:00:00Z) - 3d",
+ "| where ingestion_time() >= datetime(2026-10-02T00:00:00Z)",
+ f"| where ingestion_time() < datetime({RUN_END})",
+ ],
+ ]
+ assert all(
+ "| sort by _IngestionTime asc, _ItemId asc\n| take 2\n| extend _ItemId = _ItemId" in query
+ for query in queries[1:]
+ )
+ assert [row["_ItemId"] for row in result.rows] == ["a", "b", "c", "d"]
+ assert all("__posthog_cursor" not in row for row in result.rows)
+ assert result.saved_states == [
+ AzureLogAnalyticsResumeConfig(ingested_at="2026-10-01T23:00:00.1234567Z", item_id="b"),
+ AzureLogAnalyticsResumeConfig(ingested_at="2026-10-02T00:00:00Z"),
+ AzureLogAnalyticsResumeConfig(ingested_at=RUN_END),
+ ]
+ assert result.committed_states == result.saved_states
+ assert result.completed_state is None
+
+ def test_a_full_page_narrows_the_rest_of_that_day_to_hour_windows(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(azure_log_analytics, "PAGE_SIZE", 2)
+ answers = iter([_page([_row("2026-10-01T22:40:00Z", "a"), _row("2026-10-01T22:41:00Z", "b")])])
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == TOKEN_PATH:
+ return _token()
+ return next(answers, _page([]))
+
+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(
+ "AppTraces",
+ respond,
+ incremental_field="_IngestionTime",
+ db_incremental_field_last_value=datetime(2026, 10, 1, 22, 30, tzinfo=UTC),
+ )
+
+ assert result.raised is None
+ assert [_window_end(query) for query in _queries(result.requests)] == [
+ "2026-10-02T00:00:00Z",
+ "2026-10-01T23:00:00Z",
+ "2026-10-02T00:00:00Z",
+ RUN_END,
+ ]
+
+ def test_empty_days_advance_the_window_without_rows(self) -> None:
+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(
+ "AppTraces",
+ route(
+ {
+ TOKEN_PATH: [_token()],
+ QUERY_PATH: [_page([]), _page([]), _page([_row("2026-10-02T01:00:00Z", "z")])],
+ }
+ ),
+ incremental_field="_IngestionTime",
+ db_incremental_field_last_value=datetime(2026, 9, 30, 23, 0, tzinfo=UTC),
+ )
+
+ assert result.raised is None
+ assert [_window_end(query) for query in _queries(result.requests)] == [
+ "2026-10-01T00:00:00Z",
+ "2026-10-02T00:00:00Z",
+ RUN_END,
+ ]
+ assert _where_lines(_queries(result.requests)[0])[:2] == [
+ "| where TimeGenerated >= datetime(2026-09-30T23:00:00.000000Z) - 3d",
+ "| where ingestion_time() >= datetime(2026-09-30T23:00:00.000000Z)",
+ ]
+ assert [row["_ItemId"] for row in result.rows] == ["z"]
+
+ @pytest.mark.parametrize(
+ "inputs",
+ [
+ {},
+ {
+ "incremental_field": "_IngestionTime",
+ "should_use_incremental_field": False,
+ "db_incremental_field_last_value": datetime(2026, 10, 2, tzinfo=UTC),
+ },
+ ],
+ )
+ def test_full_refresh_starts_at_the_oldest_row_and_stops_on_an_empty_table(self, inputs: dict[str, Any]) -> None:
+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(
+ "AppTraces", route({TOKEN_PATH: [_token()], QUERY_PATH: [_min_start("")]}), **inputs
+ )
+
+ assert result.raised is None
+ assert _queries(result.requests) == [MIN_QUERY]
+ assert result.rows == []
+
+ def test_resume_starts_after_the_saved_row_and_ignores_the_watermark(self) -> None:
+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(
+ "AppTraces",
+ route({TOKEN_PATH: [_token()], QUERY_PATH: [_page([_row("2026-10-02T11:00:00Z", "z")])]}),
+ resume_state=AzureLogAnalyticsResumeConfig(ingested_at="2026-10-02T10:59:59.9999999Z", item_id="y"),
+ incremental_field="_IngestionTime",
+ db_incremental_field_last_value=datetime(2026, 10, 1, tzinfo=UTC),
+ )
+
+ assert result.raised is None
+ (query,) = _queries(result.requests)
+ assert "strcmp(_ItemId, 'y') > 0" in query
+ assert "ingestion_time() >= datetime(" not in query
+
+ def test_a_page_over_the_size_limit_is_read_again_smaller_then_the_size_resets(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(azure_log_analytics, "PAGE_SIZE", 400)
+ too_large = ScriptedResponse(
+ json={
+ "tables": [{"name": "PrimaryResult", "columns": COLUMNS, "rows": [_row("2026-10-01T22:10:00Z", "a")]}],
+ "error": {
+ "code": "PartialError",
+ "message": "Query result set has exceeded the internal data size limit",
+ },
+ }
+ )
+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(
+ "AppTraces",
+ route(
+ {
+ TOKEN_PATH: [_token()],
+ QUERY_PATH: [too_large, _page([_row("2026-10-01T22:10:00Z", "a")]), _page([]), _page([])],
+ }
+ ),
+ incremental_field="_IngestionTime",
+ db_incremental_field_last_value="2026-10-01T22:00:00Z",
+ )
+
+ assert result.raised is None
+ queries = _queries(result.requests)
+ assert [_take(query) for query in queries] == ["| take 400", "| take 200", "| take 400", "| take 400"]
+ assert [_window_end(query) for query in queries] == [
+ "2026-10-02T00:00:00Z",
+ "2026-10-01T23:00:00Z",
+ "2026-10-02T00:00:00Z",
+ RUN_END,
+ ]
+ assert [row["_ItemId"] for row in result.rows] == ["a"]
+
+ def test_a_partial_result_at_the_smallest_page_fails_the_run(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(azure_log_analytics, "PAGE_SIZE", 100)
+ partial = ScriptedResponse(
+ json={
+ "tables": [{"name": "PrimaryResult", "columns": COLUMNS, "rows": [_row("2026-10-02T10:00:00Z", "a")]}],
+ "error": {"code": "PartialError", "message": "There were some errors when processing your query."},
+ }
+ )
+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(
+ "AppTraces", route({TOKEN_PATH: [_token()], QUERY_PATH: [partial]}), **WATERMARK_TODAY
+ )
+
+ assert result.rows == []
+ assert result.raised is not None and "PartialError" in str(result.raised)
+
+ def test_expired_token_is_reminted_once(self) -> None:
+ expired = ScriptedResponse(
+ status=401, json={"error": {"code": "TokenExpired", "message": "The provided authentication has expired"}}
+ )
+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run(
+ "AppTraces",
+ route(
+ {TOKEN_PATH: [_token(), _token()], QUERY_PATH: [expired, _page([_row("2026-10-02T10:00:00Z", "a")])]}
+ ),
+ **WATERMARK_TODAY,
+ )
+
+ assert result.raised is None
+ assert result.paths == [TOKEN_PATH, QUERY_PATH, TOKEN_PATH, QUERY_PATH]
+ assert len(result.rows) == 1
+
+ def test_a_dropped_connection_is_retried_within_the_budget(self) -> None:
+ answers: Iterator[ScriptedResponse | BaseException] = iter(
+ [ConnectionResetError(), _page([_row("2026-10-02T10:00:00Z", "a")])]
+ )
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == TOKEN_PATH:
+ return _token()
+ answer = next(answers)
+ if isinstance(answer, BaseException):
+ raise answer
+ return answer
+
+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run("AppTraces", respond, **WATERMARK_TODAY)
+
+ assert result.raised is None
+ assert len(_queries(result.requests)) == 2
+ assert [row["_ItemId"] for row in result.rows] == ["a"]
+
+ def test_retries_stop_before_a_full_attempt_would_overrun_five_minutes(self) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == TOKEN_PATH:
+ return _token()
+ return ScriptedResponse(
+ status=504, json={"error": {"code": "GatewayTimeout"}}, headers={"Retry-After": "30"}
+ )
+
+ result = SourceDriver(AzureLogAnalyticsSource(), _config()).run("AppTraces", respond, **WATERMARK_TODAY)
+
+ assert isinstance(result.raised, AzureLogAnalyticsRetryableError)
+ # Waits at 0 s and 30 s leave room for a full 230 s attempt; a third retry at 60 s does not.
+ assert len(_queries(result.requests)) == 3
+
+ def test_meets_the_source_contract_on_an_incremental_table(self) -> None:
+ def start(inputs: SourceInputs) -> tuple[SourceResponse, ResumableSourceManager[Any]]:
+ inputs.incremental_field = "_IngestionTime"
+ inputs.should_use_incremental_field = True
+ inputs.db_incremental_field_last_value = datetime(2026, 9, 20, tzinfo=UTC)
+ source = AzureLogAnalyticsSource()
+ manager = source.get_resumable_source_manager(inputs)
+ return source.source_for_pipeline(_config(), manager, inputs), manager
+
+ verdict = check_extraction(start, ["AppTraces"])
+
+ assert verdict.failed == frozenset()
+ assert verdict.safe_point_exercised
+
+
+class TestRowShaping:
+ @pytest.mark.parametrize(
+ ("column_type", "value", "expected"),
+ [
+ ("datetime", "2026-10-01T10:00:00.1234567Z", datetime(2026, 10, 1, 10, 0, 0, 123456, tzinfo=UTC)),
+ ("dynamic", '{"a": [1, 2]}', {"a": [1, 2]}),
+ ("dynamic", "not json", "not json"),
+ ("bool", "True", True),
+ ("bool", 0, False),
+ ("long", 7, 7),
+ ("string", None, None),
+ ],
+ )
+ def test_converts_values_by_column_type(self, column_type: str, value: Any, expected: Any) -> None:
+ payload = {"tables": [{"columns": [{"name": "c", "type": column_type}], "rows": [[value]]}]}
+
+ assert rows_from_response(payload) == [{"c": expected}]
+
+
+class TestInputs:
+ @pytest.mark.parametrize(
+ ("raw", "expected"),
+ [
+ ("AppTraces\n\nMyEvents_CL\nAppTraces", ["AppTraces", "MyEvents_CL"]),
+ ("AppTraces, AppRequests", ["AppTraces", "AppRequests"]),
+ (None, []),
+ ],
+ )
+ def test_parses_one_table_per_line(self, raw: str | None, expected: list[str]) -> None:
+ assert parse_table_names(raw) == expected
+
+ @pytest.mark.parametrize("raw", ["AppTraces | union SecurityEvent", "App-Traces", "1Table"])
+ def test_rejects_table_names_that_are_not_plain_identifiers(self, raw: str) -> None:
+ with pytest.raises(ValueError):
+ parse_table_names(raw)
+
+ @pytest.mark.parametrize(
+ "resume",
+ [
+ AzureLogAnalyticsResumeConfig(ingested_at="2026-10-01) | union X", item_id="a"),
+ AzureLogAnalyticsResumeConfig(ingested_at="2026-10-01T10:00:00Z", item_id="a') or true or ('"),
+ ],
+ )
+ def test_rejects_a_resume_cursor_that_would_change_the_query(self, resume: AzureLogAnalyticsResumeConfig) -> None:
+ with pytest.raises(ValueError):
+ build_query("AppTraces", resume, "2026-10-02T00:00:00Z", 10)
+
+
+class TestValidateCredentials:
+ @pytest.mark.parametrize(
+ ("query_response", "table", "expected_ok", "expected_fragment"),
+ [
+ (_page([]), None, True, None),
+ (
+ ScriptedResponse(
+ status=403,
+ json={
+ "error": {
+ "code": "InsufficientAccessError",
+ "message": "The provided credentials have insufficient access to perform the requested operation",
+ }
+ },
+ ),
+ None,
+ False,
+ "Log Analytics Reader",
+ ),
+ (
+ ScriptedResponse(
+ status=404,
+ json={"error": {"code": "WorkspaceNotFoundError", "message": "Workspace could not be found"}},
+ ),
+ None,
+ False,
+ "No Log Analytics workspace",
+ ),
+ (
+ ScriptedResponse(
+ status=400,
+ json={
+ "error": {
+ "code": "BadArgumentError",
+ "message": "The request had some invalid properties",
+ "innererror": {
+ "code": "SemanticError",
+ "message": "A semantic error occurred.",
+ "innererror": {
+ "code": "SEM0100",
+ "message": "'take' operator: Failed to resolve table or column expression named 'NoSuchTable'",
+ },
+ },
+ }
+ },
+ ),
+ "NoSuchTable",
+ False,
+ "no table named NoSuchTable",
+ ),
+ ],
+ )
+ def test_maps_the_workspace_probe_to_a_message(
+ self, query_response: ScriptedResponse, table: str | None, expected_ok: bool, expected_fragment: str | None
+ ) -> None:
+ with scripted_network(route({TOKEN_PATH: [_token()], QUERY_PATH: [query_response]})) as network:
+ ok, message = AzureLogAnalyticsSource().validate_credentials(
+ _config(tables=None), team_id=1, schema_name=table
+ )
+
+ assert ok is expected_ok
+ if expected_fragment is None:
+ assert message is None
+ else:
+ assert message is not None and expected_fragment in message
+ probe = network.requests_log[-1].json()["query"]
+ assert probe == (f"{table} | take 0" if table else "print ok = 1")
+
+ def test_rejected_secret_stops_before_the_workspace_probe(self) -> None:
+ rejected = ScriptedResponse(
+ status=401,
+ json={"error": "invalid_client", "error_description": "AADSTS7000215: Invalid client secret provided."},
+ )
+ with scripted_network(route({TOKEN_PATH: [rejected]})) as network:
+ ok, message = AzureLogAnalyticsSource().validate_credentials(_config(), team_id=1)
+
+ assert ok is False
+ assert message is not None and "Microsoft Entra ID" in message
+ assert [request.path for request in network.requests_log] == [TOKEN_PATH]
+
+ @pytest.mark.parametrize(
+ ("answers", "expected_ok", "expected_fragment", "expected_probes"),
+ [
+ ([_missing("AppTrace"), _missing("MyEvents_CL")], False, "None of the listed tables exist", 2),
+ (
+ [
+ ScriptedResponse(status=400, json={"error": {"code": "BadArgumentError", "message": "Basic plan"}}),
+ ScriptedResponse(status=403, json={"error": {"code": "InsufficientAccessError", "message": "No"}}),
+ ],
+ False,
+ "None of the listed tables can be read",
+ 2,
+ ),
+ ([_missing("AppTrace"), _page([])], True, None, 2),
+ ([_page([])], True, None, 1),
+ ],
+ )
+ def test_creation_needs_one_readable_listed_table(
+ self, answers: list[ScriptedResponse], expected_ok: bool, expected_fragment: str | None, expected_probes: int
+ ) -> None:
+ with scripted_network(route({TOKEN_PATH: [_token(), _token()], QUERY_PATH: [_page([]), *answers]})) as network:
+ ok, message = AzureLogAnalyticsSource().validate_credentials(
+ _config(tables="AppTrace\nMyEvents_CL"), team_id=1
+ )
+
+ assert ok is expected_ok
+ if expected_fragment is None:
+ assert message is None
+ else:
+ assert message is not None and expected_fragment in message
+ queries = [request.json()["query"] for request in network.requests_log if request.path == QUERY_PATH]
+ assert queries == ["print ok = 1", "AppTrace | take 0", "MyEvents_CL | take 0"][: 1 + expected_probes]
+
+ def test_setup_probes_use_short_deadlines_and_no_retries(self) -> None:
+ unavailable = ScriptedResponse(
+ status=504, json={"error": {"code": "GatewayTimeout"}}, headers={"Retry-After": "1"}
+ )
+ with scripted_network(route({TOKEN_PATH: [_token()], QUERY_PATH: [unavailable]})) as network:
+ ok, message = AzureLogAnalyticsSource().validate_credentials(_config(), team_id=1)
+
+ assert ok is False
+ assert message is not None and "Could not query the Log Analytics workspace" in message
+ queries = [request for request in network.requests_log if request.path == QUERY_PATH]
+ assert len(queries) == 1
+ assert queries[0].read_timeout == 30
+
+ @pytest.mark.parametrize(
+ ("config", "schema_name"),
+ [(_config(tables="AppTraces | union SecurityEvent"), None), (_config(), "AppTraces | union SecurityEvent")],
+ )
+ def test_rejects_a_table_name_that_would_change_the_query_without_a_request(
+ self, config: AzureLogAnalyticsSourceConfig, schema_name: str | None
+ ) -> None:
+ with scripted_network([]) as network:
+ ok, message = AzureLogAnalyticsSource().validate_credentials(config, team_id=1, schema_name=schema_name)
+
+ assert ok is False
+ assert message is not None and "not a valid Log Analytics table name" in message
+ assert network.requests_log == []
+
+
+class TestSchemaDiscovery:
+ def test_blank_table_list_discovers_tables_from_usage(self) -> None:
+ usage = ScriptedResponse(
+ json={
+ "tables": [
+ {
+ "name": "PrimaryResult",
+ "columns": [{"name": "DataType", "type": "string"}],
+ "rows": [["AppRequests"], ["MyEvents_CL"], ["not a table"]],
+ }
+ ]
+ }
+ )
+ with scripted_network(route({TOKEN_PATH: [_token()], QUERY_PATH: [usage]})) as network:
+ schemas = AzureLogAnalyticsSource().get_schemas(_config(tables=None), team_id=1)
+
+ assert [schema.name for schema in schemas] == ["AppRequests", "MyEvents_CL"]
+ assert network.requests_log[-1].json()["query"].startswith("Usage | where TimeGenerated > ago(30d)")
+
+ def test_endpoint_permissions_flag_tables_the_query_api_cannot_read(self) -> None:
+ basic_plan = ScriptedResponse(
+ status=400,
+ json={
+ "error": {
+ "code": "BadArgumentError",
+ "message": "The request had some invalid properties",
+ "innererror": {"code": "SemanticError", "message": "Table 'BasicLogs_CL' has the Basic plan."},
+ }
+ },
+ )
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == TOKEN_PATH:
+ return _token()
+ query = request.json()["query"]
+ if query.startswith("BasicLogs_CL"):
+ return basic_plan
+ if query.startswith("Missing_CL"):
+ return ScriptedResponse(
+ status=400,
+ json={
+ "error": {
+ "code": "BadArgumentError",
+ "message": "The request had some invalid properties",
+ "innererror": {
+ "code": "SemanticError",
+ "message": "'take' operator: Failed to resolve table or column expression named 'Missing_CL'",
+ },
+ }
+ },
+ )
+ if query.startswith("Restricted"):
+ return ScriptedResponse(
+ status=403, json={"error": {"code": "InsufficientAccessError", "message": "Insufficient access"}}
+ )
+ if query.startswith("Throttled"):
+ return ScriptedResponse(status=429, json={}, headers={"Retry-After": "300"})
+ return _page([])
+
+ with scripted_network(respond) as network:
+ permissions = AzureLogAnalyticsSource().get_endpoint_permissions(
+ _config(),
+ team_id=1,
+ endpoints=["AppTraces", "BasicLogs_CL", "Missing_CL", "Restricted", "Throttled", "Later_CL"],
+ )
+
+ assert permissions["AppTraces"] is None
+ assert permissions["BasicLogs_CL"] is not None and "Basic and Auxiliary" in permissions["BasicLogs_CL"]
+ assert permissions["Missing_CL"] is not None and "doesn't exist in the workspace" in permissions["Missing_CL"]
+ assert permissions["Restricted"] is not None and "can't read this table" in permissions["Restricted"]
+ # A throttle counts as readable and ends probing, without retries, so setup stays fast.
+ assert permissions["Throttled"] is None and permissions["Later_CL"] is None
+ probed = [r.json()["query"].split(" ")[0] for r in network.requests_log if r.path == QUERY_PATH]
+ assert probed == ["AppTraces", "BasicLogs_CL", "Missing_CL", "Restricted", "Throttled"]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureloganalytics.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureloganalytics.py
index ebeaa64d6cd0..37ea8cb439ce 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureloganalytics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/azureloganalytics.py
@@ -6,4 +6,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common imp
@config.config
class AzureLogAnalyticsSourceConfig(config.Config):
- pass
+ tenant_id: str
+ client_id: str
+ client_secret: str
+ workspace_id: str
+ tables: str | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
index 74cb057e878d..c58cb0b3a0d8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
@@ -12,6 +12,7 @@ AwsAthena not-checkable:schemas
AwsCostExplorer fail:safe-point
AzureBlob not-checkable:schemas
AzureCostManagement not-checkable:no-request
+AzureLogAnalytics not-checkable:schemas
Baserow not-checkable:schemas
BigMailer fail:safe-point
BigQuery not-checkable:driver:google.cloud
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py
index f9ab0a124c5d..32f28988add4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py
@@ -81,6 +81,7 @@ NOT_CHECKABLE: dict[str, str] = {
"AwsAthena": "schemas",
"AzureBlob": "schemas",
"AzureCostManagement": "no-request",
+ "AzureLogAnalytics": "no-request",
"Baserow": "schemas",
"BigQuery": "driver",
"BingAds": "database",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py
index e38ff3943aba..9a0fc71c1a2d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_schema_contract.py
@@ -104,6 +104,7 @@ UNKNOWN_SCHEMA_NOT_CHECKABLE: dict[str, str] = {
"Attentive": "webhook",
"AwsAthena": "request",
"AzureBlob": "request",
+ "AzureLogAnalytics": "request",
"Baserow": "request",
"BigQuery": "driver",
"BingAds": "database",